Merge remote-tracking branch 'remotes/open/master' into merges/april-27-09-38

# Conflicts:
#	docs/source/changelog.rst
#	node-api/src/main/kotlin/net/corda/nodeapi/internal/config/ConfigUtilities.kt
#	node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

.idea/compiler.xml

@@ -10,6 +10,7 @@
       <module name="bank-of-corda-demo_integrationTest" target="1.8" />
       <module name="bank-of-corda-demo_main" target="1.8" />
       <module name="bank-of-corda-demo_test" target="1.8" />
+      <module name="behave_behave" target="1.8" />
       <module name="behave_main" target="1.8" />
       <module name="behave_scenario" target="1.8" />
       <module name="behave_test" target="1.8" />

docs/source/changelog.rst

@@ -5,7 +5,17 @@ Here's a summary of what's changed in each Corda release. For guidance on how to
 release, see :doc:`upgrade-notes`.
 
 Unreleased
-----------
+==========
+
+* Added program line argument ``on-unknown-config-keys`` to allow specifying behaviour on unknown node configuration property keys.
+  Values are: [FAIL, WARN, IGNORE], default to FAIL if unspecified.
+
+* Fix CORDA-1229. Setter-based serialization was broken with generic types when the property was stored
+  as the raw type, List for example.
+
+* java.security.cert.CRLReason added to the default Whitelist.
+
+* java.security.cert.X509CRL serialization support added.
 
 * Upgraded H2 to v1.4.197.
 
@@ -32,7 +42,7 @@ Unreleased
 * Update the fast-classpath-scanner dependent library version from 2.0.21 to 2.12.3
 
   .. note:: Whilst this is not the latest version of this library, that being 2.18.1 at time of writing, versions later
-  than 2.12.3 (including 2.12.4) exhibit a different issue.
+than 2.12.3 (including 2.12.4) exhibit a different issue.
 
 * Fixed node's behaviour on startup when there is no connectivity to network map. Node continues to work normally if it has
   all the needed network data, waiting in the background for network map to become available.

docs/source/corda-configuration-file.rst

@@ -31,6 +31,9 @@ e.g.:
 The property `"dataSourceProperties.dataSourceClassName" = "val"` in ``reference.conf``
 would be not overwritten by the property `dataSourceProperties.dataSourceClassName = "val2"` in ``node.conf``.
 
+By default the node will fail to start in presence of unknown property keys. To alter this behaviour, program line argument
+``on-unknown-config-keys`` can be set to ``WARN`` or ``IGNORE``. Default is ``FAIL`` if unspecified.
+
 Defaults
 --------
 A set of default configuration options are loaded from the built-in resource file ``/node/src/main/resources/reference.conf``.

docs/source/getting-set-up.rst

@@ -57,7 +57,8 @@ Java
 3. Toggle "Accept License Agreement"
 4. Click the download link for jdk-8uXXX-windows-x64.exe (where "XXX" is the latest minor version number)
 5. Download and run the executable to install Java (use the default settings)
-6. Open a new command prompt and run ``java -version`` to test that Java is installed correctly
+6. Add Java to the PATH environment variable by following the instructions at https://docs.oracle.com/javase/7/docs/webnotes/install/windows/jdk-installation-windows.html#path
+7. Open a new command prompt and run ``java -version`` to test that Java is installed correctly
 
 Git
 ^^^

experimental/behave/build.gradle

@@ -39,19 +39,19 @@ repositories {
 }
 
 sourceSets {
-    scenario {
+    behave {
         java {
             compileClasspath += main.output
             runtimeClasspath += main.output
-            srcDir file('src/scenario/kotlin')
+            srcDirs = ["src/main/kotlin", "src/scenario/kotlin"]
         }
         resources.srcDir file('src/scenario/resources')
     }
 }
 
 configurations {
-    scenarioCompile.extendsFrom testCompile
+    behaveCompile.extendsFrom testCompile
-    scenarioRuntime.extendsFrom testRuntime
+    behaveRuntime.extendsFrom testRuntime
 }
 
 dependencies {
@@ -98,9 +98,9 @@ dependencies {
 
     // Scenarios / End-to-End Tests
 
-    scenarioCompile "info.cukes:cucumber-java8:$cucumber_version"
+    behaveCompile "info.cukes:cucumber-java8:$cucumber_version"
-    scenarioCompile "info.cukes:cucumber-junit:$cucumber_version"
+    behaveCompile "info.cukes:cucumber-junit:$cucumber_version"
-    scenarioCompile "info.cukes:cucumber-picocontainer:$cucumber_version"
+    behaveCompile "info.cukes:cucumber-picocontainer:$cucumber_version"
 }
 
 compileKotlin {
@@ -111,24 +111,25 @@ compileTestKotlin {
     kotlinOptions.jvmTarget = "1.8"
 }
 
-compileScenarioKotlin {
-    kotlinOptions.jvmTarget = "1.8"
-}
-
 test {
     testLogging.showStandardStreams = true
 }
 
-task scenarios(type: Test) {
+task behaveJar(type: Jar) {
-    setTestClassesDirs sourceSets.scenario.output.getClassesDirs()
+    baseName "corda-behave"
-    classpath = sourceSets.scenario.runtimeClasspath
+    from sourceSets.behave.output
-    outputs.upToDateWhen { false }
+    from {
-
+        configurations.behaveCompile.collect {
-    if (project.hasProperty("tags")) {
+            it.isDirectory() ? it : zipTree(it)
-        systemProperty "cucumber.options", "--tags $tags"
+        }
-        logger.warn("Only running tests tagged with: $tags ...")
+    }
+    zip64 true
+    exclude("features/**")
+    exclude("scripts/**")
+    exclude("META-INF/*.DSA")
+    exclude("META-INF/*.RSA")
+    exclude("META-INF/*.SF")
+    manifest {
+        attributes 'Main-Class': 'net.corda.behave.scenarios.ScenarioRunner'
     }
 }
-
-//scenarios.mustRunAfter test
-//scenarios.dependsOn test

experimental/behave/prepare.sh

@@ -6,26 +6,30 @@ set -x
 # For example:
 #   corda-master   => git clone https://github.com/corda/corda
 #   r3corda-master => git clone https://github.com/corda/enterprise
-VERSION=corda-3.0
+VERSION=corda-master
-STAGING_DIR=deps/corda/${VERSION}
+STAGING_DIR=~/staging
-DRIVERS_DIR=deps/drivers
+CORDA_DIR=${STAGING_DIR}/corda/${VERSION}
+CORDAPP_DIR=${CORDA_DIR}/apps
+DRIVERS_DIR=${STAGING_DIR}/drivers
 
 # Set up directories
-mkdir -p ${STAGING_DIR}/apps
+mkdir -p ${STAGING_DIR}
+mkdir -p ${CORDA_DIR}
+mkdir -p ${CORDAPP_DIR}
 mkdir -p ${DRIVERS_DIR}
 
 # Copy Corda capsule into deps
 cd ../..
 ./gradlew clean :node:capsule:buildCordaJar :finance:jar
-cp -v $(ls node/capsule/build/libs/corda-*.jar | tail -n1) experimental/behave/${STAGING_DIR}/corda.jar
+cp -v $(ls node/capsule/build/libs/corda-*.jar | tail -n1) ${CORDA_DIR}/corda.jar
 
 # Copy finance library
-cp -v $(ls finance/build/libs/corda-finance-*.jar | tail -n1) experimental/behave/${STAGING_DIR}/apps
+cp -v $(ls finance/build/libs/corda-finance-*.jar | tail -n1) ${CORDAPP_DIR}
 
 # Download database drivers
-curl "https://search.maven.org/remotecontent?filepath=com/h2database/h2/1.4.196/h2-1.4.196.jar" > experimental/behave/${DRIVERS_DIR}/h2-1.4.196.jar
+curl "https://search.maven.org/remotecontent?filepath=com/h2database/h2/1.4.196/h2-1.4.196.jar" > ${DRIVERS_DIR}/h2-1.4.196.jar
-curl -L "http://central.maven.org/maven2/org/postgresql/postgresql/42.1.4/postgresql-42.1.4.jar" > experimental/behave/${DRIVERS_DIR}/postgresql-42.1.4.jar
+curl -L "http://central.maven.org/maven2/org/postgresql/postgresql/42.1.4/postgresql-42.1.4.jar" > ${DRIVERS_DIR}/postgresql-42.1.4.jar
 
 # Build Network Bootstrapper
 ./gradlew buildBootstrapperJar
-cp -v $(ls tools/bootstrapper/build/libs/*.jar | tail -n1) experimental/behave/${STAGING_DIR}/network-bootstrapper.jar
+cp -v $(ls tools/bootstrapper/build/libs/*.jar | tail -n1) ${CORDA_DIR}/network-bootstrapper.jar

experimental/behave/src/main/kotlin/net/corda/behave/network/Network.kt

@@ -103,11 +103,8 @@ class Network private constructor(
 
     fun copyDatabaseDrivers() {
         val driverDirectory = (targetDirectory / "libs").createDirectories()
-        log.info("Copying database drivers from $stagingRoot/deps/drivers to $driverDirectory")
+        log.info("Copying database drivers from $stagingRoot/drivers to $driverDirectory")
-        FileUtils.copyDirectory(
+        FileUtils.copyDirectory((stagingRoot / "drivers").toFile(), driverDirectory.toFile())
-                (stagingRoot / "deps" / "drivers").toFile(),
-                driverDirectory.toFile()
-        )
     }
 
     fun configureNodes(): Boolean {

experimental/behave/src/main/kotlin/net/corda/behave/node/Distribution.kt

@@ -95,7 +95,7 @@ class Distribution private constructor(
 
         private val distributions = mutableListOf<Distribution>()
 
-        private val nodePrefix = stagingRoot / "deps/corda"
+        private val nodePrefix = stagingRoot / "corda"
 
         val MASTER = fromJarFile("corda-master")
 

experimental/behave/src/main/kotlin/net/corda/behave/node/Node.kt

@@ -230,7 +230,7 @@ class Node(
 
     private fun installApps() {
         val version = config.distribution.version
-        val appDirectory = stagingRoot / "deps" / "corda" / version / "apps"
+        val appDirectory = stagingRoot / "corda" / version / "apps"
         if (appDirectory.exists()) {
             val targetAppDirectory = runtimeDirectory / "cordapps"
             FileUtils.copyDirectory(appDirectory.toFile(), targetAppDirectory.toFile())

experimental/behave/src/main/kotlin/net/corda/behave/node/configuration/Configuration.kt

@@ -55,7 +55,7 @@ class Configuration(
 
     fun writeToFile(file: Path) {
         file.writeText(this.generate())
-        log.info(this.generate())
+        log.debug(this.generate())
     }
 
     private fun generate() = listOf(basicConfig, database.config(), extraConfig)

experimental/behave/src/main/kotlin/net/corda/behave/process/JarCommand.kt

@@ -10,13 +10,12 @@
 
 package net.corda.behave.process
 
-import java.io.File
 import java.nio.file.Path
 import java.time.Duration
 
 class JarCommand(
         jarFile: Path,
-        arguments: Array<String>,
+        arguments: Array<out String>,
         directory: Path,
         timeout: Duration,
         enableRemoteDebugging: Boolean = false

experimental/behave/src/scenario/kotlin/net/corda/behave/scenarios/helpers/Cash.kt

@@ -27,9 +27,6 @@ class Cash(state: ScenarioState) : Substeps(state) {
 
     fun numberOfIssuableCurrencies(nodeName: String): Int {
         return withClient(nodeName) {
-            for (flow in it.registeredFlows()) {
-                log.info(flow)
-            }
             try {
                 val config = it.startFlow(::CashConfigDataFlow).returnValue.get(10, TimeUnit.SECONDS)
                 for (supportedCurrency in config.supportedCurrencies) {

experimental/behave/src/scenario/kotlin/net/corda/behave/scenarios/helpers/Database.kt

@@ -12,7 +12,7 @@ package net.corda.behave.scenarios.helpers
 
 import net.corda.behave.await
 import net.corda.behave.scenarios.ScenarioState
-import net.corda.behave.seconds
+import net.corda.core.utilities.seconds
 import org.assertj.core.api.Assertions.assertThat
 import java.util.concurrent.CountDownLatch
 

experimental/behave/src/scenario/kotlin/net/corda/behave/scenarios/helpers/Startup.kt

@@ -10,17 +10,17 @@
 
 package net.corda.behave.scenarios.helpers
 
-import net.corda.behave.minutes
 import net.corda.behave.process.JarCommand
 import net.corda.behave.scenarios.ScenarioState
 import net.corda.core.internal.div
+import net.corda.core.utilities.minutes
 
 class Startup(state: ScenarioState) : Substeps(state) {
 
     fun hasLoggingInformation(nodeName: String) {
         withNetwork {
             log.info("Retrieving logging information for node '$nodeName' ...")
-            if (!node(nodeName).nodeInfoGenerationOutput.find("Logs can be found in.*").any()) {
+            if (!node(nodeName).logOutput.find("Logs can be found in.*").any()) {
                 fail("Unable to find logging information for node $nodeName")
             }
 
@@ -36,7 +36,7 @@ class Startup(state: ScenarioState) : Substeps(state) {
     fun hasDatabaseDetails(nodeName: String) {
         withNetwork {
             log.info("Retrieving database details for node '$nodeName' ...")
-            if (!node(nodeName).nodeInfoGenerationOutput.find("Database connection url is.*").any()) {
+            if (!node(nodeName).logOutput.find("Database connection url is.*").any()) {
                 fail("Unable to find database details for node $nodeName")
             }
         }
@@ -114,7 +114,7 @@ class Startup(state: ScenarioState) : Substeps(state) {
             val cordappDirectory = node(nodeName).config.distribution.cordappDirectory
             val cordappJar = cordappDirectory / "$cordapp.jar"
             // Execute
-            val command = JarCommand(cordappJar, args as Array<String>, cordappDirectory, 1.minutes)
+            val command = JarCommand(cordappJar, args, cordappDirectory, 1.minutes)
             command.start()
             if (!command.waitFor())
                 fail("Failed to successfully run the CorDapp jar: $cordaApp")

experimental/behave/src/scenario/kotlin/net/corda/behave/scenarios/steps/VaultSteps.kt

@@ -21,6 +21,7 @@ class VaultSteps : StepsBlock {
 
         Then<String, Int, String>("^node (\\w+) vault contains (\\d+) (\\w+) states$") { node, count, contractType ->
             try {
+                @Suppress("UNCHECKED_CAST")
                 val contractStateTypeClass = Class.forName(contractType) as Class<ContractState>
                 if (vault.query(node, contractStateTypeClass).size == count)
                     succeed()
@@ -33,7 +34,7 @@ class VaultSteps : StepsBlock {
 
         Then<String, Long, String>("^node (\\w+) vault contains total cash of (\\d+) (\\w+)$") { node, total, currency ->
             val cashStates = vault.query(node, Cash.State::class.java)
-            val sumCashStates = cashStates.filter { it.state.data.amount.token.product.currencyCode == currency }?.sumByLong { it.state.data.amount.quantity }
+            val sumCashStates = cashStates.filter { it.state.data.amount.token.product.currencyCode == currency }.sumByLong { it.state.data.amount.quantity }
             print((sumCashStates))
             if (sumCashStates == total)
                 succeed()

experimental/behave/src/scenario/resources/features/cash/currencies.feature

@@ -2,22 +2,34 @@
 Feature: Cash - Issuable Currencies
   To have cash on ledger, certain nodes must have the ability to issue cash of various currencies.
 
-  Scenario: Node can issue no currencies by default
+  Scenario Outline: Node can issue no currencies by default
-    Given a node PartyA of version master
+    Given a node PartyA of version <Node-Version>
     And node PartyA has the finance app installed
     When the network is ready
     Then node PartyA has 0 issuable currencies
 
-  Scenario: Node has an issuable currency
+    Examples:
-    Given a node PartyA of version master
+      | Node-Version    |
+      | master          |
+
+  Scenario Outline: Node has an issuable currency
+    Given a node PartyA of version <Node-Version>
     And node PartyA can issue currencies of denomination USD
     And node PartyA has the finance app installed
     When the network is ready
     Then node PartyA has 1 issuable currency
 
-  Scenario: Node can issue a currency
+    Examples:
-    Given a node PartyA of version master
+      | Node-Version    |
-    And a nonvalidating notary Notary of version master
+      | master          |
+
+  Scenario Outline: Node can issue a currency
+    Given a node PartyA of version <Node-Version>
+    And a nonvalidating notary Notary of version <Node-Version>
     And node PartyA has the finance app installed
     When the network is ready
     Then node PartyA can issue 100 USD
+
+    Examples:
+      | Node-Version    |
+      | master          |

experimental/behave/src/scenario/resources/features/database/connection.feature

@@ -11,4 +11,6 @@ Feature: Database - Connection
     Examples:
       | Node-Version    | Database-Type     |
       | master          | H2                |
+
+# To run this scenario using postgreSQL you must ensure that Docker is running locally
 #      | master          | postgreSQL        |

experimental/behave/src/scenario/resources/features/startup/logging.feature

@@ -3,19 +3,32 @@ Feature: Startup Information - Logging
   A Corda node should inform the user of important parameters during startup so that he/she can confirm the setup and
   configure / connect relevant software to said node.
 
-  Scenario: Node shows logging information on startup
+  Scenario Outline: Node shows logging information on startup
-    Given a node PartyA of version master
+    Given a node PartyA of version <Node-Version>
-    And node PartyA uses database of type H2
+    And node PartyA uses database of type <Database-Type>
     And node PartyA is located in London, GB
     When the network is ready
     Then user can retrieve logging information for node PartyA
 
-  Scenario: Node shows database details on startup
+    Examples:
-    Given a node PartyA of version master
+      | Node-Version   | Database-Type  |
+      | master         | H2             |
+
+  Scenario Outline: Node shows database details on startup
+    Given a node PartyA of version <Node-Version>
+    And node PartyA uses database of type <Database-Type>
     When the network is ready
     Then user can retrieve database details for node PartyA
 
-  Scenario: Node shows version information on startup
+    Examples:
-    Given a node PartyA of version master
+      | Node-Version    | Database-Type  |
-    Then node PartyA is on platform version 4
+      | master          | H2             |
-    And node PartyA is on release version corda-4.0-snapshot
+
+  Scenario Outline: Node shows version information on startup
+    Given a node PartyA of version <Node-Version>
+    Then node PartyA is on platform version <Platform-Version>
+    And node PartyA is on release version <Release-Version>
+
+    Examples:
+      | Node-Version    | Platform-Version | Release-Version    |
+      | master          | 4                | corda-4.0-snapshot |

experimental/behave/src/scenario/resources/scripts/run-behave-features.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#
+# Run this script from the experimental/behave directory
+#
+# $ pwd
+# ./IdeaProjects/corda-reviews/experimental/behave
+# $ src/scenario/resources/scripts/run-behave-features.sh
+#
+# Note: please ensure you have configured your staging environment by running the top-level script: prepare.sh
+
+BUILD_DIR=$PWD
+cd ${BUILD_DIR}
+../../gradlew behaveJar
+
+BEHAVE_JAR=$(ls build/libs/corda-behave-*.jar | tail -n1)
+STAGING_ROOT=~/staging 
+
+# startup
+java -DSTAGING_ROOT=${STAGING_ROOT} -jar ${BEHAVE_JAR} --glue net.corda.behave.scenarios -path ./src/scenario/resources/features/startup/logging.feature
+
+# cash
+java -DSTAGING_ROOT=${STAGING_ROOT} -jar ${BEHAVE_JAR} --glue net.corda.behave.scenarios -path ./src/scenario/resources/features/cash/currencies.feature
+
+# database
+java -DSTAGING_ROOT=${STAGING_ROOT} -jar ${BEHAVE_JAR} --glue net.corda.behave.scenarios -path ./src/scenario/resources/features/cash/currencies.feature

network-management/src/main/kotlin/com/r3/corda/networkmanage/common/utils/Utils.kt

@@ -17,6 +17,7 @@ import net.corda.core.CordaOID
 import net.corda.core.internal.CertRole
 import net.corda.core.serialization.internal.SerializationEnvironmentImpl
 import net.corda.core.serialization.internal.nodeSerializationEnv
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import net.corda.nodeapi.internal.config.parseAs
 import net.corda.nodeapi.internal.crypto.X509CertificateFactory
 import net.corda.nodeapi.internal.crypto.X509KeyStore
@@ -46,7 +47,7 @@ data class CertPathAndKey(val certPath: List<X509Certificate>, val key: PrivateK
 inline fun <reified T : Any> parseConfig(file: Path): T {
     val config = ConfigFactory.parseFile(file.toFile(), ConfigParseOptions.defaults().setAllowMissing(true)).resolve()
     logger.info(config.root().render(ConfigRenderOptions.defaults()))
-    return config.parseAs(strict = false)
+    return config.parseAs(UnknownConfigKeysPolicy.IGNORE::handle)
 }
 
 fun buildCertPath(certPathBytes: ByteArray): CertPath = X509CertificateFactory().delegate.generateCertPath(certPathBytes.inputStream())

network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/generator/GeneratorParameters.kt

@@ -12,6 +12,7 @@ package com.r3.corda.networkmanage.hsm.generator
 
 import com.typesafe.config.ConfigFactory
 import com.typesafe.config.ConfigParseOptions
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import net.corda.nodeapi.internal.config.parseAs
 import net.corda.nodeapi.internal.crypto.CertificateType
 import java.nio.file.Path
@@ -65,5 +66,5 @@ fun parseParameters(configFile: Path): GeneratorParameters {
     return ConfigFactory
             .parseFile(configFile.toFile(), ConfigParseOptions.defaults().setAllowMissing(true))
             .resolve()
-            .parseAs(false)
+            .parseAs(UnknownConfigKeysPolicy.IGNORE::handle)
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/config/ConfigUtilities.kt

@@ -17,6 +17,7 @@ import net.corda.core.identity.CordaX500Name
 import net.corda.core.internal.noneOrSingle
 import net.corda.core.internal.uncheckedCast
 import net.corda.core.utilities.NetworkHostAndPort
+import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.lang.reflect.Field
 import java.lang.reflect.InvocationTargetException
@@ -45,14 +46,13 @@ const val CUSTOM_NODE_PROPERTIES_ROOT = "custom"
 
 // TODO Move other config parsing to use parseAs and remove this
 operator fun <T : Any> Config.getValue(receiver: Any, metadata: KProperty<*>): T {
-    return getValueInternal(metadata.name, metadata.returnType)
+    return getValueInternal(metadata.name, metadata.returnType, UnknownConfigKeysPolicy.IGNORE::handle)
 }
 
-fun <T : Any> Config.parseAs(clazz: KClass<T>, strict: Boolean = true): T {
+fun <T : Any> Config.parseAs(clazz: KClass<T>, onUnknownKeys: ((Set<String>, logger: Logger) -> Unit) = UnknownConfigKeysPolicy.FAIL::handle): T {
     require(clazz.isData) { "Only Kotlin data classes can be parsed. Offending: ${clazz.qualifiedName}" }
     val constructor = clazz.primaryConstructor!!
     val parameters = constructor.parameters
-    if (strict) {
     val parameterNames = parameters.flatMap { param ->
         mutableSetOf<String>().apply {
             param.name?.let(this::add)
@@ -66,15 +66,13 @@ fun <T : Any> Config.parseAs(clazz: KClass<T>, strict: Boolean = true): T {
             .filterNot { it == CUSTOM_NODE_PROPERTIES_ROOT }
             .filterNot(parameterNames::contains)
             .toSortedSet()
-        if (unknownConfigurationKeys.isNotEmpty()) {
+    onUnknownKeys.invoke(unknownConfigurationKeys, logger)
-            throw UnknownConfigurationKeysException.of(unknownConfigurationKeys)
+
-        }
-    }
     val args = parameters.filterNot { it.isOptional && !hasPath(it.name!!) }.associateBy({ it }) { param ->
         // Get the matching property for this parameter
         val property = clazz.memberProperties.first { it.name == param.name }
         val path = defaultToOldPath(property)
-                getValueInternal<Any>(path, param.type, strict)
+        getValueInternal<Any>(path, param.type, onUnknownKeys)
     }
     try {
         return constructor.callBy(args)
@@ -94,7 +92,7 @@ class UnknownConfigurationKeysException private constructor(val unknownKeys: Set
     }
 }
 
-inline fun <reified T : Any> Config.parseAs(strict: Boolean = true): T = parseAs(T::class, strict)
+inline fun <reified T : Any> Config.parseAs(noinline onUnknownKeys: ((Set<String>, logger: Logger) -> Unit) = UnknownConfigKeysPolicy.FAIL::handle): T = parseAs(T::class, onUnknownKeys)
 
 fun Config.toProperties(): Properties {
     return entrySet().associateByTo(
@@ -103,11 +101,11 @@ fun Config.toProperties(): Properties {
             { it.value.unwrapped().toString() })
 }
 
-private fun <T : Any> Config.getValueInternal(path: String, type: KType, strict: Boolean = true): T {
+private fun <T : Any> Config.getValueInternal(path: String, type: KType, onUnknownKeys: ((Set<String>, logger: Logger) -> Unit)): T {
-    return uncheckedCast(if (type.arguments.isEmpty()) getSingleValue(path, type, strict) else getCollectionValue(path, type, strict))
+    return uncheckedCast(if (type.arguments.isEmpty()) getSingleValue(path, type, onUnknownKeys) else getCollectionValue(path, type, onUnknownKeys))
 }
 
-private fun Config.getSingleValue(path: String, type: KType, strict: Boolean = true): Any? {
+private fun Config.getSingleValue(path: String, type: KType, onUnknownKeys: (Set<String>, logger: Logger) -> Unit): Any? {
     if (type.isMarkedNullable && !hasPath(path)) return null
     val typeClass = type.jvmErasure
     return when (typeClass) {
@@ -125,7 +123,7 @@ private fun Config.getSingleValue(path: String, type: KType, strict: Boolean = t
         UUID::class -> UUID.fromString(getString(path))
         CordaX500Name::class -> {
             when (getValue(path).valueType()) {
-                ConfigValueType.OBJECT -> getConfig(path).parseAs(strict)
+                ConfigValueType.OBJECT -> getConfig(path).parseAs(onUnknownKeys)
                 else -> CordaX500Name.parse(getString(path))
             }
         }
@@ -134,12 +132,12 @@ private fun Config.getSingleValue(path: String, type: KType, strict: Boolean = t
         else -> if (typeClass.java.isEnum) {
             parseEnum(typeClass.java, getString(path))
         } else {
-            getConfig(path).parseAs(typeClass, strict)
+            getConfig(path).parseAs(typeClass, onUnknownKeys)
         }
     }
 }
 
-private fun Config.getCollectionValue(path: String, type: KType, strict: Boolean = true): Collection<Any> {
+private fun Config.getCollectionValue(path: String, type: KType, onUnknownKeys: (Set<String>, logger: Logger) -> Unit): Collection<Any> {
     val typeClass = type.jvmErasure
     require(typeClass == List::class || typeClass == Set::class) { "$typeClass is not supported" }
     val elementClass = type.arguments[0].type?.jvmErasure ?: throw IllegalArgumentException("Cannot work with star projection: $type")
@@ -163,7 +161,7 @@ private fun Config.getCollectionValue(path: String, type: KType, strict: Boolean
         else -> if (elementClass.java.isEnum) {
             getStringList(path).map { parseEnum(elementClass.java, it) }
         } else {
-            getConfigList(path).map { it.parseAs(elementClass, strict) }
+            getConfigList(path).map { it.parseAs(elementClass, onUnknownKeys) }
         }
     }
     return if (typeClass == Set::class) values.toSet() else values
@@ -254,3 +252,17 @@ private fun Iterable<*>.toConfigIterable(field: Field): Iterable<Any?> {
 }
 
 private val logger = LoggerFactory.getLogger("net.corda.nodeapi.internal.config")
+
+enum class UnknownConfigKeysPolicy(private val handle: (Set<String>, logger: Logger) -> Unit) {
+
+    FAIL({ unknownKeys, _ -> throw UnknownConfigurationKeysException.of(unknownKeys) }),
+    WARN({ unknownKeys, logger -> logger.warn("Unknown configuration keys found: ${unknownKeys.joinToString(", ", "[", "]")}.") }),
+    IGNORE({ _, _ -> });
+
+    fun handle(unknownKeys: Set<String>, logger: Logger) {
+
+        if (unknownKeys.isNotEmpty()) {
+            handle.invoke(unknownKeys, logger)
+        }
+    }
+}

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/DefaultWhitelist.kt

@@ -30,6 +30,7 @@ object DefaultWhitelist : SerializationWhitelist {
                     Notification.Kind::class.java,
                     ArrayList::class.java,
                     Pair::class.java,
+                    Triple::class.java,
                     ByteArray::class.java,
                     UUID::class.java,
                     LinkedHashSet::class.java,

node/src/main/kotlin/net/corda/node/ArgsParser.kt

@@ -19,6 +19,7 @@ import net.corda.core.internal.exists
 import net.corda.node.services.config.ConfigHelper
 import net.corda.node.services.config.NodeConfiguration
 import net.corda.node.services.config.parseAsNodeConfiguration
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import org.slf4j.event.Level
 import java.io.PrintStream
 import java.nio.file.Path
@@ -52,6 +53,11 @@ class ArgsParser {
             .defaultsTo((Paths.get("certificates") / "network-root-truststore.jks"))
     private val networkRootTrustStorePasswordArg = optionParser.accepts("network-root-truststore-password", "Network root trust store password obtained from network operator.")
             .withRequiredArg()
+    private val unknownConfigKeysPolicy = optionParser.accepts("on-unknown-config-keys", "How to behave on unknown node configuration property keys: [WARN, FAIL, IGNORE].")
+            .withRequiredArg()
+            .withValuesConvertedBy(object : EnumConverter<UnknownConfigKeysPolicy>(UnknownConfigKeysPolicy::class.java) {})
+            .defaultsTo(UnknownConfigKeysPolicy.FAIL)
+
     private val isVersionArg = optionParser.accepts("version", "Print the version and exit")
     private val justGenerateNodeInfoArg = optionParser.accepts("just-generate-node-info",
             "Perform the node start-up task necessary to generate its nodeInfo, save it to disk, then quit")
@@ -76,6 +82,7 @@ class ArgsParser {
         val bootstrapRaftCluster = optionSet.has(bootstrapRaftClusterArg)
         val networkRootTrustStorePath = optionSet.valueOf(networkRootTrustStorePathArg)
         val networkRootTrustStorePassword = optionSet.valueOf(networkRootTrustStorePasswordArg)
+        val unknownConfigKeysPolicy = optionSet.valueOf(unknownConfigKeysPolicy)
 
         val registrationConfig = if (isRegistration) {
             requireNotNull(networkRootTrustStorePassword) { "Network root trust store password must be provided in registration mode using --network-root-truststore-password." }
@@ -95,7 +102,8 @@ class ArgsParser {
                 noLocalShell,
                 sshdServer,
                 justGenerateNodeInfo,
-                bootstrapRaftCluster)
+                bootstrapRaftCluster,
+                unknownConfigKeysPolicy)
     }
 
     fun printHelp(sink: PrintStream) = optionParser.printHelpOn(sink)
@@ -113,13 +121,14 @@ data class CmdLineOptions(val baseDirectory: Path,
                           val noLocalShell: Boolean,
                           val sshdServer: Boolean,
                           val justGenerateNodeInfo: Boolean,
-                          val bootstrapRaftCluster: Boolean) {
+                          val bootstrapRaftCluster: Boolean,
+                          val unknownConfigKeysPolicy: UnknownConfigKeysPolicy) {
     fun loadConfig(): NodeConfiguration {
         val config = ConfigHelper.loadConfig(
                 baseDirectory,
                 configFile,
                 configOverrides = ConfigFactory.parseMap(mapOf("noLocalShell" to this.noLocalShell))
-        ).parseAsNodeConfiguration()
+        ).parseAsNodeConfiguration(unknownConfigKeysPolicy::handle)
         if (nodeRegistrationOption != null) {
             require(!config.devMode) { "registration cannot occur in devMode" }
             requireNotNull(config.compatibilityZoneURL) {

node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

@@ -21,17 +21,18 @@ import net.corda.node.internal.artemis.CertificateChainCheckPolicy
 import net.corda.node.services.config.rpc.NodeRpcOptions
 import net.corda.nodeapi.internal.config.NodeSSLConfiguration
 import net.corda.nodeapi.internal.config.SSLConfiguration
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import net.corda.nodeapi.internal.config.User
 import net.corda.nodeapi.internal.config.parseAs
 import net.corda.nodeapi.internal.persistence.CordaPersistence.DataSourceConfigTag
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.tools.shell.SSHDConfiguration
+import org.slf4j.Logger
 import java.net.URL
 import java.nio.file.Path
 import java.time.Duration
 import java.util.*
 
-
 val Int.MB: Long get() = this * 1024L * 1024L
 
 interface NodeConfiguration : NodeSSLConfiguration {
@@ -152,7 +153,7 @@ data class P2PMessagingRetryConfiguration(
         val backoffBase: Double
 )
 
-fun Config.parseAsNodeConfiguration(): NodeConfiguration = parseAs<NodeConfigurationImpl>()
+fun Config.parseAsNodeConfiguration(onUnknownKeys: ((Set<String>, logger: Logger) -> Unit) = UnknownConfigKeysPolicy.FAIL::handle): NodeConfiguration = parseAs<NodeConfigurationImpl>(onUnknownKeys)
 
 data class NodeConfigurationImpl(
         /** This is not retrieved from the config file but rather from a command line argument. */

node/src/main/kotlin/net/corda/node/utilities/PersistentMap.kt

@@ -70,81 +70,32 @@ class PersistentMap<K : Any, V, E, out EK>(
 
     override val size get() = cache.estimatedSize().toInt()
 
-    private tailrec fun set(key: K, value: V, logWarning: Boolean = true, store: (K, V) -> V?, replace: (K, V) -> Unit): Boolean {
+    private tailrec fun set(key: K, value: V): Boolean {
         var insertionAttempt = false
         var isUnique = true
         val existingInCache = cache.get(key) {
             // Thread safe, if multiple threads may wait until the first one has loaded.
             insertionAttempt = true
-            // Value wasn't in the cache and wasn't in DB (because the cache is unbound).
+            // Value wasn't in the cache and wasn't in DB (because the cache is unbound) so save it.
-            // Store the value, depending on store implementation this may replace existing entry in DB.
+            merge(key, value)
-            store(key, value)
             Optional.of(value)
         }!!
         if (!insertionAttempt) {
             if (existingInCache.isPresent) {
-                // Key already exists in cache, store the new value in the DB (depends on tore implementation) and refresh cache.
+                // Key already exists in cache, store the new value in the DB and refresh cache.
                 isUnique = false
-                replace(key, value)
+                replaceValue(key, value)
             } else {
                 // This happens when the key was queried before with no value associated. We invalidate the cached null
                 // value and recursively call set again. This is to avoid race conditions where another thread queries after
                 // the invalidate but before the set.
                 cache.invalidate(key)
-                return set(key, value, logWarning, store, replace)
+                return set(key, value)
             }
         }
-        if (logWarning && !isUnique) {
-            log.warn("Double insert in ${this.javaClass.name} for entity class $persistentEntityClass key $key, not inserting the second time")
-        }
         return isUnique
     }
 
-    /**
-     * Associates the specified value with the specified key in this map and persists it.
-     * WARNING! If the map previously contained a mapping for the key, the behaviour is unpredictable and may throw an error from the underlying storage.
-     */
-    operator fun set(key: K, value: V) =
-            set(key, value,
-                    logWarning = false,
-                    store = { k: K, v: V ->
-                        currentDBSession().save(toPersistentEntity(k, v))
-                        null
-                    },
-                    replace = { _: K, _: V -> Unit }
-            )
-
-    /**
-     * Associates the specified value with the specified key in this map and persists it.
-     * WARNING! If the map previously contained a mapping for the key, the old value is not replaced.
-     * @return true if added key was unique, otherwise false
-     */
-    fun addWithDuplicatesAllowed(key: K, value: V) =
-            set(key, value,
-                    store = { k, v ->
-                        val session = currentDBSession()
-                        val existingEntry = session.find(persistentEntityClass, toPersistentEntityKey(k))
-                        if (existingEntry == null) {
-                            session.save(toPersistentEntity(k, v))
-                            null
-                        } else {
-                            fromPersistentEntity(existingEntry).second
-                        }
-                    },
-                    replace = { _: K, _: V -> Unit }
-            )
-
-    /**
-     * Associates the specified value with the specified key in this map and persists it.
-     * @return true if added key was unique, otherwise false
-     */
-    private fun addWithDuplicatesReplaced(key: K, value: V) =
-            set(key, value,
-                    logWarning = false,
-                    store = { k: K, v: V -> merge(k, v) },
-                    replace = { k: K, v: V -> replaceValue(k, v) }
-            )
-
     private fun replaceValue(key: K, value: V) {
         synchronized(this) {
             merge(key, value)
@@ -258,9 +209,13 @@ class PersistentMap<K : Any, V, E, out EK>(
             }
         }
 
+    /**
+     * Associates the specified value with the specified key in this map and persists it.
+     * @return true if added key was unique, otherwise false
+     */
     override fun put(key: K, value: V): V? {
         val old = cache.get(key)
-        addWithDuplicatesReplaced(key, value)
+        set(key, value)
         return old!!.orElse(null)
     }
 

node/src/test/kotlin/net/corda/node/ArgsParserTest.kt

@@ -13,6 +13,7 @@ package net.corda.node
 import joptsimple.OptionException
 import net.corda.core.internal.delete
 import net.corda.core.internal.div
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import net.corda.nodeapi.internal.crypto.X509KeyStore
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatExceptionOfType
@@ -52,7 +53,8 @@ class ArgsParserTest {
                 noLocalShell = false,
                 sshdServer = false,
                 justGenerateNodeInfo = false,
-                bootstrapRaftCluster = false))
+                bootstrapRaftCluster = false,
+                unknownConfigKeysPolicy = UnknownConfigKeysPolicy.FAIL))
     }
 
     @Test
@@ -171,4 +173,13 @@ class ArgsParserTest {
         val cmdLineOptions = parser.parse("--bootstrap-raft-cluster")
         assertThat(cmdLineOptions.bootstrapRaftCluster).isTrue()
     }
+
+    @Test
+    fun `on-unknown-config-keys options`() {
+
+        UnknownConfigKeysPolicy.values().forEach { onUnknownConfigKeyPolicy ->
+            val cmdLineOptions = parser.parse("--on-unknown-config-keys", onUnknownConfigKeyPolicy.name)
+            assertThat(cmdLineOptions.unknownConfigKeysPolicy).isEqualTo(onUnknownConfigKeyPolicy)
+        }
+    }
 }

node/src/test/kotlin/net/corda/node/utilities/PersistentMapTests.kt

@@ -0,0 +1,157 @@
+package net.corda.node.utilities
+
+import net.corda.core.crypto.SecureHash
+import net.corda.node.internal.configureDatabase
+import net.corda.node.services.upgrade.ContractUpgradeServiceImpl
+import net.corda.nodeapi.internal.persistence.DatabaseConfig
+import net.corda.testing.internal.rigorousMock
+import net.corda.testing.node.MockServices
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class PersistentMapTests {
+    private val databaseConfig = DatabaseConfig()
+    private val database get() = configureDatabase(dataSourceProps, databaseConfig, rigorousMock())
+    private val dataSourceProps = MockServices.makeTestDataSourceProperties()
+
+    //create a test map using an existing db table
+    private fun createTestMap(): PersistentMap<String, String, ContractUpgradeServiceImpl.DBContractUpgrade, String> {
+        return PersistentMap(
+                toPersistentEntityKey = { it },
+                fromPersistentEntity = { Pair(it.stateRef, it.upgradedContractClassName) },
+                toPersistentEntity = { key: String, value: String ->
+                    ContractUpgradeServiceImpl.DBContractUpgrade().apply {
+                        stateRef = key
+                        upgradedContractClassName = value
+                    }
+                },
+                persistentEntityClass = ContractUpgradeServiceImpl.DBContractUpgrade::class.java
+        )
+    }
+
+    @Test
+    fun `make sure persistence works`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map.put(testHash, "test")
+            assertEquals(map[testHash], "test")
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            assertEquals("test", reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure persistence works using assignment operator`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map[testHash] = "test"
+            assertEquals("test", map[testHash])
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            assertEquals("test", reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure updating works`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map.put(testHash, "test")
+
+            map.put(testHash, "updated")
+            assertEquals("updated", map[testHash])
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            assertEquals("updated", reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure updating works using assignment operator`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map[testHash] = "test"
+
+            map[testHash] = "updated"
+            assertEquals("updated", map[testHash])
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            assertEquals("updated", reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure removal works`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map[testHash] = "test"
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            //check that the item was persisted
+            assertEquals("test", reloadedMap[testHash])
+
+            reloadedMap.remove(testHash)
+            //check that the item was removed in the version of the map
+            assertEquals(null, reloadedMap[testHash])
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            //check that the item was removed from the persistent store
+            assertEquals(null, reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure persistence works against base class`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap()
+            map.put(testHash, "test")
+            assertEquals(map[testHash], "test")
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap()
+            assertEquals("test", reloadedMap[testHash])
+        }
+    }
+
+    @Test
+    fun `make sure persistence works using assignment operator base class`() {
+        val testHash = SecureHash.randomSHA256().toString()
+
+        database.transaction {
+            val map = createTestMap() as MutableMap<String, String>
+            map[testHash] = "test"
+            assertEquals("test", map[testHash])
+        }
+
+        database.transaction {
+            val reloadedMap = createTestMap() as MutableMap<String, String>
+            assertEquals("test", reloadedMap[testHash])
+        }
+    }
+}

tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt

@@ -29,6 +29,7 @@ import net.corda.node.services.config.configOf
 import net.corda.node.services.config.parseAsNodeConfiguration
 import net.corda.node.services.persistence.MigrationExporter
 import net.corda.node.services.schema.NodeSchemaService
+import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
 import net.corda.nodeapi.internal.config.parseAs
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.nodeapi.internal.persistence.SchemaMigration
@@ -149,7 +150,7 @@ private fun handleCommand(options: OptionSet, baseDirectory: Path, configFile: P
             it
         }
     }
-    val config = parsedConfig.parseAs(Configuration::class, false)
+    val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle)
 
     fun runMigrationCommand(withMigration: (SchemaMigration) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource ->
         withMigration(SchemaMigration(schemas, dataSource, true, config.database, classLoader))

tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt

@@ -285,11 +285,18 @@ object InteractiveShell {
             } catch (e: InterruptedException) {
                 // TODO: When the flow framework allows us to kill flows mid-flight, do so here.
             }
+            stateObservable.returnValue.get()?.apply {
+                if (this !is Throwable) {
+                    output.println("Flow completed with result: $this")
+                }
+            }
         } catch (e: NoApplicableConstructor) {
             output.println("No matching constructor found:", Color.red)
             e.errors.forEach { output.println("- $it", Color.red) }
         } catch (e: PermissionException) {
             output.println(e.message ?: "Access denied", Color.red)
+        } catch (e: ExecutionException) {
+          // ignoring it as already logged by the progress handler subscriber
         } finally {
             InputStreamDeserializer.closeAll()
         }