Merge pull request #733 from corda/kat-merge-180412

MERGE - 18 - 04 - 12

.ci/api-current.txt

@@ -1895,7 +1895,7 @@ public @interface net.corda.core.messaging.RPCReturnsObservables
   @org.jetbrains.annotations.NotNull public final List getNotaries()
   @org.jetbrains.annotations.NotNull public final Map getWhitelistedContractImplementations()
   public int hashCode()
-  public String toString()
+  @org.jetbrains.annotations.NotNull public String toString()
 ##
 @net.corda.core.serialization.CordaSerializable public final class net.corda.core.node.NodeInfo extends java.lang.Object
   public <init>(List, List, int, long)

core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt

@@ -47,6 +47,20 @@ data class NetworkParameters(
         require(maxMessageSize > 0) { "maxMessageSize must be at least 1" }
         require(maxTransactionSize > 0) { "maxTransactionSize must be at least 1" }
     }
+
+    override fun toString(): String {
+        return """NetworkParameters {
+  minimumPlatformVersion=$minimumPlatformVersion
+  notaries=$notaries
+  maxMessageSize=$maxMessageSize
+  maxTransactionSize=$maxTransactionSize
+  whitelistedContractImplementations {
+    ${whitelistedContractImplementations.entries.joinToString("\n    ")}
+  }
+  modifiedTime=$modifiedTime
+  epoch=$epoch
+}"""
+    }
 }
 
 /**

docs/source/changelog.rst

@@ -7,6 +7,12 @@ release, see :doc:`upgrade-notes`.
 Unreleased
 ----------
 
+* Fix CORDA-1229. Setter-based serialization was broken with generic types when the property was stored
+  as the raw type, List for example.
+
+* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer
+  needs the whitelist.txt file.
+
 * Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data.
 
 * Serializing an inner class (non-static nested class in Java, inner class in Kotlin) will be rejected explicitly by the serialization
@@ -24,6 +30,12 @@ Unreleased
   .. note:: Whilst this is not the latest version of this library, that being 2.18.1 at time of writing, versions later
   than 2.12.3 (including 2.12.4) exhibit a different issue.
 
+* Fixed security vulnerability when using the ``HashAttachmentConstraint``. Added strict check that the contract JARs
+  referenced in a transaction were deployed on the node.
+
+* Fixed node's behaviour on startup when there is no connectivity to network map. Node continues to work normally if it has
+  all the needed network data, waiting in the background for network map to become available.
+
 * Node can be shut down abruptly by ``shutdown`` function in `CordaRPCOps` or gracefully (draining flows first) through ``gracefulShutdown`` command from shell.
 
 * Carpenter Exceptions will be caught internally by the Serializer and rethrown as a ``NotSerializableException``
@@ -86,6 +98,7 @@ Unreleased
 * Shell (embedded shell available only in dev mode or via SSH) connects to the node via RPC instead of using the ``CordaRPCOps`` object directly.
   To enable RPC connectivity ensure node’s ``rpcSettings.address`` and ``rpcSettings.adminAddress`` settings are present.
 
+
 R3 Corda 3.0 Developer Preview
 ------------------------------
 

docs/source/release-notes.rst

@@ -1,20 +1,6 @@
 Release notes
 =============
 
-Unreleased
-----------
-
-* **Enum Class Evolution**
-  With the addition of AMQP serialization Corda now supports enum constant evolution.
-
-  That is the ability to alter an enum constant and, as long as certain rules are followed and the correct
-  annotations applied, have older and newer instances of that enumeration be understood.
-
-* X.509 certificates now have an extension that specifies the Corda role the certificate is used for, and the role
-  hierarchy is now enforced in the validation code. This only has impact on those developing integrations with external
-  PKI solutions, in most cases it is managed transparently by Corda. A formal specification of the extension can be
-  found at :doc:`permissioning`.
-
 R3 Corda 3.0 Developer Preview
 ------------------------------
 This Developer Preview takes us towards the launch of R3 Corda, R3's commercially supported enterprise blockchain platform.

docs/source/setting-up-a-corda-network.rst

@@ -93,34 +93,16 @@ If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`),
 
 ``java -jar network-bootstrapper.jar <nodes-root-dir> <path-to-first-corDapp> <path-to-second-corDapp> ..``
 
-The CorDapp jars will be hashed and scanned for ``Contract`` classes.
+The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part
-By default the tool would generate a file named ``whitelist.txt`` containing an entry for each contract with the hash of the jar.
+of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`).
+If the network already has a set of network parameters defined (i.e. the node directories all contain the same network-parameters
+file) then the new set of contracts will be appended to the current whitelist.
 
-For example:
+.. note:: The whitelist can only ever be appended to. Once added a contract implementation can never be removed.
 
-.. sourcecode:: none
+By default the bootstrapper tool will whitelist all the contracts found in all the CorDapp jars. To prevent certain
-
+contracts from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``. These will instead
-    net.corda.finance.contracts.asset.Obligation:decd098666b9657314870e192ced0c3519c2c9d395507a238338f8d003929de8
+use the more restrictive ``HashAttachmentConstraint``.
-    net.corda.finance.contracts.asset.Cash:decd098666b9657314870e192ced0c3519c2c9d395507a238338f8d003929de9
-
-These will be added to the ``NetworkParameters.whitelistedContractImplementations``. See :doc:`network-map`.
-
-This means that by default the Network bootstrapper tool will whitelist all contracts found in all passed CorDapps.
-
-In case there is a ``whitelist.txt`` file in the root dir already, the tool will append the new jar hashes or contracts to it.
-
-The zone operator will maintain this whitelist file, and, using the tool, will append new versions of CorDapps to it.
-
-.. warning::
-    - The zone operator must ensure that this file is *append only*.
-    - If the operator removes hashes from the list, all transactions pointing to that version will suddenly fail the constraint verification, and the entire chain is compromised.
-    - If a contract is removed from the whitelist, then all states created from that moment on will be constrained by the HashAttachmentConstraint.
-
-    Note: In future releases, we will provider a tamper-proof way of maintaining the contract whitelist.
-
-For fine-grained control of constraints, in case multiple contracts live in the same jar, the tool reads from another file:
-``exclude_whitelist.txt``, which contains a list of contracts that should not be whitelisted, and thus default to the very restrictive:
-``HashAttachmentConstraint``
 
 For example:
 
@@ -129,7 +111,6 @@ For example:
     net.corda.finance.contracts.asset.Cash
     net.corda.finance.contracts.asset.CommercialPaper
 
-
 Starting the nodes
 ~~~~~~~~~~~~~~~~~~
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt

@@ -12,7 +12,6 @@ package net.corda.nodeapi.internal.network
 
 import com.typesafe.config.ConfigFactory
 import net.corda.cordform.CordformNode
-import net.corda.core.crypto.SecureHash.Companion.parse
 import net.corda.core.identity.Party
 import net.corda.core.internal.*
 import net.corda.core.internal.concurrent.fork
@@ -21,10 +20,13 @@ import net.corda.core.node.NodeInfo
 import net.corda.core.node.NotaryInfo
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.serialization.SerializationContext
+import net.corda.core.serialization.SerializedBytes
+import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.internal.SerializationEnvironmentImpl
 import net.corda.core.serialization.internal._contextSerializationEnv
 import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.seconds
+import net.corda.nodeapi.internal.DEV_ROOT_CA
 import net.corda.nodeapi.internal.SignedNodeInfo
 import net.corda.nodeapi.internal.scanJarForContracts
 import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
@@ -33,11 +35,10 @@ import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
 import net.corda.nodeapi.internal.serialization.amqp.AMQPServerSerializationScheme
 import net.corda.nodeapi.internal.serialization.kryo.AbstractKryoSerializationScheme
 import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
-import java.io.PrintStream
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-import java.nio.file.StandardCopyOption
+import java.nio.file.StandardCopyOption.REPLACE_EXISTING
 import java.time.Instant
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeoutException
@@ -57,12 +58,11 @@ class NetworkBootstrapper {
         )
 
         private const val LOGS_DIR_NAME = "logs"
-        private const val WHITELIST_FILE_NAME = "whitelist.txt"
         private const val EXCLUDE_WHITELIST_FILE_NAME = "exclude_whitelist.txt"
 
         @JvmStatic
         fun main(args: Array<String>) {
-            val baseNodeDirectory = args.firstOrNull() ?: throw IllegalArgumentException("Expecting first argument which is the nodes' parent directory")
+            val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" }
             val cordapps = if (args.size > 1) args.toList().drop(1) else null
             NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps)
         }
@@ -80,15 +80,17 @@ class NetworkBootstrapper {
         try {
             println("Waiting for all nodes to generate their node-info files...")
             val nodeInfoFiles = gatherNodeInfoFiles(processes, nodeDirs)
-            println("Distributing all node info-files to all nodes")
+            println("Distributing all node-info files to all nodes")
             distributeNodeInfos(nodeDirs, nodeInfoFiles)
+            print("Loading existing network parameters... ")
+            val existingNetParams = loadNetworkParameters(nodeDirs)
+            println(existingNetParams ?: "none found")
             println("Gathering notary identities")
             val notaryInfos = gatherNotaryInfos(nodeInfoFiles)
-            println("Notary identities to be used in network parameters: ${notaryInfos.joinToString("; ") { it.prettyPrint() }}")
+            println("Generating contract implementations whitelist")
-            val mergedWhiteList = generateWhitelist(directory / WHITELIST_FILE_NAME, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
+            val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
-            println("Updating whitelist")
+            val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs)
-            overwriteWhitelist(directory / WHITELIST_FILE_NAME, mergedWhiteList)
+            println("${if (existingNetParams == null) "New" else "Updated"} $netParams")
-            installNetworkParameters(notaryInfos, nodeDirs, mergedWhiteList)
             println("Bootstrapping complete!")
         } finally {
             _contextSerializationEnv.set(null)
@@ -106,15 +108,15 @@ class NetworkBootstrapper {
             val nodeName = confFile.fileName.toString().removeSuffix("_node.conf")
             println("Generating directory for $nodeName")
             val nodeDir = (directory / nodeName).createDirectories()
-            confFile.moveTo(nodeDir / "node.conf", StandardCopyOption.REPLACE_EXISTING)
+            confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING)
-            webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", StandardCopyOption.REPLACE_EXISTING)
+            webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING)
-            Files.copy(cordaJar, (nodeDir / "corda.jar"), StandardCopyOption.REPLACE_EXISTING)
+            cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING)
         }
         Files.delete(cordaJar)
     }
 
     private fun extractCordaJarTo(directory: Path): Path {
-        val cordaJarPath = (directory / "corda.jar")
+        val cordaJarPath = directory / "corda.jar"
         if (!cordaJarPath.exists()) {
             println("No corda jar found in root directory. Extracting from jar")
             Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath)
@@ -147,7 +149,7 @@ class NetworkBootstrapper {
         }
 
         return try {
-            future.getOrThrow(60.seconds)
+            future.getOrThrow(timeout = 60.seconds)
         } catch (e: TimeoutException) {
             println("...still waiting. If this is taking longer than usual, check the node logs.")
             future.getOrThrow()
@@ -158,7 +160,7 @@ class NetworkBootstrapper {
         for (nodeDir in nodeDirs) {
             val additionalNodeInfosDir = (nodeDir / CordformNode.NODE_INFO_DIRECTORY).createDirectories()
             for (nodeInfoFile in nodeInfoFiles) {
-                nodeInfoFile.copyToDirectory(additionalNodeInfosDir, StandardCopyOption.REPLACE_EXISTING)
+                nodeInfoFile.copyToDirectory(additionalNodeInfosDir, REPLACE_EXISTING)
             }
         }
     }
@@ -178,25 +180,67 @@ class NetworkBootstrapper {
         }.distinct() // We need distinct as nodes part of a distributed notary share the same notary identity
     }
 
-    private fun installNetworkParameters(notaryInfos: List<NotaryInfo>, nodeDirs: List<Path>, whitelist: Map<String, List<AttachmentId>>) {
+    private fun loadNetworkParameters(nodeDirs: List<Path>): NetworkParameters? {
-        // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
+        val netParamsFilesGrouped = nodeDirs.mapNotNull {
-        val copier = NetworkParametersCopier(NetworkParameters(
+            val netParamsFile = it / NETWORK_PARAMS_FILE_NAME
+            if (netParamsFile.exists()) netParamsFile else null
+        }.groupBy { SerializedBytes<SignedNetworkParameters>(it.readAll()) }
+
+        when (netParamsFilesGrouped.size) {
+            0 -> return null
+            1 -> return netParamsFilesGrouped.keys.first().deserialize().verifiedNetworkMapCert(DEV_ROOT_CA.certificate)
+        }
+
+        val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " +
+                "network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n")
+
+        netParamsFilesGrouped.forEach { bytes, netParamsFiles ->
+            netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ")
+            msg.append(":\n")
+            val netParamsString = try {
+                bytes.deserialize().verifiedNetworkMapCert(DEV_ROOT_CA.certificate).toString()
+            } catch (e: Exception) {
+                "Invalid network parameters file: $e"
+            }
+            msg.append(netParamsString)
+            msg.append("\n\n")
+        }
+
+        throw IllegalStateException(msg.toString())
+    }
+
+    private fun installNetworkParameters(notaryInfos: List<NotaryInfo>,
+                                         whitelist: Map<String, List<AttachmentId>>,
+                                         existingNetParams: NetworkParameters?,
+                                         nodeDirs: List<Path>): NetworkParameters {
+        val networkParameters = if (existingNetParams != null) {
+            existingNetParams.copy(
+                    notaries = notaryInfos,
+                    modifiedTime = Instant.now(),
+                    whitelistedContractImplementations = whitelist,
+                    epoch = existingNetParams.epoch + 1
+            )
+        } else {
+            NetworkParameters(
                     minimumPlatformVersion = 1,
                     notaries = notaryInfos,
                     modifiedTime = Instant.now(),
                     maxMessageSize = 10485760,
                     maxTransactionSize = Int.MAX_VALUE,
-                epoch = 1,
+                    whitelistedContractImplementations = whitelist,
-                whitelistedContractImplementations = whitelist
+                    epoch = 1
-        ), overwriteFile = true)
+            )
-
+        }
+        // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
+        val copier = NetworkParametersCopier(networkParameters, overwriteFile = true)
         nodeDirs.forEach { copier.install(it) }
+        return networkParameters
     }
 
-    private fun generateWhitelist(whitelistFile: Path, excludeWhitelistFile: Path, cordapps: List<String>?): Map<String, List<AttachmentId>> {
+    private fun generateWhitelist(networkParameters: NetworkParameters?,
-        val existingWhitelist = if (whitelistFile.exists()) readContractWhitelist(whitelistFile) else emptyMap()
+                                  excludeWhitelistFile: Path,
-
+                                  cordapps: List<String>?): Map<String, List<AttachmentId>> {
-        println(if (existingWhitelist.isEmpty()) "No existing whitelist file found." else "Found existing whitelist: $whitelistFile")
+        val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap()
 
         val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList()
         if (excludeContracts.isNotEmpty()) {
@@ -210,38 +254,15 @@ class NetworkBootstrapper {
             }
         }?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap()
 
-        println("Calculating whitelist for current installed CorDapps..")
+        return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
-
-        val merged = (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
             val existing = existingWhitelist[contractClassName] ?: emptyList()
             val newHash = newWhiteList[contractClassName]
             contractClassName to (if (newHash == null || newHash in existing) existing else existing + newHash)
         }.toMap()
-
-        println("CorDapp whitelist " + (if (existingWhitelist.isEmpty()) "generated" else "updated") + " in $whitelistFile")
-        return merged
-    }
-
-    private fun overwriteWhitelist(whitelistFile: Path, mergedWhiteList: Map<String, List<AttachmentId>>) {
-        PrintStream(whitelistFile.toFile().outputStream()).use { out ->
-            mergedWhiteList.forEach { (contract, attachments) ->
-                out.println("$contract:${attachments.joinToString(",")}")
-            }
-        }
-    }
-
-    private fun readContractWhitelist(file: Path): Map<String, List<AttachmentId>> {
-        return file.readAllLines()
-                .map { line -> line.split(":") }
-                .map { (contract, attachmentIds) ->
-                    contract to (attachmentIds.split(",").map(::parse))
-                }.toMap()
     }
 
     private fun readExcludeWhitelist(file: Path): List<String> = file.readAllLines().map(String::trim)
 
-    private fun NotaryInfo.prettyPrint(): String = "${identity.name} (${if (validating) "" else "non-"}validating)"
-
     private fun NodeInfo.notaryIdentity(): Party {
         return when (legalIdentities.size) {
             // Single node notaries have just one identity like all other nodes. This identity is the notary identity

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt

@@ -315,7 +315,7 @@ fun propertiesForSerializationFromSetters(
                         "takes too many arguments")
             }
 
-            val setterType = setter.parameterTypes[0]!!
+            val setterType = setter.genericParameterTypes[0]!!
 
             if ((property.value.field != null) &&
                     (!(TypeToken.of(property.value.field?.genericType!!).isSupertypeOf(setterType)))) {
@@ -324,11 +324,11 @@ fun propertiesForSerializationFromSetters(
                         "${property.value.field?.genericType!!}")
             }
 
-            // make sure the setter returns the same type (within inheritance bounds) the getter accepts
+            // Make sure the getter returns the same type (within inheritance bounds) the setter accepts.
-            if (!(TypeToken.of (setterType).isSupertypeOf(getter.returnType))) {
+            if (!(TypeToken.of (getter.genericReturnType).isSupertypeOf(setterType))) {
                 throw NotSerializableException("Defined setter for parameter ${property.value.field?.name} " +
                         "takes parameter of type $setterType yet the defined getter returns a value of type " +
-                        "${getter.returnType}")
+                        "${getter.returnType} [${getter.genericReturnType}]")
             }
             this += PropertyAccessorGetterSetter(
                     idx++,

node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/SetterConstructorTests.java

@@ -17,6 +17,8 @@ import org.junit.Test;
 import static org.junit.Assert.*;
 
 import java.io.NotSerializableException;
+import java.util.ArrayList;
+import java.util.List;
 
 public class SetterConstructorTests {
 
@@ -74,6 +76,13 @@ public class SetterConstructorTests {
         public void setC(int c) { this.c = c; }
     }
 
+    static class CIntList {
+        private List<Integer> l;
+
+        public List getL() { return l; }
+        public void setL(List<Integer> l) { this.l = l; }
+    }
+
     static class Inner1 {
         private String a;
 
@@ -325,4 +334,29 @@ public class SetterConstructorTests {
         Assertions.assertThatThrownBy(() -> new SerializationOutput(factory1).serialize(tm)).isInstanceOf(
                 NotSerializableException.class);
     }
+
+    // This not blowing up means it's working
+    @Test
+    public void intList() throws NotSerializableException {
+        CIntList cil = new CIntList();
+
+        List<Integer> l = new ArrayList<>();
+        l.add(1);
+        l.add(2);
+        l.add(3);
+
+        cil.setL(l);
+
+        EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter();
+        FingerPrinter fingerPrinter = new SerializerFingerPrinter();
+        SerializerFactory factory1 = new SerializerFactory(
+                AllWhitelist.INSTANCE,
+                ClassLoader.getSystemClassLoader(),
+                evolutionSerialiserGetter,
+                fingerPrinter);
+
+        // if we've got super / sub types on the setter vs the underlying type the wrong way around this will
+        // explode. See CORDA-1229 (https://r3-cev.atlassian.net/browse/CORDA-1229)
+        new DeserializationInput(factory1).deserialize(new SerializationOutput(factory1).serialize(cil), CIntList.class);
+    }
 }

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt

@@ -1320,6 +1320,5 @@ class SerializationOutputTests(private val compression: CordaSerializationEncodi
             C(12).serializeE()
         }.withMessageContaining("has synthetic fields and is likely a nested inner class")
     }
-
 }