CORDA-1312: Removed the need to have whitelist.txt for updating the contracts whitelist using the bootstrapper. (#2954)

Instead the current whitelist is read in from the existing network parameters file.

.ci/api-current.txt

@@ -1884,7 +1884,7 @@ public @interface net.corda.core.messaging.RPCReturnsObservables
   @org.jetbrains.annotations.NotNull public final List getNotaries()
   @org.jetbrains.annotations.NotNull public final Map getWhitelistedContractImplementations()
   public int hashCode()
-  public String toString()
+  @org.jetbrains.annotations.NotNull public String toString()
 ##
 @net.corda.core.serialization.CordaSerializable public final class net.corda.core.node.NodeInfo extends java.lang.Object
   public <init>(List, List, int, long)

core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt

@@ -37,6 +37,20 @@ data class NetworkParameters(
         require(maxMessageSize > 0) { "maxMessageSize must be at least 1" }
         require(maxTransactionSize > 0) { "maxTransactionSize must be at least 1" }
     }
+
+    override fun toString(): String {
+        return """NetworkParameters {
+  minimumPlatformVersion=$minimumPlatformVersion
+  notaries=$notaries
+  maxMessageSize=$maxMessageSize
+  maxTransactionSize=$maxTransactionSize
+  whitelistedContractImplementations {
+    ${whitelistedContractImplementations.entries.joinToString("\n    ")}
+  }
+  modifiedTime=$modifiedTime
+  epoch=$epoch
+}"""
+    }
 }
 
 /**

docs/source/changelog.rst

@@ -19,6 +19,9 @@ Unreleased
 * Shell (embedded available only in dev mode or via SSH) connects to the node via RPC instead of using the ``CordaRPCOps`` object directly.
   To enable RPC connectivity ensure node’s ``rpcSettings.address`` and ``rpcSettings.adminAddress`` settings are present.
 
+* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer
+  needs the whitelist.txt file.
+
 * Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data.
 
 * Serializing an inner class (non-static nested class in Java, inner class in Kotlin) will be rejected explicitly by the serialization

docs/source/setting-up-a-corda-network.rst

@@ -93,34 +93,16 @@ If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`),
 
 ``java -jar network-bootstrapper.jar <nodes-root-dir> <path-to-first-corDapp> <path-to-second-corDapp> ..``
 
-The CorDapp jars will be hashed and scanned for ``Contract`` classes.
+The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part
-By default the tool would generate a file named ``whitelist.txt`` containing an entry for each contract with the hash of the jar.
+of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`).
+If the network already has a set of network parameters defined (i.e. the node directories all contain the same network-parameters
+file) then the new set of contracts will be appended to the current whitelist.
 
-For example:
+.. note:: The whitelist can only ever be appended to. Once added a contract implementation can never be removed.
 
-.. sourcecode:: none
+By default the bootstrapper tool will whitelist all the contracts found in all the CorDapp jars. To prevent certain
-
+contracts from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``. These will instead
-    net.corda.finance.contracts.asset.Obligation:decd098666b9657314870e192ced0c3519c2c9d395507a238338f8d003929de8
+use the more restrictive ``HashAttachmentConstraint``.
-    net.corda.finance.contracts.asset.Cash:decd098666b9657314870e192ced0c3519c2c9d395507a238338f8d003929de9
-
-These will be added to the ``NetworkParameters.whitelistedContractImplementations``. See :doc:`network-map`.
-
-This means that by default the Network bootstrapper tool will whitelist all contracts found in all passed CorDapps.
-
-In case there is a ``whitelist.txt`` file in the root dir already, the tool will append the new jar hashes or contracts to it.
-
-The zone operator will maintain this whitelist file, and, using the tool, will append new versions of CorDapps to it.
-
-.. warning::
-    - The zone operator must ensure that this file is *append only*.
-    - If the operator removes hashes from the list, all transactions pointing to that version will suddenly fail the constraint verification, and the entire chain is compromised.
-    - If a contract is removed from the whitelist, then all states created from that moment on will be constrained by the HashAttachmentConstraint.
-
-    Note: In future releases, we will provider a tamper-proof way of maintaining the contract whitelist.
-
-For fine-grained control of constraints, in case multiple contracts live in the same jar, the tool reads from another file:
-``exclude_whitelist.txt``, which contains a list of contracts that should not be whitelisted, and thus default to the very restrictive:
-``HashAttachmentConstraint``
 
 For example:
 
@@ -129,7 +111,6 @@ For example:
     net.corda.finance.contracts.asset.Cash
     net.corda.finance.contracts.asset.CommercialPaper
 
-
 Starting the nodes
 ~~~~~~~~~~~~~~~~~~
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt

@@ -2,7 +2,6 @@ package net.corda.nodeapi.internal.network
 
 import com.typesafe.config.ConfigFactory
 import net.corda.cordform.CordformNode
-import net.corda.core.crypto.SecureHash.Companion.parse
 import net.corda.core.identity.Party
 import net.corda.core.internal.*
 import net.corda.core.internal.concurrent.fork
@@ -11,10 +10,13 @@ import net.corda.core.node.NodeInfo
 import net.corda.core.node.NotaryInfo
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.serialization.SerializationContext
+import net.corda.core.serialization.SerializedBytes
+import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.internal.SerializationEnvironmentImpl
 import net.corda.core.serialization.internal._contextSerializationEnv
 import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.seconds
+import net.corda.nodeapi.internal.DEV_ROOT_CA
 import net.corda.nodeapi.internal.SignedNodeInfo
 import net.corda.nodeapi.internal.scanJarForContracts
 import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
@@ -23,11 +25,10 @@ import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
 import net.corda.nodeapi.internal.serialization.amqp.AMQPServerSerializationScheme
 import net.corda.nodeapi.internal.serialization.kryo.AbstractKryoSerializationScheme
 import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
-import java.io.PrintStream
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
-import java.nio.file.StandardCopyOption
+import java.nio.file.StandardCopyOption.REPLACE_EXISTING
 import java.time.Instant
 import java.util.concurrent.Executors
 import java.util.concurrent.TimeoutException
@@ -47,12 +48,11 @@ class NetworkBootstrapper {
         )
 
         private const val LOGS_DIR_NAME = "logs"
-        private const val WHITELIST_FILE_NAME = "whitelist.txt"
         private const val EXCLUDE_WHITELIST_FILE_NAME = "exclude_whitelist.txt"
 
         @JvmStatic
         fun main(args: Array<String>) {
-            val baseNodeDirectory = args.firstOrNull() ?: throw IllegalArgumentException("Expecting first argument which is the nodes' parent directory")
+            val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" }
             val cordapps = if (args.size > 1) args.toList().drop(1) else null
             NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps)
         }
@@ -70,15 +70,17 @@ class NetworkBootstrapper {
         try {
             println("Waiting for all nodes to generate their node-info files...")
             val nodeInfoFiles = gatherNodeInfoFiles(processes, nodeDirs)
-            println("Distributing all node info-files to all nodes")
+            println("Distributing all node-info files to all nodes")
             distributeNodeInfos(nodeDirs, nodeInfoFiles)
+            print("Loading existing network parameters... ")
+            val existingNetParams = loadNetworkParameters(nodeDirs)
+            println(existingNetParams ?: "none found")
             println("Gathering notary identities")
             val notaryInfos = gatherNotaryInfos(nodeInfoFiles)
-            println("Notary identities to be used in network parameters: ${notaryInfos.joinToString("; ") { it.prettyPrint() }}")
+            println("Generating contract implementations whitelist")
-            val mergedWhiteList = generateWhitelist(directory / WHITELIST_FILE_NAME, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
+            val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
-            println("Updating whitelist")
+            val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs)
-            overwriteWhitelist(directory / WHITELIST_FILE_NAME, mergedWhiteList)
+            println("${if (existingNetParams == null) "New" else "Updated"} $netParams")
-            installNetworkParameters(notaryInfos, nodeDirs, mergedWhiteList)
             println("Bootstrapping complete!")
         } finally {
             _contextSerializationEnv.set(null)
@@ -96,15 +98,15 @@ class NetworkBootstrapper {
             val nodeName = confFile.fileName.toString().removeSuffix("_node.conf")
             println("Generating directory for $nodeName")
             val nodeDir = (directory / nodeName).createDirectories()
-            confFile.moveTo(nodeDir / "node.conf", StandardCopyOption.REPLACE_EXISTING)
+            confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING)
-            webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", StandardCopyOption.REPLACE_EXISTING)
+            webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING)
-            Files.copy(cordaJar, (nodeDir / "corda.jar"), StandardCopyOption.REPLACE_EXISTING)
+            cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING)
         }
         Files.delete(cordaJar)
     }
 
     private fun extractCordaJarTo(directory: Path): Path {
-        val cordaJarPath = (directory / "corda.jar")
+        val cordaJarPath = directory / "corda.jar"
         if (!cordaJarPath.exists()) {
             println("No corda jar found in root directory. Extracting from jar")
             Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath)
@@ -137,7 +139,7 @@ class NetworkBootstrapper {
         }
 
         return try {
-            future.getOrThrow(60.seconds)
+            future.getOrThrow(timeout = 60.seconds)
         } catch (e: TimeoutException) {
             println("...still waiting. If this is taking longer than usual, check the node logs.")
             future.getOrThrow()
@@ -148,7 +150,7 @@ class NetworkBootstrapper {
         for (nodeDir in nodeDirs) {
             val additionalNodeInfosDir = (nodeDir / CordformNode.NODE_INFO_DIRECTORY).createDirectories()
             for (nodeInfoFile in nodeInfoFiles) {
-                nodeInfoFile.copyToDirectory(additionalNodeInfosDir, StandardCopyOption.REPLACE_EXISTING)
+                nodeInfoFile.copyToDirectory(additionalNodeInfosDir, REPLACE_EXISTING)
             }
         }
     }
@@ -168,25 +170,67 @@ class NetworkBootstrapper {
         }.distinct() // We need distinct as nodes part of a distributed notary share the same notary identity
     }
 
-    private fun installNetworkParameters(notaryInfos: List<NotaryInfo>, nodeDirs: List<Path>, whitelist: Map<String, List<AttachmentId>>) {
+    private fun loadNetworkParameters(nodeDirs: List<Path>): NetworkParameters? {
-        // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
+        val netParamsFilesGrouped = nodeDirs.mapNotNull {
-        val copier = NetworkParametersCopier(NetworkParameters(
+            val netParamsFile = it / NETWORK_PARAMS_FILE_NAME
+            if (netParamsFile.exists()) netParamsFile else null
+        }.groupBy { SerializedBytes<SignedNetworkParameters>(it.readAll()) }
+
+        when (netParamsFilesGrouped.size) {
+            0 -> return null
+            1 -> return netParamsFilesGrouped.keys.first().deserialize().verifiedNetworkMapCert(DEV_ROOT_CA.certificate)
+        }
+
+        val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " +
+                "network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n")
+
+        netParamsFilesGrouped.forEach { bytes, netParamsFiles ->
+            netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ")
+            msg.append(":\n")
+            val netParamsString = try {
+                bytes.deserialize().verifiedNetworkMapCert(DEV_ROOT_CA.certificate).toString()
+            } catch (e: Exception) {
+                "Invalid network parameters file: $e"
+            }
+            msg.append(netParamsString)
+            msg.append("\n\n")
+        }
+
+        throw IllegalStateException(msg.toString())
+    }
+
+    private fun installNetworkParameters(notaryInfos: List<NotaryInfo>,
+                                         whitelist: Map<String, List<AttachmentId>>,
+                                         existingNetParams: NetworkParameters?,
+                                         nodeDirs: List<Path>): NetworkParameters {
+        val networkParameters = if (existingNetParams != null) {
+            existingNetParams.copy(
+                    notaries = notaryInfos,
+                    modifiedTime = Instant.now(),
+                    whitelistedContractImplementations = whitelist,
+                    epoch = existingNetParams.epoch + 1
+            )
+        } else {
+            NetworkParameters(
                     minimumPlatformVersion = 1,
                     notaries = notaryInfos,
                     modifiedTime = Instant.now(),
                     maxMessageSize = 10485760,
                     maxTransactionSize = Int.MAX_VALUE,
-                epoch = 1,
+                    whitelistedContractImplementations = whitelist,
-                whitelistedContractImplementations = whitelist
+                    epoch = 1
-        ), overwriteFile = true)
+            )
-
+        }
+        // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
+        val copier = NetworkParametersCopier(networkParameters, overwriteFile = true)
         nodeDirs.forEach { copier.install(it) }
+        return networkParameters
     }
 
-    private fun generateWhitelist(whitelistFile: Path, excludeWhitelistFile: Path, cordapps: List<String>?): Map<String, List<AttachmentId>> {
+    private fun generateWhitelist(networkParameters: NetworkParameters?,
-        val existingWhitelist = if (whitelistFile.exists()) readContractWhitelist(whitelistFile) else emptyMap()
+                                  excludeWhitelistFile: Path,
-
+                                  cordapps: List<String>?): Map<String, List<AttachmentId>> {
-        println(if (existingWhitelist.isEmpty()) "No existing whitelist file found." else "Found existing whitelist: $whitelistFile")
+        val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap()
 
         val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList()
         if (excludeContracts.isNotEmpty()) {
@@ -200,38 +244,15 @@ class NetworkBootstrapper {
             }
         }?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap()
 
-        println("Calculating whitelist for current installed CorDapps..")
+        return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
-
-        val merged = (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
             val existing = existingWhitelist[contractClassName] ?: emptyList()
             val newHash = newWhiteList[contractClassName]
             contractClassName to (if (newHash == null || newHash in existing) existing else existing + newHash)
         }.toMap()
-
-        println("CorDapp whitelist " + (if (existingWhitelist.isEmpty()) "generated" else "updated") + " in $whitelistFile")
-        return merged
-    }
-
-    private fun overwriteWhitelist(whitelistFile: Path, mergedWhiteList: Map<String, List<AttachmentId>>) {
-        PrintStream(whitelistFile.toFile().outputStream()).use { out ->
-            mergedWhiteList.forEach { (contract, attachments) ->
-                out.println("$contract:${attachments.joinToString(",")}")
-            }
-        }
-    }
-
-    private fun readContractWhitelist(file: Path): Map<String, List<AttachmentId>> {
-        return file.readAllLines()
-                .map { line -> line.split(":") }
-                .map { (contract, attachmentIds) ->
-                    contract to (attachmentIds.split(",").map(::parse))
-                }.toMap()
     }
 
     private fun readExcludeWhitelist(file: Path): List<String> = file.readAllLines().map(String::trim)
 
-    private fun NotaryInfo.prettyPrint(): String = "${identity.name} (${if (validating) "" else "non-"}validating)"
-
     private fun NodeInfo.notaryIdentity(): Party {
         return when (legalIdentities.size) {
             // Single node notaries have just one identity like all other nodes. This identity is the notary identity