[CORDA-442] Make cordformation serialize NodeInfos to disk during deployment. (#1546)

Initial PR for https://r3-cev.atlassian.net/projects/CORDA/issues/CORDA-442

Allow for cordformation not to specify which node is the network map.
When that happens Cordformation will start each node and make it serialize its NodeInfo to disk.
This make 'depolyNodes' slower. On my machine for the traderDemo it's ~25s

PersistentNetworkMapCache will load files from disk at startup.
Additionally nodeinfos are loaded in the networkMapCache only if they're newer than the currently known version.

constants.properties

@@ -1,4 +1,4 @@
-gradlePluginsVersion=1.0.1
+gradlePluginsVersion=1.1.0
 kotlinVersion=1.1.50
 guavaVersion=21.0
 bouncycastleVersion=1.57

docs/source/changelog.rst

@@ -7,6 +7,12 @@ from the previous milestone release.
 UNRELEASED
 ----------
 
+* ``Cordform`` and node identity generation
+  * Cordform may not specify a value for ``NetworkMap``, when that happens, during the task execution the following happens:
+    1. Each node is started and its signed serialized NodeInfo is written to disk in the node base directory.
+    2. Every serialized ``NodeInfo`` above is copied in every other node "additional-node-info" folder under the NodeInfo folder.
+  * Nodes read all the nodes stored in ``additional-node-info`` when the ``NetworkMapService`` starts up.
+
 * ``Cordapp`` now has a name field for identifying CorDapps and all CorDapp names are printed to console at startup.
 
 * Enums now respsect the whitelist applied to the Serializer factory serializing / deserializing them. If the enum isn't

gradle-plugins/cordform-common/src/main/java/net/corda/cordform/CordformNode.java

@@ -9,6 +9,11 @@ import java.util.List;
 import java.util.Map;
 
 public class CordformNode implements NodeDefinition {
+    /**
+     * Path relative to the running node where the serialized NodeInfos are stored.
+     */
+    public static final String NODE_INFO_DIRECTORY = "additional-node-infos";
+
     protected static final String DEFAULT_HOST = "localhost";
 
     /**

gradle-plugins/cordformation/src/main/groovy/net/corda/plugins/Cordform.groovy

@@ -3,6 +3,7 @@ package net.corda.plugins
 import static org.gradle.api.tasks.SourceSet.MAIN_SOURCE_SET_NAME
 import net.corda.cordform.CordformContext
 import net.corda.cordform.CordformDefinition
+import net.corda.cordform.CordformNode
 import org.apache.tools.ant.filters.FixCrLfFilter
 import org.bouncycastle.asn1.x500.X500Name
 import org.gradle.api.DefaultTask
@@ -61,8 +62,8 @@ class Cordform extends DefaultTask {
      * @return A node instance.
      */
     private Node getNodeByName(String name) {
-        for(Node node : nodes) {
+        for (Node node : nodes) {
-            if(node.name == name) {
+            if (node.name == name) {
                 return node
             }
         }
@@ -109,10 +110,14 @@ class Cordform extends DefaultTask {
      */
     @TaskAction
     void build() {
-        String networkMapNodeName
+        String networkMapNodeName = initializeConfigurationAndGetNetworkMapNodeName()
+        installRunScript()
+        finalizeConfiguration(networkMapNodeName)
+    }
+
+    private initializeConfigurationAndGetNetworkMapNodeName() {
         if (null != definitionClass) {
             def cd = loadCordformDefinition()
-            networkMapNodeName = cd.networkMapNodeName.toString()
             cd.nodeConfigurers.each { nc ->
                 node { Node it ->
                     nc.accept it
@@ -124,21 +129,55 @@ class Cordform extends DefaultTask {
                     project.projectDir.toPath().resolve(getNodeByName(nodeName).nodeDir.toPath())
                 }
             }
+            return cd.networkMapNodeName.toString()
         } else {
-            networkMapNodeName = this.networkMapNodeName
             nodes.each {
                 it.rootDir directory
             }
+            return this.networkMapNodeName
         }
-        installRunScript()
+    }
-        def networkMapNode = getNodeByName(networkMapNodeName)
+
-        if (networkMapNode == null)
+    private finalizeConfiguration(String networkMapNodeName) {
-            throw new IllegalStateException("The networkMap property refers to a node that isn't configured ($networkMapNodeName)")
+        Node networkMapNode = getNodeByName(networkMapNodeName)
-        nodes.each {
+        if (networkMapNode == null) {
-            if(it != networkMapNode) {
+            nodes.each {
-                it.networkMapAddress(networkMapNode.getP2PAddress(), networkMapNodeName)
+                it.build()
+            }
+            generateNodeInfos()
+            logger.info("Starting without networkMapNode, this an experimental feature")
+        } else {
+            nodes.each {
+                if (it != networkMapNode) {
+                    it.networkMapAddress(networkMapNode.getP2PAddress(), networkMapNodeName)
+                }
+                it.build()
+            }
+        }
+    }
+
+    Path fullNodePath(Node node) {
+        return project.projectDir.toPath().resolve(node.nodeDir.toPath())
+    }
+
+    private generateNodeInfos() {
+        nodes.each { Node node ->
+            def process = new ProcessBuilder("java", "-jar", Node.NODEJAR_NAME, "--just-generate-node-info")
+                    .directory(fullNodePath(node).toFile())
+                    .redirectErrorStream(true)
+                    .start()
+                    .waitFor()
+        }
+        for (source in nodes) {
+            for (destination in nodes) {
+                if (source.nodeDir != destination.nodeDir) {
+                    project.copy {
+                        from fullNodePath(source).toString()
+                        include 'nodeInfo-*'
+                        into fullNodePath(destination).resolve(Node.NODE_INFO_DIRECTORY).toString()
+                    }
+                }
             }
-            it.build()
         }
     }
 }

node/src/main/kotlin/net/corda/node/ArgsParser.kt

@@ -34,6 +34,8 @@ class ArgsParser {
     private val noLocalShellArg = optionParser.accepts("no-local-shell", "Do not start the embedded shell locally.")
     private val isRegistrationArg = optionParser.accepts("initial-registration", "Start initial node registration with Corda network to obtain certificate from the permissioning server.")
     private val isVersionArg = optionParser.accepts("version", "Print the version and exit")
+    private val justGenerateNodeInfoArg = optionParser.accepts("just-generate-node-info",
+            "Perform the node start-up task necessary to generate its nodeInfo, save it to disk, then quit")
     private val helpArg = optionParser.accepts("help").forHelp()
 
     fun parse(vararg args: String): CmdLineOptions {
@@ -50,7 +52,9 @@ class ArgsParser {
         val isVersion = optionSet.has(isVersionArg)
         val noLocalShell = optionSet.has(noLocalShellArg)
         val sshdServer = optionSet.has(sshdServerArg)
-        return CmdLineOptions(baseDirectory, configFile, help, loggingLevel, logToConsole, isRegistration, isVersion, noLocalShell, sshdServer)
+        val justGenerateNodeInfo = optionSet.has(justGenerateNodeInfoArg)
+        return CmdLineOptions(baseDirectory, configFile, help, loggingLevel, logToConsole, isRegistration, isVersion,
+                noLocalShell, sshdServer, justGenerateNodeInfo)
     }
 
     fun printHelp(sink: PrintStream) = optionParser.printHelpOn(sink)
@@ -64,7 +68,8 @@ data class CmdLineOptions(val baseDirectory: Path,
                           val isRegistration: Boolean,
                           val isVersion: Boolean,
                           val noLocalShell: Boolean,
-                          val sshdServer: Boolean) {
+                          val sshdServer: Boolean,
+                          val justGenerateNodeInfo : Boolean) {
     fun loadConfig() = ConfigHelper
                 .loadConfig(baseDirectory, configFile)
                 .parseAs<FullNodeConfiguration>()

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -169,20 +169,36 @@ abstract class AbstractNode(open val configuration: NodeConfiguration,
         return CordaRPCOpsImpl(services, smm, database)
     }
 
-    open fun start(): StartedNode<AbstractNode> {
+    private fun saveOwnNodeInfo() {
-        require(started == null) { "Node has already been started" }
+        NodeInfoSerializer().saveToFile(configuration.baseDirectory, info, services.keyManagementService)
+    }
+
+    private fun initCertificate() {
         if (configuration.devMode) {
             log.warn("Corda node is running in dev mode.")
             configuration.configureWithDevSSLCertificate()
         }
         validateKeystore()
+    }
 
+    open fun generateNodeInfo() {
+        check(started == null) { "Node has already been started" }
+        initCertificate()
+        log.info("Generating nodeInfo ...")
+        initialiseDatabasePersistence {
+            makeServices()
+            saveOwnNodeInfo()
+        }
+    }
+
+    open fun start(): StartedNode<AbstractNode> {
+        check(started == null) { "Node has already been started" }
+        initCertificate()
         log.info("Node starting up ...")
-
         // Do all of this in a database transaction so anything that might need a connection has one.
         val startedImpl = initialiseDatabasePersistence {
             val tokenizableServices = makeServices()
-
+            saveOwnNodeInfo()
             smm = StateMachineManager(services,
                     checkpointStorage,
                     serverThread,
@@ -391,6 +407,7 @@ abstract class AbstractNode(open val configuration: NodeConfiguration,
                 services.transactionVerifierService, services.validatedTransactions, services.contractUpgradeService,
                 services, cordappProvider, this)
         makeNetworkServices(tokenizableServices)
+
         return tokenizableServices
     }
 

node/src/main/kotlin/net/corda/node/internal/Node.kt

@@ -310,6 +310,11 @@ open class Node(override val configuration: FullNodeConfiguration,
     private val _startupComplete = openFuture<Unit>()
     val startupComplete: CordaFuture<Unit> get() = _startupComplete
 
+    override fun generateNodeInfo() {
+        initialiseSerialization()
+        super.generateNodeInfo()
+    }
+
     override fun start(): StartedNode<Node> {
         if (initialiseSerialization) {
             initialiseSerialization()

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -92,18 +92,24 @@ open class NodeStartup(val args: Array<String>) {
 
     open protected fun startNode(conf: FullNodeConfiguration, versionInfo: VersionInfo, startTime: Long, cmdlineOptions: CmdLineOptions) {
         val advertisedServices = conf.calculateServices()
-        val node = createNode(conf, versionInfo, advertisedServices).start()
+        val node = createNode(conf, versionInfo, advertisedServices)
-        printPluginsAndServices(node.internals)
+        if (cmdlineOptions.justGenerateNodeInfo) {
-        node.internals.nodeReadyFuture.thenMatch({
+            // Perform the minimum required start-up logic to be able to write a nodeInfo to disk
+            node.generateNodeInfo()
+            return
+        }
+        val startedNode = node.start()
+        printPluginsAndServices(startedNode.internals)
+        startedNode.internals.nodeReadyFuture.thenMatch({
             val elapsed = (System.currentTimeMillis() - startTime) / 10 / 100.0
-            val name = node.info.legalIdentitiesAndCerts.first().name.organisation
+            val name = startedNode.info.legalIdentitiesAndCerts.first().name.organisation
             Node.printBasicNodeInfo("Node for \"$name\" started up and registered in $elapsed sec")
 
             // Don't start the shell if there's no console attached.
             val runShell = !cmdlineOptions.noLocalShell && System.console() != null
-            node.internals.startupComplete.then {
+            startedNode.internals.startupComplete.then {
                 try {
-                    InteractiveShell.startShell(cmdlineOptions.baseDirectory, runShell, cmdlineOptions.sshdServer, node)
+                    InteractiveShell.startShell(cmdlineOptions.baseDirectory, runShell, cmdlineOptions.sshdServer, startedNode)
                 } catch(e: Throwable) {
                     logger.error("Shell failed to start", e)
                 }
@@ -112,7 +118,7 @@ open class NodeStartup(val args: Array<String>) {
         {
             th -> logger.error("Unexpected exception during registration", th)
         })
-        node.internals.run()
+        startedNode.internals.run()
     }
 
     open protected fun logStartupInfo(versionInfo: VersionInfo, cmdlineOptions: CmdLineOptions, conf: FullNodeConfiguration) {

node/src/main/kotlin/net/corda/node/services/network/NodeInfoSerializer.kt

@@ -0,0 +1,85 @@
+package net.corda.node.services.network
+
+import net.corda.cordform.CordformNode
+import net.corda.core.crypto.SecureHash
+import net.corda.core.crypto.SignedData
+import net.corda.core.internal.createDirectories
+import net.corda.core.internal.div
+import net.corda.core.internal.isDirectory
+import net.corda.core.node.NodeInfo
+import net.corda.core.node.services.KeyManagementService
+import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.serialize
+import net.corda.core.utilities.ByteSequence
+import net.corda.core.utilities.loggerFor
+import java.io.File
+import java.nio.file.Files
+import java.nio.file.Path
+
+/**
+ * Class containing the logic to serialize and de-serialize a [NodeInfo] to disk and reading it back.
+ */
+class NodeInfoSerializer {
+
+    companion object {
+        val logger = loggerFor<NodeInfoSerializer>()
+    }
+
+    /**
+     * Saves the given [NodeInfo] to a path.
+     * The node is 'encoded' as a SignedData<NodeInfo>, signed with the owning key of its first identity.
+     * The name of the written file will be "nodeInfo-" followed by the hash of the content. The hash in the filename
+     * is used so that one can freely copy these files without fearing to overwrite another one.
+     *
+     * @param path the path where to write the file, if non-existent it will be created.
+     * @param nodeInfo the NodeInfo to serialize.
+     * @param keyManager a KeyManagementService used to sign the NodeInfo data.
+     */
+    fun saveToFile(path: Path, nodeInfo: NodeInfo, keyManager: KeyManagementService) {
+        try {
+            path.createDirectories()
+            val serializedBytes = nodeInfo.serialize()
+            val regSig = keyManager.sign(serializedBytes.bytes, nodeInfo.legalIdentities.first().owningKey)
+            val signedData = SignedData(serializedBytes, regSig)
+            val file = (path / ("nodeInfo-" + SecureHash.sha256(serializedBytes.bytes).toString())).toFile()
+            file.writeBytes(signedData.serialize().bytes)
+        } catch (e : Exception) {
+            logger.warn("Couldn't write node info to file: $e")
+        }
+    }
+
+    /**
+     * Loads all the files contained in a given path and returns the deserialized [NodeInfo]s.
+     * Signatures are checked before returning a value.
+     *
+     * @param nodePath the node base path. NodeInfo files are searched for in nodePath/[NODE_INFO_FOLDER]
+     * @return a list of [NodeInfo]s
+     */
+    fun loadFromDirectory(nodePath: Path): List<NodeInfo> {
+        val result = mutableListOf<NodeInfo>()
+        val nodeInfoDirectory = nodePath / CordformNode.NODE_INFO_DIRECTORY
+        if (!nodeInfoDirectory.isDirectory()) {
+            logger.info("$nodeInfoDirectory isn't a Directory, not loading NodeInfo from files")
+            return result
+        }
+        for (path in Files.list(nodeInfoDirectory)) {
+            val file = path.toFile()
+            if (file.isFile) {
+                try {
+                    logger.info("Reading NodeInfo from file: $file")
+                    val nodeInfo = loadFromFile(file)
+                    result.add(nodeInfo)
+                } catch (e: Exception) {
+                    logger.error("Exception parsing NodeInfo from file. $file" , e)
+                }
+            }
+        }
+        logger.info("Succesfully read ${result.size} NodeInfo files.")
+        return result
+    }
+
+    private fun loadFromFile(file: File): NodeInfo {
+        val signedData = ByteSequence.of(file.readBytes()).deserialize<SignedData<NodeInfo>>()
+        return signedData.verified()
+    }
+}

node/src/main/kotlin/net/corda/node/services/network/PersistentNetworkMapCache.kt

@@ -1,11 +1,11 @@
 package net.corda.node.services.network
 
 import net.corda.core.concurrent.CordaFuture
+import net.corda.core.internal.bufferUntilSubscribed
 import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.internal.VisibleForTesting
-import net.corda.core.internal.bufferUntilSubscribed
 import net.corda.core.internal.concurrent.map
 import net.corda.core.internal.concurrent.openFuture
 import net.corda.core.messaging.DataFeed
@@ -88,9 +88,17 @@ open class PersistentNetworkMapCache(private val serviceHub: ServiceHubInternal)
         }
 
     init {
+        loadFromFiles()
         serviceHub.database.transaction { loadFromDB() }
     }
 
+    private fun loadFromFiles() {
+        logger.info("Loading network map from files..")
+        for (node in NodeInfoSerializer().loadFromDirectory(serviceHub.configuration.baseDirectory)) {
+            addNode(node)
+        }
+    }
+
     override fun getPartyInfo(party: Party): PartyInfo? {
         val nodes = serviceHub.database.transaction { queryByIdentityKey(party.owningKey) }
         if (nodes.size == 1 && nodes[0].isLegalIdentity(party)) {
@@ -159,6 +167,12 @@ open class PersistentNetworkMapCache(private val serviceHub: ServiceHubInternal)
     override fun addNode(node: NodeInfo) {
         logger.info("Adding node with info: $node")
         synchronized(_changed) {
+            registeredNodes[node.legalIdentities.first().owningKey]?.let {
+                if (it.serial > node.serial) {
+                    logger.info("Discarding older nodeInfo for ${node.legalIdentities.first().name}")
+                    return
+                }
+            }
             val previousNode = registeredNodes.put(node.legalIdentities.first().owningKey, node) // TODO hack... we left the first one as special one
             if (previousNode == null) {
                 logger.info("No previous node found")
@@ -225,8 +239,6 @@ open class PersistentNetworkMapCache(private val serviceHub: ServiceHubInternal)
         }
 
     private fun processRegistration(reg: NodeRegistration) {
-        // TODO: Implement filtering by sequence number, so we only accept changes that are
-        // more recent than the latest change we've processed.
         when (reg.type) {
             AddOrRemove.ADD -> addNode(reg.node)
             AddOrRemove.REMOVE -> removeNode(reg.node)
@@ -263,8 +275,7 @@ open class PersistentNetworkMapCache(private val serviceHub: ServiceHubInternal)
                     logger.info("Loaded node info: $nodeInfo")
                     val publicKey = parsePublicKeyBase58(nodeInfo.legalIdentitiesAndCerts.single { it.isMain }.owningKey)
                     val node = nodeInfo.toNodeInfo()
-                    registeredNodes.put(publicKey, node)
+                    addNode(node)
-                    changePublisher.onNext(MapChange.Added(node)) // Redeploy bridges after reading from DB on startup.
                     _loadDBSuccess = true // This is used in AbstractNode to indicate that node is ready.
                 } catch (e: Exception) {
                     logger.warn("Exception parsing network map from the database.", e)

node/src/test/kotlin/net/corda/node/ArgsParserTest.kt

@@ -23,7 +23,8 @@ class ArgsParserTest {
                 isRegistration = false,
                 isVersion = false,
                 noLocalShell = false,
-                sshdServer = false))
+                sshdServer = false,
+                justGenerateNodeInfo = false))
     }
 
     @Test
@@ -117,4 +118,10 @@ class ArgsParserTest {
         val cmdLineOptions = parser.parse("--version")
         assertThat(cmdLineOptions.isVersion).isTrue()
     }
+
+    @Test
+    fun `generate node infos`() {
+        val cmdLineOptions = parser.parse("--just-generate-node-info")
+        assertThat(cmdLineOptions.justGenerateNodeInfo).isTrue()
+    }
 }

node/src/test/kotlin/net/corda/node/services/network/NodeInfoSerializerTest.kt

@@ -0,0 +1,67 @@
+package net.corda.node.services.network
+
+import net.corda.cordform.CordformNode
+import net.corda.core.internal.div
+import net.corda.core.node.NodeInfo
+import net.corda.core.node.services.KeyManagementService
+import net.corda.node.services.identity.InMemoryIdentityService
+import net.corda.testing.*
+import net.corda.testing.node.MockKeyManagementService
+import net.corda.testing.node.NodeBasedTest
+import org.junit.Before
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import java.nio.charset.Charset
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.contentOf
+
+class NodeInfoSerializerTest : NodeBasedTest() {
+
+    @Rule @JvmField var folder = TemporaryFolder()
+
+    lateinit var keyManagementService: KeyManagementService
+
+    // Object under test
+    val nodeInfoSerializer = NodeInfoSerializer()
+
+    companion object {
+        val nodeInfoFileRegex = Regex("nodeInfo\\-.*")
+        val nodeInfo = NodeInfo(listOf(), listOf(getTestPartyAndCertificate(ALICE)), 0, 0)
+    }
+
+    @Before
+    fun start() {
+        val identityService = InMemoryIdentityService(trustRoot = DEV_TRUST_ROOT)
+        keyManagementService = MockKeyManagementService(identityService, ALICE_KEY)
+    }
+
+    @Test
+    fun `save a NodeInfo`() {
+        nodeInfoSerializer.saveToFile(folder.root.toPath(), nodeInfo, keyManagementService)
+
+        assertEquals(1, folder.root.list().size)
+        val fileName = folder.root.list()[0]
+        assertTrue(fileName.matches(nodeInfoFileRegex))
+        val file = (folder.root.path / fileName).toFile()
+        // Just check that something is written, another tests verifies that the written value can be read back.
+        assertThat(contentOf(file)).isNotEmpty()
+    }
+
+    @Test
+    fun `load an empty Directory`() {
+        assertEquals(0, nodeInfoSerializer.loadFromDirectory(folder.root.toPath()).size)
+    }
+
+    @Test
+    fun `load a non empty Directory`() {
+        val nodeInfoFolder = folder.newFolder(CordformNode.NODE_INFO_DIRECTORY)
+        nodeInfoSerializer.saveToFile(nodeInfoFolder.toPath(), nodeInfo, keyManagementService)
+        val nodeInfos = nodeInfoSerializer.loadFromDirectory(folder.root.toPath())
+
+        assertEquals(1, nodeInfos.size)
+        assertEquals(nodeInfo, nodeInfos.first())
+    }
+}