CORDA-535: Move implementation specific configuration values out of n… (#4058)

The configuration objects for specific notary implementations have been replaced
by a single untyped "extraConfig" Config object that is left to the notary service
itself to parse.

* Remove the raft bootstrapping command from node, we'll need a different
mechanism for that.

* Remove pre-generated identity config value.

* Split up obtainIdentity() in AbstractNode to make it easier to read.

* A temporary workaround for the bootstrapper tool to support BFT notaries.

* Update docs

* Add upgrade notes

* Fix rebase issue

* Add a config diff for the bft notary as well

node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt

@@ -88,12 +88,6 @@ class NodeCmdLineOptions {
     )
     var justGenerateRpcSslCerts: Boolean = false
 
-    @Option(
-            names = ["--bootstrap-raft-cluster"],
-            description = ["Bootstraps Raft cluster. The node forms a single node cluster (ignoring otherwise configured peer addresses), acting as a seed for other nodes to join the cluster."]
-    )
-    var bootstrapRaftCluster: Boolean = false
-
     @Option(
             names = ["-c", "--clear-network-map-cache"],
             description = ["Clears local copy of network map, on node startup it will be restored from server or file system."]

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -268,7 +268,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         check(started == null) { "Node has already been started" }
         log.info("Generating nodeInfo ...")
         val trustRoot = initKeyStores()
-        val (identity, identityKeyPair) = obtainIdentity(notaryConfig = null)
+        val (identity, identityKeyPair) = obtainIdentity()
         startDatabase()
         val nodeCa = configuration.signingCertificateStore.get()[CORDA_CLIENT_CA]
         identityService.start(trustRoot, listOf(identity.certificate, nodeCa))
@@ -323,7 +323,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         networkMapCache.start(netParams.notaries)
 
         startDatabase()
-        val (identity, identityKeyPair) = obtainIdentity(notaryConfig = null)
+        val (identity, identityKeyPair) = obtainIdentity()
         identityService.start(trustRoot, listOf(identity.certificate, nodeCa))
 
         val (keyPairs, nodeInfoAndSigned, myNotaryIdentity) = database.transaction {
@@ -400,8 +400,8 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         val keyPairs = mutableSetOf(identityKeyPair)
 
         val myNotaryIdentity = configuration.notary?.let {
-            if (it.isClusterConfig) {
-                val (notaryIdentity, notaryIdentityKeyPair) = obtainIdentity(it)
+            if (it.serviceLegalName != null) {
+                val (notaryIdentity, notaryIdentityKeyPair) = loadNotaryClusterIdentity(it.serviceLegalName)
                 keyPairs += notaryIdentityKeyPair
                 notaryIdentity
             } else {
@@ -840,24 +840,14 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
                                                  myNotaryIdentity: PartyAndCertificate?,
                                                  networkParameters: NetworkParameters)
 
-    private fun obtainIdentity(notaryConfig: NotaryConfig?): Pair<PartyAndCertificate, KeyPair> {
+    /** Loads or generates the node's legal identity and key-pair. */
+    private fun obtainIdentity(): Pair<PartyAndCertificate, KeyPair> {
         val keyStore = configuration.signingCertificateStore.get()
+        val legalName = configuration.myLegalName
 
-        val (id, singleName) = if (notaryConfig == null || !notaryConfig.isClusterConfig) {
-            // Node's main identity or if it's a single node notary.
-            Pair(NODE_IDENTITY_ALIAS_PREFIX, configuration.myLegalName)
-        } else {
-            // The node is part of a distributed notary whose identity must already be generated beforehand.
-            Pair(DISTRIBUTED_NOTARY_ALIAS_PREFIX, null)
-        }
         // TODO: Integrate with Key management service?
-        val privateKeyAlias = "$id-private-key"
-
+        val privateKeyAlias = "$NODE_IDENTITY_ALIAS_PREFIX-private-key"
         if (privateKeyAlias !in keyStore) {
-            // We shouldn't have a distributed notary at this stage, so singleName should NOT be null.
-            requireNotNull(singleName) {
-                "Unable to find in the key store the identity of the distributed notary the node is part of"
-            }
             log.info("$privateKeyAlias not found in key store, generating fresh key!")
             keyStore.storeLegalIdentity(privateKeyAlias, generateKeyPair())
         }
@@ -865,30 +855,41 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         val (x509Cert, keyPair) = keyStore.query { getCertificateAndKeyPair(privateKeyAlias, keyStore.entryPassword) }
 
         // TODO: Use configuration to indicate composite key should be used instead of public key for the identity.
-        val compositeKeyAlias = "$id-composite-key"
+        val certificates = keyStore.query { getCertificateChain(privateKeyAlias) }
+        check(certificates.first() == x509Cert) {
+            "Certificates from key store do not line up!"
+        }
+
+        val subject = CordaX500Name.build(certificates.first().subjectX500Principal)
+        if (subject != legalName) {
+            throw ConfigurationException("The name '$legalName' for $NODE_IDENTITY_ALIAS_PREFIX doesn't match what's in the key store: $subject")
+        }
+
+        val certPath = X509Utilities.buildCertPath(certificates)
+        return Pair(PartyAndCertificate(certPath), keyPair)
+    }
+
+    /** Loads pre-generated notary service cluster identity. */
+    private fun loadNotaryClusterIdentity(serviceLegalName: CordaX500Name): Pair<PartyAndCertificate, KeyPair> {
+        val keyStore = configuration.signingCertificateStore.get()
+
+        val privateKeyAlias = "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key"
+        val keyPair = keyStore.query { getCertificateAndKeyPair(privateKeyAlias) }.keyPair
+
+        val compositeKeyAlias = "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key"
         val certificates = if (compositeKeyAlias in keyStore) {
-            // Use composite key instead if it exists.
             val certificate = keyStore[compositeKeyAlias]
             // We have to create the certificate chain for the composite key manually, this is because we don't have a keystore
             // provider that understand compositeKey-privateKey combo. The cert chain is created using the composite key certificate +
             // the tail of the private key certificates, as they are both signed by the same certificate chain.
             listOf(certificate) + keyStore.query { getCertificateChain(privateKeyAlias) }.drop(1)
-        } else {
-            keyStore.query { getCertificateChain(privateKeyAlias) }.let {
-                check(it[0] == x509Cert) { "Certificates from key store do not line up!" }
-                it
-            }
-        }
+        } else throw IllegalStateException("The identity public key for the notary service $serviceLegalName was not found in the key store.")
 
-        val subject = CordaX500Name.build(certificates[0].subjectX500Principal)
-        if (singleName != null && subject != singleName) {
-            throw ConfigurationException("The name '$singleName' for $id doesn't match what's in the key store: $subject")
-        } else if (notaryConfig != null && notaryConfig.isClusterConfig && notaryConfig.serviceLegalName != null && subject != notaryConfig.serviceLegalName) {
-            // Note that we're not checking if `notaryConfig.serviceLegalName` is not present for backwards compatibility.
-            throw ConfigurationException("The name of the notary service '${notaryConfig.serviceLegalName}' for $id doesn't " +
+        val subject = CordaX500Name.build(certificates.first().subjectX500Principal)
+        if (subject != serviceLegalName) {
+            throw ConfigurationException("The name of the notary service '$serviceLegalName' for $DISTRIBUTED_NOTARY_ALIAS_PREFIX doesn't " +
                     "match what's in the key store: $subject. You might need to adjust the configuration of `notary.serviceLegalName`.")
         }
-
         val certPath = X509Utilities.buildCertPath(certificates)
         return Pair(PartyAndCertificate(certPath), keyPair)
     }

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -18,7 +18,6 @@ import net.corda.node.*
 import net.corda.node.internal.Node.Companion.isValidJavaVersion
 import net.corda.node.internal.cordapp.MultipleCordappsForFlowException
 import net.corda.node.services.config.NodeConfiguration
-import net.corda.node.services.config.NodeConfigurationImpl
 import net.corda.node.services.config.shouldStartLocalShell
 import net.corda.node.services.config.shouldStartSSHDaemon
 import net.corda.node.utilities.createKeyPairAndSelfSignedTLSCertificate
@@ -27,7 +26,6 @@ import net.corda.node.utilities.registration.NodeRegistrationException
 import net.corda.node.utilities.registration.NodeRegistrationHelper
 import net.corda.node.utilities.saveToKeyStore
 import net.corda.node.utilities.saveToTrustStore
-import net.corda.core.internal.PLATFORM_VERSION
 import net.corda.nodeapi.internal.addShutdownHook
 import net.corda.nodeapi.internal.config.UnknownConfigurationKeysException
 import net.corda.nodeapi.internal.persistence.CouldNotCreateDataSourceException
@@ -189,14 +187,7 @@ open class NodeStartup : CordaCliWrapper("corda", "Runs a Corda Node") {
         if (cmdLineOptions.devMode == true) {
             println("Config:\n${rawConfig.root().render(ConfigRenderOptions.defaults())}")
         }
-        val configuration = configurationResult.getOrThrow()
-        return if (cmdLineOptions.bootstrapRaftCluster) {
-            println("Bootstrapping raft cluster (starting up as seed node).")
-            // Ignore the configured clusterAddresses to make the node bootstrap a cluster instead of joining.
-            (configuration as NodeConfigurationImpl).copy(notary = configuration.notary?.copy(raft = configuration.notary?.raft?.copy(clusterAddresses = emptyList())))
-        } else {
-            configuration
-        }
+        return configurationResult.getOrThrow()
     }
 
     private fun checkRegistrationMode(): Boolean {
@@ -298,12 +289,12 @@ open class NodeStartup : CordaCliWrapper("corda", "Runs a Corda Node") {
         val console: Console? = System.console()
 
         when (console) {
-        // In this case, the JVM is not connected to the console so we need to exit.
+            // In this case, the JVM is not connected to the console so we need to exit.
             null -> {
                 println("Not connected to console. Exiting")
                 exitProcess(1)
             }
-        // Otherwise we can proceed normally.
+            // Otherwise we can proceed normally.
             else -> {
                 while (true) {
                     val keystorePassword1 = console.readPassword("Enter the RPC keystore password => ")

node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

@@ -11,12 +11,7 @@ import net.corda.core.utilities.loggerFor
 import net.corda.core.utilities.seconds
 import net.corda.node.services.config.rpc.NodeRpcOptions
 import net.corda.nodeapi.BrokerRpcSslOptions
-import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier
-import net.corda.nodeapi.internal.config.SslConfiguration
-import net.corda.nodeapi.internal.config.MutualSslConfiguration
-import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
-import net.corda.nodeapi.internal.config.User
-import net.corda.nodeapi.internal.config.parseAs
+import net.corda.nodeapi.internal.config.*
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.tools.shell.SSHDConfiguration
 import org.slf4j.Logger
@@ -73,7 +68,7 @@ interface NodeConfiguration {
     val flowMonitorPeriodMillis: Duration get() = DEFAULT_FLOW_MONITOR_PERIOD_MILLIS
     val flowMonitorSuspensionLoggingThresholdMillis: Duration get() = DEFAULT_FLOW_MONITOR_SUSPENSION_LOGGING_THRESHOLD_MILLIS
     val crlCheckSoftFail: Boolean
-    val jmxReporterType : JmxReporterType? get() = defaultJmxReporterType
+    val jmxReporterType: JmxReporterType? get() = defaultJmxReporterType
 
     val baseDirectory: Path
     val certificatesDirectory: Path
@@ -119,34 +114,16 @@ fun NodeConfiguration.shouldStartSSHDaemon() = this.sshd != null
 fun NodeConfiguration.shouldStartLocalShell() = !this.noLocalShell && System.console() != null && this.devMode
 fun NodeConfiguration.shouldInitCrashShell() = shouldStartLocalShell() || shouldStartSSHDaemon()
 
-data class NotaryConfig(val validating: Boolean,
-                        val raft: RaftConfig? = null,
-                        val bftSMaRt: BFTSMaRtConfiguration? = null,
-                        val serviceLegalName: CordaX500Name? = null,
-                        val className: String = "net.corda.node.services.transactions.SimpleNotaryService"
-) {
-    init {
-        require(raft == null || bftSMaRt == null) {
-            "raft and bftSMaRt configs cannot be specified together"
-        }
-    }
-
-    val isClusterConfig: Boolean get() = raft != null || bftSMaRt != null
-}
-
-data class RaftConfig(val nodeAddress: NetworkHostAndPort, val clusterAddresses: List<NetworkHostAndPort>)
-
-/** @param exposeRaces for testing only, so its default is not in reference.conf but here. */
-data class BFTSMaRtConfiguration(
-        val replicaId: Int,
-        val clusterAddresses: List<NetworkHostAndPort>,
-        val debug: Boolean = false,
-        val exposeRaces: Boolean = false
-) {
-    init {
-        require(replicaId >= 0) { "replicaId cannot be negative" }
-    }
-}
+data class NotaryConfig(
+        /** Specifies whether the notary validates transactions or not. */
+        val validating: Boolean,
+        /** The legal name of cluster in case of a distributed notary service. */
+        val serviceLegalName: CordaX500Name? = null,
+        /** The name of the notary service class to load. */
+        val className: String = "net.corda.node.services.transactions.SimpleNotaryService",
+        /** Notary implementation-specific configuration parameters. */
+        val extraConfig: Config? = null
+)
 
 /**
  * Used as an alternative to the older compatibilityZoneURL to allow the doorman and network map
@@ -167,7 +144,7 @@ data class NetworkServicesConfig(
         val doormanURL: URL,
         val networkMapURL: URL,
         val pnm: UUID? = null,
-        val inferred : Boolean = false
+        val inferred: Boolean = false
 )
 
 /**
@@ -360,7 +337,7 @@ data class NodeConfigurationImpl(
 
     override val effectiveH2Settings: NodeH2Settings?
         get() = when {
-            h2port != null -> NodeH2Settings(address = NetworkHostAndPort(host="localhost", port=h2port))
+            h2port != null -> NodeH2Settings(address = NetworkHostAndPort(host = "localhost", port = h2port))
             else -> h2Settings
         }
 
@@ -372,7 +349,7 @@ data class NodeConfigurationImpl(
             "Cannot specify both 'rpcUsers' and 'security' in configuration"
         }
         @Suppress("DEPRECATION")
-        if(certificateChainCheckPolicies.isNotEmpty()) {
+        if (certificateChainCheckPolicies.isNotEmpty()) {
             logger.warn("""You are configuring certificateChainCheckPolicies. This is a setting that is not used, and will be removed in a future version.
                 |Please contact the R3 team on the public slack to discuss your use case.
             """.trimMargin())

node/src/main/resources/net.corda.node.internal.NodeStartup.yml

@@ -6,11 +6,6 @@
     required: false
     multiParam: true
     acceptableValues: []
-  - parameterName: "--bootstrap-raft-cluster"
-    parameterType: "boolean"
-    required: false
-    multiParam: false
-    acceptableValues: []
   - parameterName: "--clear-network-map-cache"
     parameterType: "boolean"
     required: false

node/src/test/kotlin/net/corda/node/internal/NodeStartupTest.kt

@@ -35,7 +35,6 @@ class NodeStartupTest {
         assertThat(startup.cmdLineOptions.sshdServer).isEqualTo(false)
         assertThat(startup.cmdLineOptions.justGenerateNodeInfo).isEqualTo(false)
         assertThat(startup.cmdLineOptions.justGenerateRpcSslCerts).isEqualTo(false)
-        assertThat(startup.cmdLineOptions.bootstrapRaftCluster).isEqualTo(false)
         assertThat(startup.cmdLineOptions.unknownConfigKeysPolicy).isEqualTo(UnknownConfigKeysPolicy.FAIL)
         assertThat(startup.cmdLineOptions.devMode).isEqualTo(null)
         assertThat(startup.cmdLineOptions.clearNetworkMapCache).isEqualTo(false)

node/src/test/kotlin/net/corda/node/services/TimedFlowTests.kt

@@ -15,9 +15,7 @@ import net.corda.core.internal.bufferUntilSubscribed
 import net.corda.core.internal.notary.NotaryServiceFlow
 import net.corda.core.internal.notary.TrustedAuthorityNotaryService
 import net.corda.core.internal.notary.UniquenessProvider
-import net.corda.core.node.AppServiceHub
 import net.corda.core.node.NotaryInfo
-import net.corda.core.node.services.CordaService
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.ProgressTracker
@@ -82,13 +80,14 @@ class TimedFlowTests {
 
         private fun startClusterAndNode(mockNet: InternalMockNetwork): Pair<Party, TestStartedNode> {
             val replicaIds = (0 until CLUSTER_SIZE)
+            val serviceLegalName = CordaX500Name("Custom Notary", "Zurich", "CH")
             val notaryIdentity = DevIdentityGenerator.generateDistributedNotaryCompositeIdentity(
                     replicaIds.map { mockNet.baseDirectory(mockNet.nextNodeId + it) },
-                    CordaX500Name("Custom Notary", "Zurich", "CH"))
+                    serviceLegalName)
 
             val networkParameters = NetworkParametersCopier(testNetworkParameters(listOf(NotaryInfo(notaryIdentity, true))))
             val notaryConfig = mock<NotaryConfig> {
-                whenever(it.isClusterConfig).thenReturn(true)
+                whenever(it.serviceLegalName).thenReturn(serviceLegalName)
                 whenever(it.validating).thenReturn(true)
                 whenever(it.className).thenReturn(TestNotaryService::class.java.name)
             }