From 3e00676851a86dd1adaebec5ad8b8784035b58d6 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 9 Jan 2018 08:17:59 +0000 Subject: [PATCH] =?UTF-8?q?Raft=20notaries=20can=20share=20a=20single=20ke?= =?UTF-8?q?y=20pair=20for=20the=20service=20identity=20(i=E2=80=A6=20(#226?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Raft notaries can share a single key pair for the service identity (in contrast to a shared composite public key, and individual signing key pairs). This allows adjusting the cluster size on the fly. --- .../nodeapi/internal/DevIdentityGenerator.kt | 70 ++++++++---- .../node/services/BFTNotaryServiceTests.kt | 2 +- .../node/services/DistributedServiceTests.kt | 106 ++++++++++-------- .../net/corda/node/internal/AbstractNode.kt | 8 +- .../net/corda/notarydemo/BFTNotaryCordform.kt | 2 +- .../kotlin/net/corda/notarydemo/Notarise.kt | 4 +- .../corda/notarydemo/RaftNotaryCordform.kt | 2 +- .../net/corda/testing/node/NotarySpec.kt | 6 +- .../testing/node/internal/DriverDSLImpl.kt | 57 +++++++--- .../testing/node/internal/DummyClusterSpec.kt | 20 ++++ 10 files changed, 184 insertions(+), 93 deletions(-) create mode 100644 testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt index 0cb8472f89..6b17aa4049 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt @@ -12,6 +12,9 @@ import net.corda.nodeapi.internal.config.NodeSSLConfiguration import net.corda.nodeapi.internal.crypto.* import org.slf4j.LoggerFactory import java.nio.file.Path +import java.security.KeyPair +import java.security.PublicKey +import java.security.cert.X509Certificate /** * Contains utility methods for generating identities for a node. @@ -47,37 +50,56 @@ object DevIdentityGenerator { return identity.party } - fun generateDistributedNotaryIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { + fun generateDistributedNotaryCompositeIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { require(dirs.isNotEmpty()) - log.trace { "Generating identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } - val keyPairs = (1..dirs.size).map { generateKeyPair() } - val compositeKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) - + log.trace { "Generating composite identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getCertificate(X509Utilities.CORDA_ROOT_CA) + val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) + val keyPairs = (1..dirs.size).map { generateKeyPair() } + val notaryKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) keyPairs.zip(dirs) { keyPair, nodeDir -> - val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, compositeKey).map { publicKey -> - X509Utilities.createCertificate( - CertificateType.SERVICE_IDENTITY, - intermediateCa.certificate, - intermediateCa.keyPair, - notaryName.x500Principal, - publicKey) - } - val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" - val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") - keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) - keystore.setKeyEntry( - "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", - keyPair.private, - "cordacadevkeypass".toCharArray(), - arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) - keystore.save(distServKeyStoreFile, "cordacadevpass") + generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, nodeDir, rootCert) } - return Party(notaryName, compositeKey) + return Party(notaryName, notaryKey) + } + + fun generateDistributedNotarySingularIdentity(dirs: List, notaryName: CordaX500Name): Party { + require(dirs.isNotEmpty()) + + log.trace { "Generating singular identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } + val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") + val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") + val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) + + val keyPair = generateKeyPair() + val notaryKey = keyPair.public + dirs.forEach { dir -> + generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, dir, rootCert) + } + return Party(notaryName, notaryKey) + } + + private fun generateCertificates(keyPair: KeyPair, notaryKey: PublicKey, intermediateCa: CertificateAndKeyPair, notaryName: CordaX500Name, nodeDir: Path, rootCert: X509Certificate) { + val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, notaryKey).map { publicKey -> + X509Utilities.createCertificate( + CertificateType.SERVICE_IDENTITY, + intermediateCa.certificate, + intermediateCa.keyPair, + notaryName.x500Principal, + publicKey) + } + val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" + val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") + keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) + keystore.setKeyEntry( + "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", + keyPair.private, + "cordacadevkeypass".toCharArray(), + arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) + keystore.save(distServKeyStoreFile, "cordacadevpass") } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt index 0fa5ed8d36..bdaec3ec16 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt @@ -60,7 +60,7 @@ class BFTNotaryServiceTests { (Paths.get("config") / "currentView").deleteIfExists() // XXX: Make config object warn if this exists? val replicaIds = (0 until clusterSize) - notary = DevIdentityGenerator.generateDistributedNotaryIdentity( + notary = DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( replicaIds.map { mockNet.baseDirectory(mockNet.nextNodeId + it) }, CordaX500Name("BFT", "Zurich", "CH")) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt index 92723d52d8..9720df12f8 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt @@ -16,11 +16,11 @@ import net.corda.node.services.Permissions.Companion.startFlow import net.corda.nodeapi.internal.config.User import net.corda.testing.* import net.corda.testing.driver.NodeHandle +import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver -import net.corda.testing.node.ClusterSpec import net.corda.testing.node.NotarySpec +import net.corda.testing.node.internal.DummyClusterSpec import org.assertj.core.api.Assertions.assertThat -import org.junit.Ignore import org.junit.Test import rx.Observable import java.util.* @@ -32,18 +32,23 @@ class DistributedServiceTests { private lateinit var raftNotaryIdentity: Party private lateinit var notaryStateMachines: Observable> - private fun setup(testBlock: () -> Unit) { + private fun setup(compositeIdentity: Boolean = false, testBlock: () -> Unit) { val testUser = User("test", "test", permissions = setOf( startFlow(), startFlow(), invokeRpc(CordaRPCOps::nodeInfo), invokeRpc(CordaRPCOps::stateMachinesFeed)) ) - driver( extraCordappPackagesToScan = listOf("net.corda.finance.contracts"), - notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, rpcUsers = listOf(testUser), cluster = ClusterSpec.Raft(clusterSize = 3)))) - { + notarySpecs = listOf( + NotarySpec( + DUMMY_NOTARY_NAME, + rpcUsers = listOf(testUser), + cluster = DummyClusterSpec(clusterSize = 3, compositeServiceIdentity = compositeIdentity)) + ), + portAllocation = PortAllocation.RandomFree + ) { alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(testUser)).getOrThrow() raftNotaryIdentity = defaultNotaryIdentity notaryNodes = defaultNotaryHandle.nodeHandles.getOrThrow().map { it as NodeHandle.OutOfProcess } @@ -72,11 +77,60 @@ class DistributedServiceTests { } } + // TODO This should be in RaftNotaryServiceTests + @Test + fun `cluster survives if a notary is killed`() { + setup { + // Issue 100 pounds, then pay ourselves 10x5 pounds + issueCash(100.POUNDS) + + for (i in 1..10) { + paySelf(5.POUNDS) + } + + // Now kill a notary node + with(notaryNodes[0].process) { + destroy() + waitFor() + } + + // Pay ourselves another 20x5 pounds + for (i in 1..20) { + paySelf(5.POUNDS) + } + + val notarisationsPerNotary = HashMap() + notaryStateMachines.expectEvents(isStrict = false) { + replicate>(30) { + expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> + update as StateMachineUpdate.Added + notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } + } + } + } + + println("Notarisation distribution: $notarisationsPerNotary") + require(notarisationsPerNotary.size == 3) + } + } + // TODO Use a dummy distributed service rather than a Raft Notary Service as this test is only about Artemis' ability // to handle distributed services - @Ignore("Test has undeterministic capacity to hang, ignore till fixed") @Test - fun `requests are distributed evenly amongst the nodes`() = setup { + fun `requests are distributed evenly amongst the nodes`() { + setup { + checkRequestsDistributedEvenly() + } + } + + @Test + fun `requests are distributed evenly amongst the nodes with a composite public key`() { + setup(true) { + checkRequestsDistributedEvenly() + } + } + + private fun checkRequestsDistributedEvenly() { // Issue 100 pounds, then pay ourselves 50x2 pounds issueCash(100.POUNDS) @@ -102,42 +156,6 @@ class DistributedServiceTests { require(notarisationsPerNotary.values.all { it > 10 }) } - // TODO This should be in RaftNotaryServiceTests - @Ignore("Test has undeterministic capacity to hang, ignore till fixed") - @Test - fun `cluster survives if a notary is killed`() = setup { - // Issue 100 pounds, then pay ourselves 10x5 pounds - issueCash(100.POUNDS) - - for (i in 1..10) { - paySelf(5.POUNDS) - } - - // Now kill a notary node - with(notaryNodes[0].process) { - destroy() - waitFor() - } - - // Pay ourselves another 20x5 pounds - for (i in 1..20) { - paySelf(5.POUNDS) - } - - val notarisationsPerNotary = HashMap() - notaryStateMachines.expectEvents(isStrict = false) { - replicate>(30) { - expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> - update as StateMachineUpdate.Added - notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } - } - } - } - - println("Notarisation distribution: $notarisationsPerNotary") - require(notarisationsPerNotary.size == 3) - } - private fun issueCash(amount: Amount) { aliceProxy.startFlow(::CashIssueFlow, amount, OpaqueBytes.of(0), raftNotaryIdentity).returnValue.getOrThrow() } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 0edc5a1624..6d864f7538 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -173,11 +173,11 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } private inline fun signNodeInfo(nodeInfo: NodeInfo, sign: (PublicKey, SerializedBytes) -> DigitalSignature): SignedNodeInfo { - // For now we assume the node has only one identity (excluding any composite ones) - val owningKey = nodeInfo.legalIdentities.single { it.owningKey !is CompositeKey }.owningKey + // For now we exclude any composite identities, see [SignedNodeInfo] + val owningKeys = nodeInfo.legalIdentities.map { it.owningKey }.filter { it !is CompositeKey } val serialised = nodeInfo.serialize() - val signature = sign(owningKey, serialised) - return SignedNodeInfo(serialised, listOf(signature)) + val signatures = owningKeys.map { sign(it, serialised) } + return SignedNodeInfo(serialised, signatures) } open fun generateAndSaveNodeInfo(): NodeInfo { diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt index 1b2967c953..8e4028956f 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt @@ -62,7 +62,7 @@ class BFTNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotaryIdentity( + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName, minCorrectReplicas(clusterSize) diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt index 6a59809cf6..a86a6be0ea 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt @@ -1,6 +1,7 @@ package net.corda.notarydemo import net.corda.client.rpc.CordaRPCClient +import net.corda.core.crypto.CompositeKey import net.corda.core.crypto.toStringShort import net.corda.core.identity.PartyAndCertificate import net.corda.core.messaging.CordaRPCOps @@ -38,7 +39,8 @@ private class NotaryDemoClientApi(val rpc: CordaRPCOps) { /** Makes calls to the node rpc to start transaction notarisation. */ fun notarise(count: Int) { - println("Notary: \"${notary.name}\", with composite key: ${notary.owningKey.toStringShort()}") + val keyType = if (notary.owningKey is CompositeKey) "composite" else "public" + println("Notary: \"${notary.name}\", with $keyType key: ${notary.owningKey.toStringShort()}") val transactions = buildTransactions(count) println("Notarised ${transactions.size} transactions:") transactions.zip(notariseTransactions(transactions)).forEach { (tx, signersFuture) -> diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt index abceabbe77..f999ad68e6 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt @@ -58,7 +58,7 @@ class RaftNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotaryIdentity( + DevIdentityGenerator.generateDistributedNotarySingularIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName ) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt index b6533ebd98..128ed9070b 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt @@ -14,10 +14,12 @@ data class NotarySpec( ) @DoNotImplement -sealed class ClusterSpec { +abstract class ClusterSpec { abstract val clusterSize: Int - data class Raft(override val clusterSize: Int) : ClusterSpec() { + data class Raft( + override val clusterSize: Int + ) : ClusterSpec() { init { require(clusterSize > 0) } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index e7b65d2572..1e4f710449 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -267,7 +267,7 @@ class DriverDSLImpl( if (cordform.notary == null) continue val name = CordaX500Name.parse(cordform.name) val notaryConfig = ConfigFactory.parseMap(cordform.notary).parseAs() - // We need to first group the nodes that form part of a cluser. We assume for simplicity that nodes of the + // We need to first group the nodes that form part of a cluster. We assume for simplicity that nodes of the // same cluster type and validating flag are part of the same cluster. if (notaryConfig.raft != null) { val key = if (notaryConfig.validating) VALIDATING_RAFT else NON_VALIDATING_RAFT @@ -282,10 +282,17 @@ class DriverDSLImpl( } clusterNodes.asMap().forEach { type, nodeNames -> - val identity = DevIdentityGenerator.generateDistributedNotaryIdentity( - dirs = nodeNames.map { baseDirectory(it) }, - notaryName = type.clusterName - ) + val identity = if (type == ClusterType.NON_VALIDATING_RAFT || type == ClusterType.VALIDATING_RAFT) { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = nodeNames.map { baseDirectory(it) }, + notaryName = type.clusterName + ) + } else { + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + dirs = nodeNames.map { baseDirectory(it) }, + notaryName = type.clusterName + ) + } notaryInfos += NotaryInfo(identity, type.validating) } @@ -382,13 +389,30 @@ class DriverDSLImpl( private fun startNotaryIdentityGeneration(): CordaFuture> { return executorService.fork { notarySpecs.map { spec -> - val identity = if (spec.cluster == null) { - DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) - } else { - DevIdentityGenerator.generateDistributedNotaryIdentity( - dirs = generateNodeNames(spec).map { baseDirectory(it) }, - notaryName = spec.name - ) + val identity = when (spec.cluster) { + null -> { + DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) + } + is ClusterSpec.Raft -> { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } + is DummyClusterSpec -> { + if (spec.cluster.compositeServiceIdentity) { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } else { + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } + } + else -> throw UnsupportedOperationException("Cluster spec ${spec.cluster} not supported by Driver") } NotaryInfo(identity, spec.validating) } @@ -433,9 +457,12 @@ class DriverDSLImpl( private fun startNotaries(localNetworkMap: LocalNetworkMap?): List>> { return notarySpecs.map { - when { - it.cluster == null -> startSingleNotary(it, localNetworkMap) - it.cluster is ClusterSpec.Raft -> startRaftNotaryCluster(it, localNetworkMap) + when (it.cluster) { + null -> startSingleNotary(it, localNetworkMap) + is ClusterSpec.Raft, + // DummyCluster is used for testing the notary communication path, and it does not matter + // which underlying consensus algorithm is used, so we just stick to Raft + is DummyClusterSpec -> startRaftNotaryCluster(it, localNetworkMap) else -> throw IllegalArgumentException("BFT-SMaRt not supported") } } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt new file mode 100644 index 0000000000..13a5d3df2a --- /dev/null +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt @@ -0,0 +1,20 @@ +package net.corda.testing.node.internal + +import net.corda.testing.node.ClusterSpec + +/** + * Only used for testing the notary communication path. Can be configured to act as a Raft (singular identity), + * or a BFT (composite key identity) notary service. + */ +data class DummyClusterSpec( + override val clusterSize: Int, + /** + * If *true*, the cluster will use a shared composite public key for the service identity, with individual + * private keys. If *false*, the same "singular" key pair will be shared by all replicas. + */ + val compositeServiceIdentity: Boolean = false +) : ClusterSpec() { + init { + require(clusterSize > 0) + } +} \ No newline at end of file