From 5df48e0f5ecded83ba47950aaceeaed8f5b6441d Mon Sep 17 00:00:00 2001 From: Anthony Keenan <34482776+anthonykr3@users.noreply.github.com> Date: Mon, 8 Jan 2018 14:42:51 +0000 Subject: [PATCH 01/17] Allow testing of platform version when building test node info (#2333) --- .../corda/testing/internal/TestNodeInfoBuilder.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestNodeInfoBuilder.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestNodeInfoBuilder.kt index a367cff7e6..bdeba56fc6 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestNodeInfoBuilder.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestNodeInfoBuilder.kt @@ -22,17 +22,17 @@ class TestNodeInfoBuilder { } } - fun build(serial: Long = 1): NodeInfo { + fun build(serial: Long = 1, platformVersion: Int = 1): NodeInfo { return NodeInfo( listOf(NetworkHostAndPort("my.${identitiesAndPrivateKeys[0].first.party.name.organisation}.com", 1234)), identitiesAndPrivateKeys.map { it.first }, - 1, + platformVersion, serial ) } - fun buildWithSigned(serial: Long = 1): Pair { - val nodeInfo = build(serial) + fun buildWithSigned(serial: Long = 1, platformVersion: Int = 1): Pair { + val nodeInfo = build(serial, platformVersion) val privateKeys = identitiesAndPrivateKeys.map { it.second } return Pair(nodeInfo, nodeInfo.signWith(privateKeys)) } @@ -42,10 +42,10 @@ class TestNodeInfoBuilder { } } -fun createNodeInfoAndSigned(vararg names: CordaX500Name, serial: Long = 1): Pair { +fun createNodeInfoAndSigned(vararg names: CordaX500Name, serial: Long = 1, platformVersion: Int = 1): Pair { val nodeInfoBuilder = TestNodeInfoBuilder() names.forEach { nodeInfoBuilder.addIdentity(it) } - return nodeInfoBuilder.buildWithSigned(serial) + return nodeInfoBuilder.buildWithSigned(serial, platformVersion) } fun NodeInfo.signWith(keys: List): SignedNodeInfo { From 38fd99792fab400fb05c3220b968d68ed0f4efab Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Wed, 13 Dec 2017 16:50:41 +0100 Subject: [PATCH 02/17] Minor: add some discussion to the JavaDocs about why attachments are zip files. --- .../kotlin/net/corda/core/contracts/Attachment.kt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/net/corda/core/contracts/Attachment.kt b/core/src/main/kotlin/net/corda/core/contracts/Attachment.kt index e55a91d546..641e730417 100644 --- a/core/src/main/kotlin/net/corda/core/contracts/Attachment.kt +++ b/core/src/main/kotlin/net/corda/core/contracts/Attachment.kt @@ -12,11 +12,20 @@ import java.util.jar.JarInputStream * An attachment is a ZIP (or an optionally signed JAR) that contains one or more files. Attachments are meant to * contain public static data which can be referenced from transactions and utilised from contracts. Good examples * of how attachments are meant to be used include: + * * - Calendar data * - Fixes (e.g. LIBOR) * - Smart contract code * - Legal documents - * - Facts generated by oracles which might be reused a lot + * - Facts generated by oracles which might be reused a lot. + * + * At the moment, non-ZIP attachments are not supported. Support may come in a future release. Using ZIP files for + * attachments makes it easy to ensure data on the ledger is compressed, which is useful considering that attachments + * may be widely replicated around the network. It also allows the jarsigner tool to be used to sign an attachment + * using ordinary certificates of the kind that many organisations already have, and improves the efficiency of + * attachment resolution in cases where the attachment is logically made up of many small files - e.g. is bytecode. + * Finally, using ZIPs ensures files have a timestamp associated with them, and enables informational attachments + * to be password protected (although in current releases password protected ZIPs are likely to fail to work). */ @CordaSerializable interface Attachment : NamedByHash { From 4a995870c8c462d8d64d5173879079ba852f3ec5 Mon Sep 17 00:00:00 2001 From: igor nitto Date: Mon, 8 Jan 2018 17:50:29 +0000 Subject: [PATCH 03/17] Fix example-node.conf in docs (#2334) --- docs/source/example-code/src/main/resources/example-node.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/example-code/src/main/resources/example-node.conf b/docs/source/example-code/src/main/resources/example-node.conf index 8ae9a42475..c01a10fd78 100644 --- a/docs/source/example-code/src/main/resources/example-node.conf +++ b/docs/source/example-code/src/main/resources/example-node.conf @@ -12,7 +12,7 @@ rpcAddress : "my-corda-node:10003" webAddress : "localhost:10004" useHTTPS : false rpcUsers : [ - { username=user1, password=letmein, permissions=[ StartProtocol.net.corda.protocols.CashProtocol ] } + { username=user1, password=letmein, permissions=[ StartFlow.net.corda.protocols.CashProtocol ] } ] devMode : true // certificateSigningService : "https://testnet.certificate.corda.net" From 3e00676851a86dd1adaebec5ad8b8784035b58d6 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 9 Jan 2018 08:17:59 +0000 Subject: [PATCH 04/17] =?UTF-8?q?Raft=20notaries=20can=20share=20a=20singl?= =?UTF-8?q?e=20key=20pair=20for=20the=20service=20identity=20(i=E2=80=A6?= =?UTF-8?q?=20(#2269)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Raft notaries can share a single key pair for the service identity (in contrast to a shared composite public key, and individual signing key pairs). This allows adjusting the cluster size on the fly. --- .../nodeapi/internal/DevIdentityGenerator.kt | 70 ++++++++---- .../node/services/BFTNotaryServiceTests.kt | 2 +- .../node/services/DistributedServiceTests.kt | 106 ++++++++++-------- .../net/corda/node/internal/AbstractNode.kt | 8 +- .../net/corda/notarydemo/BFTNotaryCordform.kt | 2 +- .../kotlin/net/corda/notarydemo/Notarise.kt | 4 +- .../corda/notarydemo/RaftNotaryCordform.kt | 2 +- .../net/corda/testing/node/NotarySpec.kt | 6 +- .../testing/node/internal/DriverDSLImpl.kt | 57 +++++++--- .../testing/node/internal/DummyClusterSpec.kt | 20 ++++ 10 files changed, 184 insertions(+), 93 deletions(-) create mode 100644 testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt index 0cb8472f89..6b17aa4049 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt @@ -12,6 +12,9 @@ import net.corda.nodeapi.internal.config.NodeSSLConfiguration import net.corda.nodeapi.internal.crypto.* import org.slf4j.LoggerFactory import java.nio.file.Path +import java.security.KeyPair +import java.security.PublicKey +import java.security.cert.X509Certificate /** * Contains utility methods for generating identities for a node. @@ -47,37 +50,56 @@ object DevIdentityGenerator { return identity.party } - fun generateDistributedNotaryIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { + fun generateDistributedNotaryCompositeIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { require(dirs.isNotEmpty()) - log.trace { "Generating identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } - val keyPairs = (1..dirs.size).map { generateKeyPair() } - val compositeKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) - + log.trace { "Generating composite identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getCertificate(X509Utilities.CORDA_ROOT_CA) + val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) + val keyPairs = (1..dirs.size).map { generateKeyPair() } + val notaryKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) keyPairs.zip(dirs) { keyPair, nodeDir -> - val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, compositeKey).map { publicKey -> - X509Utilities.createCertificate( - CertificateType.SERVICE_IDENTITY, - intermediateCa.certificate, - intermediateCa.keyPair, - notaryName.x500Principal, - publicKey) - } - val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" - val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") - keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) - keystore.setKeyEntry( - "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", - keyPair.private, - "cordacadevkeypass".toCharArray(), - arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) - keystore.save(distServKeyStoreFile, "cordacadevpass") + generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, nodeDir, rootCert) } - return Party(notaryName, compositeKey) + return Party(notaryName, notaryKey) + } + + fun generateDistributedNotarySingularIdentity(dirs: List, notaryName: CordaX500Name): Party { + require(dirs.isNotEmpty()) + + log.trace { "Generating singular identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } + val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") + val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") + val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) + + val keyPair = generateKeyPair() + val notaryKey = keyPair.public + dirs.forEach { dir -> + generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, dir, rootCert) + } + return Party(notaryName, notaryKey) + } + + private fun generateCertificates(keyPair: KeyPair, notaryKey: PublicKey, intermediateCa: CertificateAndKeyPair, notaryName: CordaX500Name, nodeDir: Path, rootCert: X509Certificate) { + val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, notaryKey).map { publicKey -> + X509Utilities.createCertificate( + CertificateType.SERVICE_IDENTITY, + intermediateCa.certificate, + intermediateCa.keyPair, + notaryName.x500Principal, + publicKey) + } + val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" + val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") + keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) + keystore.setKeyEntry( + "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", + keyPair.private, + "cordacadevkeypass".toCharArray(), + arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) + keystore.save(distServKeyStoreFile, "cordacadevpass") } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt index 0fa5ed8d36..bdaec3ec16 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt @@ -60,7 +60,7 @@ class BFTNotaryServiceTests { (Paths.get("config") / "currentView").deleteIfExists() // XXX: Make config object warn if this exists? val replicaIds = (0 until clusterSize) - notary = DevIdentityGenerator.generateDistributedNotaryIdentity( + notary = DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( replicaIds.map { mockNet.baseDirectory(mockNet.nextNodeId + it) }, CordaX500Name("BFT", "Zurich", "CH")) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt index 92723d52d8..9720df12f8 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt @@ -16,11 +16,11 @@ import net.corda.node.services.Permissions.Companion.startFlow import net.corda.nodeapi.internal.config.User import net.corda.testing.* import net.corda.testing.driver.NodeHandle +import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver -import net.corda.testing.node.ClusterSpec import net.corda.testing.node.NotarySpec +import net.corda.testing.node.internal.DummyClusterSpec import org.assertj.core.api.Assertions.assertThat -import org.junit.Ignore import org.junit.Test import rx.Observable import java.util.* @@ -32,18 +32,23 @@ class DistributedServiceTests { private lateinit var raftNotaryIdentity: Party private lateinit var notaryStateMachines: Observable> - private fun setup(testBlock: () -> Unit) { + private fun setup(compositeIdentity: Boolean = false, testBlock: () -> Unit) { val testUser = User("test", "test", permissions = setOf( startFlow(), startFlow(), invokeRpc(CordaRPCOps::nodeInfo), invokeRpc(CordaRPCOps::stateMachinesFeed)) ) - driver( extraCordappPackagesToScan = listOf("net.corda.finance.contracts"), - notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, rpcUsers = listOf(testUser), cluster = ClusterSpec.Raft(clusterSize = 3)))) - { + notarySpecs = listOf( + NotarySpec( + DUMMY_NOTARY_NAME, + rpcUsers = listOf(testUser), + cluster = DummyClusterSpec(clusterSize = 3, compositeServiceIdentity = compositeIdentity)) + ), + portAllocation = PortAllocation.RandomFree + ) { alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(testUser)).getOrThrow() raftNotaryIdentity = defaultNotaryIdentity notaryNodes = defaultNotaryHandle.nodeHandles.getOrThrow().map { it as NodeHandle.OutOfProcess } @@ -72,11 +77,60 @@ class DistributedServiceTests { } } + // TODO This should be in RaftNotaryServiceTests + @Test + fun `cluster survives if a notary is killed`() { + setup { + // Issue 100 pounds, then pay ourselves 10x5 pounds + issueCash(100.POUNDS) + + for (i in 1..10) { + paySelf(5.POUNDS) + } + + // Now kill a notary node + with(notaryNodes[0].process) { + destroy() + waitFor() + } + + // Pay ourselves another 20x5 pounds + for (i in 1..20) { + paySelf(5.POUNDS) + } + + val notarisationsPerNotary = HashMap() + notaryStateMachines.expectEvents(isStrict = false) { + replicate>(30) { + expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> + update as StateMachineUpdate.Added + notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } + } + } + } + + println("Notarisation distribution: $notarisationsPerNotary") + require(notarisationsPerNotary.size == 3) + } + } + // TODO Use a dummy distributed service rather than a Raft Notary Service as this test is only about Artemis' ability // to handle distributed services - @Ignore("Test has undeterministic capacity to hang, ignore till fixed") @Test - fun `requests are distributed evenly amongst the nodes`() = setup { + fun `requests are distributed evenly amongst the nodes`() { + setup { + checkRequestsDistributedEvenly() + } + } + + @Test + fun `requests are distributed evenly amongst the nodes with a composite public key`() { + setup(true) { + checkRequestsDistributedEvenly() + } + } + + private fun checkRequestsDistributedEvenly() { // Issue 100 pounds, then pay ourselves 50x2 pounds issueCash(100.POUNDS) @@ -102,42 +156,6 @@ class DistributedServiceTests { require(notarisationsPerNotary.values.all { it > 10 }) } - // TODO This should be in RaftNotaryServiceTests - @Ignore("Test has undeterministic capacity to hang, ignore till fixed") - @Test - fun `cluster survives if a notary is killed`() = setup { - // Issue 100 pounds, then pay ourselves 10x5 pounds - issueCash(100.POUNDS) - - for (i in 1..10) { - paySelf(5.POUNDS) - } - - // Now kill a notary node - with(notaryNodes[0].process) { - destroy() - waitFor() - } - - // Pay ourselves another 20x5 pounds - for (i in 1..20) { - paySelf(5.POUNDS) - } - - val notarisationsPerNotary = HashMap() - notaryStateMachines.expectEvents(isStrict = false) { - replicate>(30) { - expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> - update as StateMachineUpdate.Added - notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } - } - } - } - - println("Notarisation distribution: $notarisationsPerNotary") - require(notarisationsPerNotary.size == 3) - } - private fun issueCash(amount: Amount) { aliceProxy.startFlow(::CashIssueFlow, amount, OpaqueBytes.of(0), raftNotaryIdentity).returnValue.getOrThrow() } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 0edc5a1624..6d864f7538 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -173,11 +173,11 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } private inline fun signNodeInfo(nodeInfo: NodeInfo, sign: (PublicKey, SerializedBytes) -> DigitalSignature): SignedNodeInfo { - // For now we assume the node has only one identity (excluding any composite ones) - val owningKey = nodeInfo.legalIdentities.single { it.owningKey !is CompositeKey }.owningKey + // For now we exclude any composite identities, see [SignedNodeInfo] + val owningKeys = nodeInfo.legalIdentities.map { it.owningKey }.filter { it !is CompositeKey } val serialised = nodeInfo.serialize() - val signature = sign(owningKey, serialised) - return SignedNodeInfo(serialised, listOf(signature)) + val signatures = owningKeys.map { sign(it, serialised) } + return SignedNodeInfo(serialised, signatures) } open fun generateAndSaveNodeInfo(): NodeInfo { diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt index 1b2967c953..8e4028956f 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt @@ -62,7 +62,7 @@ class BFTNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotaryIdentity( + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName, minCorrectReplicas(clusterSize) diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt index 6a59809cf6..a86a6be0ea 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt @@ -1,6 +1,7 @@ package net.corda.notarydemo import net.corda.client.rpc.CordaRPCClient +import net.corda.core.crypto.CompositeKey import net.corda.core.crypto.toStringShort import net.corda.core.identity.PartyAndCertificate import net.corda.core.messaging.CordaRPCOps @@ -38,7 +39,8 @@ private class NotaryDemoClientApi(val rpc: CordaRPCOps) { /** Makes calls to the node rpc to start transaction notarisation. */ fun notarise(count: Int) { - println("Notary: \"${notary.name}\", with composite key: ${notary.owningKey.toStringShort()}") + val keyType = if (notary.owningKey is CompositeKey) "composite" else "public" + println("Notary: \"${notary.name}\", with $keyType key: ${notary.owningKey.toStringShort()}") val transactions = buildTransactions(count) println("Notarised ${transactions.size} transactions:") transactions.zip(notariseTransactions(transactions)).forEach { (tx, signersFuture) -> diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt index abceabbe77..f999ad68e6 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt @@ -58,7 +58,7 @@ class RaftNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotaryIdentity( + DevIdentityGenerator.generateDistributedNotarySingularIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName ) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt index b6533ebd98..128ed9070b 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt @@ -14,10 +14,12 @@ data class NotarySpec( ) @DoNotImplement -sealed class ClusterSpec { +abstract class ClusterSpec { abstract val clusterSize: Int - data class Raft(override val clusterSize: Int) : ClusterSpec() { + data class Raft( + override val clusterSize: Int + ) : ClusterSpec() { init { require(clusterSize > 0) } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index e7b65d2572..1e4f710449 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -267,7 +267,7 @@ class DriverDSLImpl( if (cordform.notary == null) continue val name = CordaX500Name.parse(cordform.name) val notaryConfig = ConfigFactory.parseMap(cordform.notary).parseAs() - // We need to first group the nodes that form part of a cluser. We assume for simplicity that nodes of the + // We need to first group the nodes that form part of a cluster. We assume for simplicity that nodes of the // same cluster type and validating flag are part of the same cluster. if (notaryConfig.raft != null) { val key = if (notaryConfig.validating) VALIDATING_RAFT else NON_VALIDATING_RAFT @@ -282,10 +282,17 @@ class DriverDSLImpl( } clusterNodes.asMap().forEach { type, nodeNames -> - val identity = DevIdentityGenerator.generateDistributedNotaryIdentity( - dirs = nodeNames.map { baseDirectory(it) }, - notaryName = type.clusterName - ) + val identity = if (type == ClusterType.NON_VALIDATING_RAFT || type == ClusterType.VALIDATING_RAFT) { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = nodeNames.map { baseDirectory(it) }, + notaryName = type.clusterName + ) + } else { + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + dirs = nodeNames.map { baseDirectory(it) }, + notaryName = type.clusterName + ) + } notaryInfos += NotaryInfo(identity, type.validating) } @@ -382,13 +389,30 @@ class DriverDSLImpl( private fun startNotaryIdentityGeneration(): CordaFuture> { return executorService.fork { notarySpecs.map { spec -> - val identity = if (spec.cluster == null) { - DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) - } else { - DevIdentityGenerator.generateDistributedNotaryIdentity( - dirs = generateNodeNames(spec).map { baseDirectory(it) }, - notaryName = spec.name - ) + val identity = when (spec.cluster) { + null -> { + DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) + } + is ClusterSpec.Raft -> { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } + is DummyClusterSpec -> { + if (spec.cluster.compositeServiceIdentity) { + DevIdentityGenerator.generateDistributedNotarySingularIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } else { + DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) + } + } + else -> throw UnsupportedOperationException("Cluster spec ${spec.cluster} not supported by Driver") } NotaryInfo(identity, spec.validating) } @@ -433,9 +457,12 @@ class DriverDSLImpl( private fun startNotaries(localNetworkMap: LocalNetworkMap?): List>> { return notarySpecs.map { - when { - it.cluster == null -> startSingleNotary(it, localNetworkMap) - it.cluster is ClusterSpec.Raft -> startRaftNotaryCluster(it, localNetworkMap) + when (it.cluster) { + null -> startSingleNotary(it, localNetworkMap) + is ClusterSpec.Raft, + // DummyCluster is used for testing the notary communication path, and it does not matter + // which underlying consensus algorithm is used, so we just stick to Raft + is DummyClusterSpec -> startRaftNotaryCluster(it, localNetworkMap) else -> throw IllegalArgumentException("BFT-SMaRt not supported") } } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt new file mode 100644 index 0000000000..13a5d3df2a --- /dev/null +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt @@ -0,0 +1,20 @@ +package net.corda.testing.node.internal + +import net.corda.testing.node.ClusterSpec + +/** + * Only used for testing the notary communication path. Can be configured to act as a Raft (singular identity), + * or a BFT (composite key identity) notary service. + */ +data class DummyClusterSpec( + override val clusterSize: Int, + /** + * If *true*, the cluster will use a shared composite public key for the service identity, with individual + * private keys. If *false*, the same "singular" key pair will be shared by all replicas. + */ + val compositeServiceIdentity: Boolean = false +) : ClusterSpec() { + init { + require(clusterSize > 0) + } +} \ No newline at end of file From 46d88ed74064cb8a65ab01b6437013d562cdced1 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Tue, 9 Jan 2018 11:56:48 +0000 Subject: [PATCH 05/17] Adds JavaDocs for waitForAllNodesToFinish param --- .../src/main/kotlin/net/corda/testing/driver/Driver.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt index 11e95e450b..0824108dae 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt @@ -159,6 +159,8 @@ data class JmxPolicy(val startJmxHttpServer: Boolean = false, * @param useTestClock If true the test clock will be used in Node. * @param startNodesInProcess Provides the default behaviour of whether new nodes should start inside this process or * not. Note that this may be overridden in [DriverDSL.startNode]. + * @param waitForAllNodesToFinish If true, the nodes will not shut down automatically after executing the code in the driver DSL block. + * It will wait for them to be shut down externally instead. * @param notarySpecs The notaries advertised for this network. These nodes will be started automatically and will be * available from [DriverDSL.notaryHandles]. Defaults to a simple validating notary. * @param jmxPolicy Used to specify whether to expose JMX metrics via Jolokia HHTP/JSON. Defines two attributes: From 63e1bdaa94d5d9fcf52595c62595a88db7d23d72 Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 9 Jan 2018 10:28:49 +0000 Subject: [PATCH 06/17] =?UTF-8?q?Revert=20"Raft=20notaries=20can=20share?= =?UTF-8?q?=20a=20single=20key=20pair=20for=20the=20service=20identity=20(?= =?UTF-8?q?i=E2=80=A6=20(#2269)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 3e00676851a86dd1adaebec5ad8b8784035b58d6. --- .../nodeapi/internal/DevIdentityGenerator.kt | 68 ++++------- .../node/services/BFTNotaryServiceTests.kt | 2 +- .../node/services/DistributedServiceTests.kt | 106 ++++++++---------- .../net/corda/node/internal/AbstractNode.kt | 8 +- .../net/corda/notarydemo/BFTNotaryCordform.kt | 2 +- .../kotlin/net/corda/notarydemo/Notarise.kt | 4 +- .../corda/notarydemo/RaftNotaryCordform.kt | 2 +- .../net/corda/testing/node/NotarySpec.kt | 6 +- .../testing/node/internal/DriverDSLImpl.kt | 57 +++------- .../testing/node/internal/DummyClusterSpec.kt | 20 ---- 10 files changed, 92 insertions(+), 183 deletions(-) delete mode 100644 testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt index 6b17aa4049..0cb8472f89 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt @@ -12,9 +12,6 @@ import net.corda.nodeapi.internal.config.NodeSSLConfiguration import net.corda.nodeapi.internal.crypto.* import org.slf4j.LoggerFactory import java.nio.file.Path -import java.security.KeyPair -import java.security.PublicKey -import java.security.cert.X509Certificate /** * Contains utility methods for generating identities for a node. @@ -50,56 +47,37 @@ object DevIdentityGenerator { return identity.party } - fun generateDistributedNotaryCompositeIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { + fun generateDistributedNotaryIdentity(dirs: List, notaryName: CordaX500Name, threshold: Int = 1): Party { require(dirs.isNotEmpty()) - log.trace { "Generating composite identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } - val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) - + log.trace { "Generating identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } val keyPairs = (1..dirs.size).map { generateKeyPair() } - val notaryKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) - keyPairs.zip(dirs) { keyPair, nodeDir -> - generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, nodeDir, rootCert) - } + val compositeKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) - return Party(notaryName, notaryKey) - } - - fun generateDistributedNotarySingularIdentity(dirs: List, notaryName: CordaX500Name): Party { - require(dirs.isNotEmpty()) - - log.trace { "Generating singular identity \"$notaryName\" for nodes: ${dirs.joinToString()}" } val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) + val rootCert = caKeyStore.getCertificate(X509Utilities.CORDA_ROOT_CA) - val keyPair = generateKeyPair() - val notaryKey = keyPair.public - dirs.forEach { dir -> - generateCertificates(keyPair, notaryKey, intermediateCa, notaryName, dir, rootCert) + keyPairs.zip(dirs) { keyPair, nodeDir -> + val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, compositeKey).map { publicKey -> + X509Utilities.createCertificate( + CertificateType.SERVICE_IDENTITY, + intermediateCa.certificate, + intermediateCa.keyPair, + notaryName.x500Principal, + publicKey) + } + val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" + val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") + keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) + keystore.setKeyEntry( + "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", + keyPair.private, + "cordacadevkeypass".toCharArray(), + arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) + keystore.save(distServKeyStoreFile, "cordacadevpass") } - return Party(notaryName, notaryKey) - } - private fun generateCertificates(keyPair: KeyPair, notaryKey: PublicKey, intermediateCa: CertificateAndKeyPair, notaryName: CordaX500Name, nodeDir: Path, rootCert: X509Certificate) { - val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, notaryKey).map { publicKey -> - X509Utilities.createCertificate( - CertificateType.SERVICE_IDENTITY, - intermediateCa.certificate, - intermediateCa.keyPair, - notaryName.x500Principal, - publicKey) - } - val distServKeyStoreFile = (nodeDir / "certificates").createDirectories() / "distributedService.jks" - val keystore = loadOrCreateKeyStore(distServKeyStoreFile, "cordacadevpass") - keystore.setCertificateEntry("$DISTRIBUTED_NOTARY_ALIAS_PREFIX-composite-key", compositeKeyCert) - keystore.setKeyEntry( - "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", - keyPair.private, - "cordacadevkeypass".toCharArray(), - arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) - keystore.save(distServKeyStoreFile, "cordacadevpass") + return Party(notaryName, compositeKey) } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt index bdaec3ec16..0fa5ed8d36 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/BFTNotaryServiceTests.kt @@ -60,7 +60,7 @@ class BFTNotaryServiceTests { (Paths.get("config") / "currentView").deleteIfExists() // XXX: Make config object warn if this exists? val replicaIds = (0 until clusterSize) - notary = DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + notary = DevIdentityGenerator.generateDistributedNotaryIdentity( replicaIds.map { mockNet.baseDirectory(mockNet.nextNodeId + it) }, CordaX500Name("BFT", "Zurich", "CH")) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt index 9720df12f8..92723d52d8 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt @@ -16,11 +16,11 @@ import net.corda.node.services.Permissions.Companion.startFlow import net.corda.nodeapi.internal.config.User import net.corda.testing.* import net.corda.testing.driver.NodeHandle -import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver +import net.corda.testing.node.ClusterSpec import net.corda.testing.node.NotarySpec -import net.corda.testing.node.internal.DummyClusterSpec import org.assertj.core.api.Assertions.assertThat +import org.junit.Ignore import org.junit.Test import rx.Observable import java.util.* @@ -32,23 +32,18 @@ class DistributedServiceTests { private lateinit var raftNotaryIdentity: Party private lateinit var notaryStateMachines: Observable> - private fun setup(compositeIdentity: Boolean = false, testBlock: () -> Unit) { + private fun setup(testBlock: () -> Unit) { val testUser = User("test", "test", permissions = setOf( startFlow(), startFlow(), invokeRpc(CordaRPCOps::nodeInfo), invokeRpc(CordaRPCOps::stateMachinesFeed)) ) + driver( extraCordappPackagesToScan = listOf("net.corda.finance.contracts"), - notarySpecs = listOf( - NotarySpec( - DUMMY_NOTARY_NAME, - rpcUsers = listOf(testUser), - cluster = DummyClusterSpec(clusterSize = 3, compositeServiceIdentity = compositeIdentity)) - ), - portAllocation = PortAllocation.RandomFree - ) { + notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, rpcUsers = listOf(testUser), cluster = ClusterSpec.Raft(clusterSize = 3)))) + { alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(testUser)).getOrThrow() raftNotaryIdentity = defaultNotaryIdentity notaryNodes = defaultNotaryHandle.nodeHandles.getOrThrow().map { it as NodeHandle.OutOfProcess } @@ -77,60 +72,11 @@ class DistributedServiceTests { } } - // TODO This should be in RaftNotaryServiceTests - @Test - fun `cluster survives if a notary is killed`() { - setup { - // Issue 100 pounds, then pay ourselves 10x5 pounds - issueCash(100.POUNDS) - - for (i in 1..10) { - paySelf(5.POUNDS) - } - - // Now kill a notary node - with(notaryNodes[0].process) { - destroy() - waitFor() - } - - // Pay ourselves another 20x5 pounds - for (i in 1..20) { - paySelf(5.POUNDS) - } - - val notarisationsPerNotary = HashMap() - notaryStateMachines.expectEvents(isStrict = false) { - replicate>(30) { - expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> - update as StateMachineUpdate.Added - notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } - } - } - } - - println("Notarisation distribution: $notarisationsPerNotary") - require(notarisationsPerNotary.size == 3) - } - } - // TODO Use a dummy distributed service rather than a Raft Notary Service as this test is only about Artemis' ability // to handle distributed services + @Ignore("Test has undeterministic capacity to hang, ignore till fixed") @Test - fun `requests are distributed evenly amongst the nodes`() { - setup { - checkRequestsDistributedEvenly() - } - } - - @Test - fun `requests are distributed evenly amongst the nodes with a composite public key`() { - setup(true) { - checkRequestsDistributedEvenly() - } - } - - private fun checkRequestsDistributedEvenly() { + fun `requests are distributed evenly amongst the nodes`() = setup { // Issue 100 pounds, then pay ourselves 50x2 pounds issueCash(100.POUNDS) @@ -156,6 +102,42 @@ class DistributedServiceTests { require(notarisationsPerNotary.values.all { it > 10 }) } + // TODO This should be in RaftNotaryServiceTests + @Ignore("Test has undeterministic capacity to hang, ignore till fixed") + @Test + fun `cluster survives if a notary is killed`() = setup { + // Issue 100 pounds, then pay ourselves 10x5 pounds + issueCash(100.POUNDS) + + for (i in 1..10) { + paySelf(5.POUNDS) + } + + // Now kill a notary node + with(notaryNodes[0].process) { + destroy() + waitFor() + } + + // Pay ourselves another 20x5 pounds + for (i in 1..20) { + paySelf(5.POUNDS) + } + + val notarisationsPerNotary = HashMap() + notaryStateMachines.expectEvents(isStrict = false) { + replicate>(30) { + expect(match = { it.second is StateMachineUpdate.Added }) { (notary, update) -> + update as StateMachineUpdate.Added + notarisationsPerNotary.compute(notary) { _, number -> number?.plus(1) ?: 1 } + } + } + } + + println("Notarisation distribution: $notarisationsPerNotary") + require(notarisationsPerNotary.size == 3) + } + private fun issueCash(amount: Amount) { aliceProxy.startFlow(::CashIssueFlow, amount, OpaqueBytes.of(0), raftNotaryIdentity).returnValue.getOrThrow() } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 6d864f7538..0edc5a1624 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -173,11 +173,11 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } private inline fun signNodeInfo(nodeInfo: NodeInfo, sign: (PublicKey, SerializedBytes) -> DigitalSignature): SignedNodeInfo { - // For now we exclude any composite identities, see [SignedNodeInfo] - val owningKeys = nodeInfo.legalIdentities.map { it.owningKey }.filter { it !is CompositeKey } + // For now we assume the node has only one identity (excluding any composite ones) + val owningKey = nodeInfo.legalIdentities.single { it.owningKey !is CompositeKey }.owningKey val serialised = nodeInfo.serialize() - val signatures = owningKeys.map { sign(it, serialised) } - return SignedNodeInfo(serialised, signatures) + val signature = sign(owningKey, serialised) + return SignedNodeInfo(serialised, listOf(signature)) } open fun generateAndSaveNodeInfo(): NodeInfo { diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt index 8e4028956f..1b2967c953 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/BFTNotaryCordform.kt @@ -62,7 +62,7 @@ class BFTNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + DevIdentityGenerator.generateDistributedNotaryIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName, minCorrectReplicas(clusterSize) diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt index a86a6be0ea..6a59809cf6 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/Notarise.kt @@ -1,7 +1,6 @@ package net.corda.notarydemo import net.corda.client.rpc.CordaRPCClient -import net.corda.core.crypto.CompositeKey import net.corda.core.crypto.toStringShort import net.corda.core.identity.PartyAndCertificate import net.corda.core.messaging.CordaRPCOps @@ -39,8 +38,7 @@ private class NotaryDemoClientApi(val rpc: CordaRPCOps) { /** Makes calls to the node rpc to start transaction notarisation. */ fun notarise(count: Int) { - val keyType = if (notary.owningKey is CompositeKey) "composite" else "public" - println("Notary: \"${notary.name}\", with $keyType key: ${notary.owningKey.toStringShort()}") + println("Notary: \"${notary.name}\", with composite key: ${notary.owningKey.toStringShort()}") val transactions = buildTransactions(count) println("Notarised ${transactions.size} transactions:") transactions.zip(notariseTransactions(transactions)).forEach { (tx, signersFuture) -> diff --git a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt index f999ad68e6..abceabbe77 100644 --- a/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt +++ b/samples/notary-demo/src/main/kotlin/net/corda/notarydemo/RaftNotaryCordform.kt @@ -58,7 +58,7 @@ class RaftNotaryCordform : CordformDefinition() { } override fun setup(context: CordformContext) { - DevIdentityGenerator.generateDistributedNotarySingularIdentity( + DevIdentityGenerator.generateDistributedNotaryIdentity( notaryNames.map { context.baseDirectory(it.toString()) }, clusterName ) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt index 128ed9070b..b6533ebd98 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt @@ -14,12 +14,10 @@ data class NotarySpec( ) @DoNotImplement -abstract class ClusterSpec { +sealed class ClusterSpec { abstract val clusterSize: Int - data class Raft( - override val clusterSize: Int - ) : ClusterSpec() { + data class Raft(override val clusterSize: Int) : ClusterSpec() { init { require(clusterSize > 0) } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index 1e4f710449..e7b65d2572 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -267,7 +267,7 @@ class DriverDSLImpl( if (cordform.notary == null) continue val name = CordaX500Name.parse(cordform.name) val notaryConfig = ConfigFactory.parseMap(cordform.notary).parseAs() - // We need to first group the nodes that form part of a cluster. We assume for simplicity that nodes of the + // We need to first group the nodes that form part of a cluser. We assume for simplicity that nodes of the // same cluster type and validating flag are part of the same cluster. if (notaryConfig.raft != null) { val key = if (notaryConfig.validating) VALIDATING_RAFT else NON_VALIDATING_RAFT @@ -282,17 +282,10 @@ class DriverDSLImpl( } clusterNodes.asMap().forEach { type, nodeNames -> - val identity = if (type == ClusterType.NON_VALIDATING_RAFT || type == ClusterType.VALIDATING_RAFT) { - DevIdentityGenerator.generateDistributedNotarySingularIdentity( - dirs = nodeNames.map { baseDirectory(it) }, - notaryName = type.clusterName - ) - } else { - DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( - dirs = nodeNames.map { baseDirectory(it) }, - notaryName = type.clusterName - ) - } + val identity = DevIdentityGenerator.generateDistributedNotaryIdentity( + dirs = nodeNames.map { baseDirectory(it) }, + notaryName = type.clusterName + ) notaryInfos += NotaryInfo(identity, type.validating) } @@ -389,30 +382,13 @@ class DriverDSLImpl( private fun startNotaryIdentityGeneration(): CordaFuture> { return executorService.fork { notarySpecs.map { spec -> - val identity = when (spec.cluster) { - null -> { - DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) - } - is ClusterSpec.Raft -> { - DevIdentityGenerator.generateDistributedNotarySingularIdentity( - dirs = generateNodeNames(spec).map { baseDirectory(it) }, - notaryName = spec.name - ) - } - is DummyClusterSpec -> { - if (spec.cluster.compositeServiceIdentity) { - DevIdentityGenerator.generateDistributedNotarySingularIdentity( - dirs = generateNodeNames(spec).map { baseDirectory(it) }, - notaryName = spec.name - ) - } else { - DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( - dirs = generateNodeNames(spec).map { baseDirectory(it) }, - notaryName = spec.name - ) - } - } - else -> throw UnsupportedOperationException("Cluster spec ${spec.cluster} not supported by Driver") + val identity = if (spec.cluster == null) { + DevIdentityGenerator.installKeyStoreWithNodeIdentity(baseDirectory(spec.name), spec.name) + } else { + DevIdentityGenerator.generateDistributedNotaryIdentity( + dirs = generateNodeNames(spec).map { baseDirectory(it) }, + notaryName = spec.name + ) } NotaryInfo(identity, spec.validating) } @@ -457,12 +433,9 @@ class DriverDSLImpl( private fun startNotaries(localNetworkMap: LocalNetworkMap?): List>> { return notarySpecs.map { - when (it.cluster) { - null -> startSingleNotary(it, localNetworkMap) - is ClusterSpec.Raft, - // DummyCluster is used for testing the notary communication path, and it does not matter - // which underlying consensus algorithm is used, so we just stick to Raft - is DummyClusterSpec -> startRaftNotaryCluster(it, localNetworkMap) + when { + it.cluster == null -> startSingleNotary(it, localNetworkMap) + it.cluster is ClusterSpec.Raft -> startRaftNotaryCluster(it, localNetworkMap) else -> throw IllegalArgumentException("BFT-SMaRt not supported") } } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt deleted file mode 100644 index 13a5d3df2a..0000000000 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DummyClusterSpec.kt +++ /dev/null @@ -1,20 +0,0 @@ -package net.corda.testing.node.internal - -import net.corda.testing.node.ClusterSpec - -/** - * Only used for testing the notary communication path. Can be configured to act as a Raft (singular identity), - * or a BFT (composite key identity) notary service. - */ -data class DummyClusterSpec( - override val clusterSize: Int, - /** - * If *true*, the cluster will use a shared composite public key for the service identity, with individual - * private keys. If *false*, the same "singular" key pair will be shared by all replicas. - */ - val compositeServiceIdentity: Boolean = false -) : ClusterSpec() { - init { - require(clusterSize > 0) - } -} \ No newline at end of file From a6cf04b49c5dcdfb8ac4892e7ffd92b83901b21f Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Tue, 9 Jan 2018 15:08:29 +0000 Subject: [PATCH 07/17] Ignore flaky Raft Notary integration test while it's being investigated --- .../kotlin/net/corda/node/services/RaftNotaryServiceTests.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/RaftNotaryServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/RaftNotaryServiceTests.kt index 4f70f23f5d..67b6670629 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/RaftNotaryServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/RaftNotaryServiceTests.kt @@ -20,6 +20,7 @@ import net.corda.testing.dummyCommand import net.corda.testing.node.ClusterSpec import net.corda.testing.node.NotarySpec import net.corda.testing.node.startFlow +import org.junit.Ignore import org.junit.Test import java.util.* import kotlin.test.assertEquals @@ -28,6 +29,7 @@ import kotlin.test.assertFailsWith class RaftNotaryServiceTests { private val notaryName = CordaX500Name("RAFT Notary Service", "London", "GB") + @Ignore("Test has undeterministic capacity to hang, ignore till fixed") @Test fun `detect double spend`() { driver( From 979d7f2c6387ea6a79190acb51e3ed2bd1b988f5 Mon Sep 17 00:00:00 2001 From: Anthony Keenan <34482776+anthonykr3@users.noreply.github.com> Date: Tue, 9 Jan 2018 16:55:16 +0000 Subject: [PATCH 08/17] ENT-1226 Improve Network client Error Handling (#2344) * Improve Network client Error Handling * Reformatted NetworkMapServer * Removed line that is now redundant --- .../node/services/network/NetworkMapClient.kt | 20 ++++++----- .../services/network/NetworkMapClientTest.kt | 16 +++++++++ .../node/internal/network/NetworkMapServer.kt | 35 ++++++++++++------- 3 files changed, 50 insertions(+), 21 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt index db7bea9480..5064361c7b 100644 --- a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt @@ -18,9 +18,11 @@ import net.corda.nodeapi.internal.network.SignedNetworkMap import net.corda.nodeapi.internal.SignedNodeInfo import okhttp3.CacheControl import okhttp3.Headers +import org.apache.commons.io.IOUtils import rx.Subscription import java.io.BufferedReader import java.io.Closeable +import java.io.IOException import java.net.HttpURLConnection import java.net.URL import java.security.cert.X509Certificate @@ -33,15 +35,15 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val trustedRoot: X509C fun publish(signedNodeInfo: SignedNodeInfo) { val publishURL = URL("$networkMapUrl/publish") - val conn = publishURL.openHttpConnection() - conn.doOutput = true - conn.requestMethod = "POST" - conn.setRequestProperty("Content-Type", "application/octet-stream") - conn.outputStream.use { signedNodeInfo.serialize().open().copyTo(it) } - - // This will throw IOException if the response code is not HTTP 200. - // This gives a much better exception then reading the error stream. - conn.inputStream.close() + publishURL.openHttpConnection().apply { + doOutput = true + requestMethod = "POST" + setRequestProperty("Content-Type", "application/octet-stream") + outputStream.use { signedNodeInfo.serialize().open().copyTo(it) } + if (responseCode != 200) { + throw IOException("Response Code $responseCode: ${IOUtils.toString(errorStream)}") + } + } } fun getNetworkMap(): NetworkMapResponse { diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt index 9f342b2e0b..fa0a67fe15 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt @@ -9,13 +9,17 @@ import net.corda.testing.BOB_NAME import net.corda.testing.DEV_ROOT_CA import net.corda.testing.SerializationEnvironmentRule import net.corda.testing.driver.PortAllocation +import net.corda.testing.internal.TestNodeInfoBuilder import net.corda.testing.internal.createNodeInfoAndSigned +import net.corda.testing.internal.signWith import net.corda.testing.node.internal.network.NetworkMapServer import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import java.io.IOException import java.net.URL import kotlin.test.assertEquals import kotlin.test.assertNotNull @@ -63,6 +67,18 @@ class NetworkMapClientTest { assertEquals(nodeInfo2, networkMapClient.getNodeInfo(nodeInfoHash2)) } + @Test + fun `errors return a meaningful error message`() { + val nodeInfoBuilder = TestNodeInfoBuilder() + val (_, aliceKey) = nodeInfoBuilder.addIdentity(ALICE_NAME) + nodeInfoBuilder.addIdentity(BOB_NAME) + val nodeInfo3 = nodeInfoBuilder.build() + val signedNodeInfo3 = nodeInfo3.signWith(listOf(aliceKey)) + + assertThatThrownBy { networkMapClient.publish(signedNodeInfo3) } + .isInstanceOf(IOException::class.java) + .hasMessage("Response Code 403: Missing signatures. Found 1 expected 2") + } @Test fun `download NetworkParameter correctly`() { diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt index be1a11c293..810fcedf77 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt @@ -25,6 +25,7 @@ import org.glassfish.jersey.servlet.ServletContainer import java.io.Closeable import java.io.InputStream import java.net.InetSocketAddress +import java.security.SignatureException import java.time.Duration import java.time.Instant import javax.security.auth.x500.X500Principal @@ -32,6 +33,7 @@ import javax.ws.rs.* import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response import javax.ws.rs.core.Response.ok +import javax.ws.rs.core.Response.status class NetworkMapServer(cacheTimeout: Duration, hostAndPort: NetworkHostAndPort, @@ -58,10 +60,10 @@ class NetworkMapServer(cacheTimeout: Duration, private val server: Server var networkParameters: NetworkParameters = stubNetworkParameters - set(networkParameters) { - check(field == stubNetworkParameters) { "Network parameters can be set only once" } - field = networkParameters - } + set(networkParameters) { + check(field == stubNetworkParameters) { "Network parameters can be set only once" } + field = networkParameters + } private val serializedParameters get() = networkParameters.serialize() private val service = InMemoryNetworkMapService(cacheTimeout, networkMapKeyAndCert(rootCa)) @@ -108,19 +110,28 @@ class NetworkMapServer(cacheTimeout: Duration, private val networkMapKeyAndCert: CertificateAndKeyPair) { private val nodeInfoMap = mutableMapOf() private val parametersHash by lazy { serializedParameters.hash } - private val signedParameters by lazy { SignedData( - serializedParameters, - DigitalSignature.WithKey(networkMapKeyAndCert.keyPair.public, Crypto.doSign(networkMapKeyAndCert.keyPair.private, serializedParameters.bytes))) } + private val signedParameters by lazy { + SignedData( + serializedParameters, + DigitalSignature.WithKey(networkMapKeyAndCert.keyPair.public, Crypto.doSign(networkMapKeyAndCert.keyPair.private, serializedParameters.bytes))) + } @POST @Path("publish") @Consumes(MediaType.APPLICATION_OCTET_STREAM) fun publishNodeInfo(input: InputStream): Response { - val registrationData = input.readBytes().deserialize() - val nodeInfo = registrationData.verified() - val nodeInfoHash = nodeInfo.serialize().sha256() - nodeInfoMap.put(nodeInfoHash, registrationData) - return ok().build() + return try { + val registrationData = input.readBytes().deserialize() + val nodeInfo = registrationData.verified() + val nodeInfoHash = nodeInfo.serialize().sha256() + nodeInfoMap.put(nodeInfoHash, registrationData) + ok() + } catch (e: Exception) { + when (e) { + is SignatureException -> status(Response.Status.FORBIDDEN).entity(e.message) + else -> status(Response.Status.INTERNAL_SERVER_ERROR).entity(e.message) + } + }.build() } @GET From cacdba872eabdbb14fa5b81192f5977c8c72b879 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Wed, 10 Jan 2018 11:41:49 +0000 Subject: [PATCH 09/17] CORDA-908 - Support private properties in AMQP serialization (#2336) CORDA-908 - Support private properties in AMQP serialization * Review comments * Fix tests * Review Comments * review comments * review comments --- docs/source/release-notes.rst | 28 +++-- docs/source/serialization.rst | 91 +++++++++++--- .../serialization/amqp/EvolutionSerializer.kt | 2 +- .../serialization/amqp/ObjectSerializer.kt | 2 + .../serialization/amqp/PropertySerializer.kt | 100 +++++++++++---- .../serialization/amqp/SerializationHelper.kt | 69 ++++++----- .../serialization/amqp/SerializerFactory.kt | 2 + .../amqp/custom/ThrowableSerializer.kt | 2 +- .../serialization/amqp/ErrorMessageTests.java | 2 + .../amqp/JavaPrivatePropertyTests.java | 79 ++++++++++++ .../serialization/amqp/AMQPTestUtils.kt | 2 - .../serialization/amqp/ErrorMessagesTests.kt | 9 ++ .../amqp/PrivatePropertyTests.kt | 117 ++++++++++++++++++ .../carpenter/ClassCarpenterTestUtils.kt | 9 +- 14 files changed, 430 insertions(+), 84 deletions(-) create mode 100644 node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java create mode 100644 node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/PrivatePropertyTests.kt diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index 64d9ad8881..7b3e96fe4c 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -16,19 +16,31 @@ Unreleased That is the ability to alter an enum constant and, as long as certain rules are followed and the correct annotations applied, have older and newer instances of that enumeration be understood. -* **AMQP Enabled** +* **AMQP Enabled**: + AMQP Serialization is now enabled for both peer to peer communication and the writing of states to the vault. This change + brings a stable format Corda can support internally throughout it's lifetime that meets the needs of Corda and our + users. -AMQP Serialization is now enabled for both peer to peer communication and writing states to the vault. This change -brings a stable format Corda can support internally throughout it's lifetime that meets the needs of Corda and our -users. + Details on the AMQP serialization framework can be found in the :doc:`serialization` document :ref:`here `. + This provides an introduction and overview of the framework whilst more specific details on object evolution as it relates to + serialization is similarly found in pages :doc:`serialization-default-evolution` and :doc:`serialization-enum-evolution` + respectively. Recommendations on how best to code CorDapps using your own :ref:`custom types `. + + .. note:: This release delivers the bulk of our transition from Kryo serialisation to AMQP serialisation. This means that many of the restrictions + that were documented in previous versions of Corda are now enforced. (https://docs.corda.net/releases/release-V1.0/serialization.html). + + In particular, you are advised to review the section titled "Custom Types". To aid with the transition, we have included support + in this release for default construction of objects and their instantiation through getters as well as objects with inaccessible + private fields but it is not guaranteed that this support will continue into future versions; the restrictions documented at the + link above are the canonical source. * **Custom Serializers** -To allow interop with third party libraries that cannot be recompiled we add functionality that allows custom serializers -to be written for those classes. If needed, a proxy object can be created as an interim step that allows Corda's internal -serializers to operate on those types. + To allow interop with third party libraries that cannot be recompiled we add functionality that allows custom serializers + to be written for those classes. If needed, a proxy object can be created as an interim step that allows Corda's internal + serializers to operate on those types. -A good example of this is the SIMM valuation demo which has a number of such serializers defined in the plugin/customserializers package + A good example of this is the SIMM valuation demo which has a number of such serializers defined in the plugin/customserializers package Release 2.0 ---------- diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index ed5a278feb..f217d749b9 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -1,6 +1,8 @@ Object serialization ==================== +.. contents:: + What is serialization (and deserialization)? -------------------------------------------- @@ -52,7 +54,7 @@ was a compelling use case for the definition and development of a custom format #. A desire to have a schema describing what has been serialized along-side the actual data: - #. To assist with versioning, both in terms of being able to interpret long ago archivEd data (e.g. trades from + #. To assist with versioning, both in terms of being able to interpret long ago archived data (e.g. trades from a decade ago, long after the code has changed) and between differing code versions. #. To make it easier to write user interfaces that can navigate the serialized form of data. #. To support cross platform (non-JVM) interaction, where the format of a class file is not so easily interpreted. @@ -76,7 +78,7 @@ Finally, for the checkpointing of flows Corda will continue to use the existing This separation of serialization schemes into different contexts allows us to use the most suitable framework for that context rather than attempting to force a one size fits all approach. Where ``Kryo`` is more suited to the serialization of a programs stack frames, being more flexible -than our AMQP framework in what it can construct and serialize, that flexibility makes it exceptionally difficult to make secure. Conversly +than our AMQP framework in what it can construct and serialize, that flexibility makes it exceptionally difficult to make secure. Conversely our AMQP framework allows us to concentrate on a robust a secure framework that can be reasoned about thus made safer with far fewer unforeseen security holes. @@ -282,22 +284,6 @@ serialised form val e2 = e.serialize().deserialize() // e2.c will be 20, not 100!!! -.. warning:: Private properties in Kotlin classes render the class unserializable *unless* a public - getter is manually defined. For example: - - .. container:: codeset - - .. sourcecode:: kotlin - - data class C(val a: Int, private val b: Int) { - // Without this C cannot be serialized - public fun getB() = b - } - - .. note:: This is particularly relevant as IDE's can often point out where they believe a - property can be made private without knowing this can break Corda serialization. Should - this happen then a run time warning will be generated when the class fails to serialize - Setter Instantiation '''''''''''''''''''' @@ -330,6 +316,75 @@ For example: public void setC(int c) { this.c = c; } } +Inaccessible Private Properties +``````````````````````````````` + +Whilst the Corda AMQP serialization framework supports private object properties without publicly +accessible getter methods this development idiom is strongly discouraged. + +For example. + + .. container:: codeset + + Kotlin: + + .. sourcecode:: kotlin + + data class C(val a: Int, private val b: Int) + + Java: + + .. sourcecode:: Java + + class C { + public Integer a; + private Integer b; + + C(Integer a, Integer b) { + this.a = a; + this.b = b; + } + } + +When designing stateful objects is should be remembered that they are not, despite appearances, traditional +programmatic constructs. They are signed over, transformed, serialised, and relationally mapped. As such, +all elements should be publicly accessible by design + +.. warning:: IDEs will indiciate erroneously that properties can be given something other than public + visibility. Ignore this as whilst it will work, as discussed above there are many reasons why this isn't + a good idea and those are beyond the scope of the IDEs inference rules + +Providing a public getter, as per the following example, is acceptable + + .. container:: codeset + + Kotlin: + + .. sourcecode:: kotlin + + data class C(val a: Int, private val b: Int) { + public fun getB() = b + } + + Java: + + .. sourcecode:: Java + + class C { + public Integer a; + private Integer b; + + C(Integer a, Integer b) { + this.a = a; + this.b = b; + } + + public Integer getB() { + return b; + } + } + + Enums ````` diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt index 663a153cbd..9293729306 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt @@ -96,7 +96,7 @@ class EvolutionSerializer( old.fields.forEach { val returnType = it.getTypeAsClass(factory.classloader) oldArgs[it.name] = OldParam( - returnType, idx++, PropertySerializer.make(it.name, null, returnType, factory)) + returnType, idx++, PropertySerializer.make(it.name, PublicPropertyReader(null), returnType, factory)) } val readers = constructor.parameters.map { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt index 6cba3bbea5..a9d099b2e3 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt @@ -26,6 +26,8 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS propertiesForSerialization(kotlinConstructor, clazz, factory) } + fun getPropertySerializers() = propertySerializers + private val typeName = nameForType(clazz) override val typeDescriptor = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${fingerprintForType(type, factory)}") diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt index d296a8fd91..af7c089552 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt @@ -1,17 +1,74 @@ package net.corda.nodeapi.internal.serialization.amqp -import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.loggerFor import org.apache.qpid.proton.amqp.Binary import org.apache.qpid.proton.codec.Data import java.lang.reflect.Method import java.lang.reflect.Type +import java.lang.reflect.Field import kotlin.reflect.full.memberProperties import kotlin.reflect.jvm.javaGetter +import kotlin.reflect.jvm.kotlinProperty + +abstract class PropertyReader { + abstract fun read(obj: Any?): Any? + abstract fun isNullable(): Boolean +} + +class PublicPropertyReader(private val readMethod: Method?) : PropertyReader() { + init { + readMethod?.isAccessible = true + } + + private fun Method.returnsNullable(): Boolean { + try { + val returnTypeString = this.declaringClass.kotlin.memberProperties.firstOrNull { it.javaGetter == this }?.returnType?.toString() ?: "?" + return returnTypeString.endsWith('?') || returnTypeString.endsWith('!') + } catch (e: kotlin.reflect.jvm.internal.KotlinReflectionInternalError) { + // This might happen for some types, e.g. kotlin.Throwable? - the root cause of the issue is: https://youtrack.jetbrains.com/issue/KT-13077 + // TODO: Revisit this when Kotlin issue is fixed. + + loggerFor().error("Unexpected internal Kotlin error", e) + return true + } + } + + override fun read(obj: Any?): Any? { + return readMethod!!.invoke(obj) + } + + override fun isNullable(): Boolean = readMethod?.returnsNullable() ?: false +} + +class PrivatePropertyReader(val field: Field, parentType: Type) : PropertyReader() { + init { + loggerFor().warn("Create property Serializer for private property '${field.name}' not " + + "exposed by a getter on class '$parentType'\n" + + "\tNOTE: This behaviour will be deprecated at some point in the future and a getter required") + } + + override fun read(obj: Any?): Any? { + field.isAccessible = true + val rtn = field.get(obj) + field.isAccessible = false + return rtn + } + + override fun isNullable() = try { + field.kotlinProperty?.returnType?.isMarkedNullable ?: false + } catch (e: kotlin.reflect.jvm.internal.KotlinReflectionInternalError) { + // This might happen for some types, e.g. kotlin.Throwable? - the root cause of the issue is: https://youtrack.jetbrains.com/issue/KT-13077 + // TODO: Revisit this when Kotlin issue is fixed. + loggerFor().error("Unexpected internal Kotlin error", e) + true + } +} + /** * Base class for serialization of a property of an object. */ -sealed class PropertySerializer(val name: String, val readMethod: Method?, val resolvedType: Type) { +sealed class PropertySerializer(val name: String, val propertyReader: PropertyReader, val resolvedType: Type) { abstract fun writeClassInfo(output: SerializationOutput) abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? @@ -44,25 +101,11 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r } private fun generateMandatory(): Boolean { - return isJVMPrimitive || readMethod?.returnsNullable() == false - } - - private fun Method.returnsNullable(): Boolean { - try { - val returnTypeString = this.declaringClass.kotlin.memberProperties.firstOrNull { it.javaGetter == this }?.returnType?.toString() ?: "?" - return returnTypeString.endsWith('?') || returnTypeString.endsWith('!') - } catch (e: kotlin.reflect.jvm.internal.KotlinReflectionInternalError) { - // This might happen for some types, e.g. kotlin.Throwable? - the root cause of the issue is: https://youtrack.jetbrains.com/issue/KT-13077 - // TODO: Revisit this when Kotlin issue is fixed. - logger.error("Unexpected internal Kotlin error", e) - return true - } + return isJVMPrimitive || !(propertyReader.isNullable()) } companion object { - private val logger = contextLogger() - fun make(name: String, readMethod: Method?, resolvedType: Type, factory: SerializerFactory): PropertySerializer { - readMethod?.isAccessible = true + fun make(name: String, readMethod: PropertyReader, resolvedType: Type, factory: SerializerFactory): PropertySerializer { if (SerializerFactory.isPrimitive(resolvedType)) { return when (resolvedType) { Char::class.java, Character::class.java -> AMQPCharPropertySerializer(name, readMethod) @@ -78,7 +121,8 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r * A property serializer for a complex type (another object). */ class DescribedTypePropertySerializer( - name: String, readMethod: Method?, + name: String, + readMethod: PropertyReader, resolvedType: Type, private val lazyTypeSerializer: () -> AMQPSerializer<*>) : PropertySerializer(name, readMethod, resolvedType) { // This is lazy so we don't get an infinite loop when a method returns an instance of the class. @@ -90,12 +134,15 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r } } - override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? = ifThrowsAppend({ nameForDebug }) { + override fun readProperty( + obj: Any?, + schemas: SerializationSchemas, + input: DeserializationInput): Any? = ifThrowsAppend({ nameForDebug }) { input.readObjectOrNull(obj, schemas, resolvedType) } override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) = ifThrowsAppend({ nameForDebug }) { - output.writeObjectOrNull(readMethod!!.invoke(obj), data, resolvedType) + output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType) } private val nameForDebug = "$name(${resolvedType.typeName})" @@ -104,7 +151,10 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r /** * A property serializer for most AMQP primitive type (Int, String, etc). */ - class AMQPPrimitivePropertySerializer(name: String, readMethod: Method?, resolvedType: Type) : PropertySerializer(name, readMethod, resolvedType) { + class AMQPPrimitivePropertySerializer( + name: String, + readMethod: PropertyReader, + resolvedType: Type) : PropertySerializer(name, readMethod, resolvedType) { override fun writeClassInfo(output: SerializationOutput) {} override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? { @@ -112,7 +162,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r } override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) { - val value = readMethod!!.invoke(obj) + val value = propertyReader.read(obj) if (value is ByteArray) { data.putObject(Binary(value)) } else { @@ -126,7 +176,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r * value of the character is stored in numeric UTF-16 form and on deserialisation requires explicit * casting back to a char otherwise it's treated as an Integer and a TypeMismatch occurs */ - class AMQPCharPropertySerializer(name: String, readMethod: Method?) : + class AMQPCharPropertySerializer(name: String, readMethod: PropertyReader) : PropertySerializer(name, readMethod, Character::class.java) { override fun writeClassInfo(output: SerializationOutput) {} @@ -135,7 +185,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r } override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) { - val input = readMethod!!.invoke(obj) + val input = propertyReader.read(obj) if (input != null) data.putShort((input as Char).toShort()) else data.putNull() } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt index 5e3e28711a..e7b21f5e38 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt @@ -20,6 +20,7 @@ import kotlin.reflect.full.findAnnotation import kotlin.reflect.full.primaryConstructor import kotlin.reflect.jvm.isAccessible import kotlin.reflect.jvm.javaType +import kotlin.reflect.jvm.kotlinProperty /** * Annotation indicating a constructor to be used to reconstruct instances of a class during deserialization. @@ -29,8 +30,8 @@ import kotlin.reflect.jvm.javaType annotation class ConstructorForDeserialization data class ConstructorDestructorMethods( - val getters : Collection, - val setters : Collection) + val getters: Collection, + val setters: Collection) /** * Code for finding the constructor we will use for deserialization. @@ -100,26 +101,31 @@ private fun propertiesForSerializationFromConstructor( for (param in kotlinConstructor.parameters) { val name = param.name ?: throw NotSerializableException("Constructor parameter of $clazz has no name.") - val matchingProperty = properties[name] ?: - try { - clazz.getDeclaredField(param.name) - throw NotSerializableException("Property '$name' or its getter is non public, this renders class '$clazz' unserializable") - } catch (e: NoSuchFieldException) { - throw NotSerializableException("No property matching constructor parameter named '$name' of '$clazz'. " + - "If using Java, check that you have the -parameters option specified in the Java compiler. " + - "Alternately, provide a proxy serializer (SerializationCustomSerializer) if recompiling isn't an option") - } + if (name in properties) { + val matchingProperty = properties[name]!! - // Check that the method has a getter in java. - val getter = matchingProperty.readMethod ?: throw NotSerializableException("Property has no getter method for $name of $clazz. " + - "If using Java and the parameter name looks anonymous, check that you have the -parameters option specified in the Java compiler." + - "Alternately, provide a proxy serializer (SerializationCustomSerializer) if recompiling isn't an option") - val returnType = resolveTypeVariables(getter.genericReturnType, type) - if (constructorParamTakesReturnTypeOfGetter(returnType, getter.genericReturnType, param)) { - rc += PropertySerializer.make(name, getter, returnType, factory) + // Check that the method has a getter in java. + val getter = matchingProperty.readMethod ?: throw NotSerializableException("Property has no getter method for $name of $clazz. " + + "If using Java and the parameter name looks anonymous, check that you have the -parameters option specified in the Java compiler." + + "Alternately, provide a proxy serializer (SerializationCustomSerializer) if recompiling isn't an option") + val returnType = resolveTypeVariables(getter.genericReturnType, type) + if (constructorParamTakesReturnTypeOfGetter(returnType, getter.genericReturnType, param)) { + rc += PropertySerializer.make(name, PublicPropertyReader(getter), returnType, factory) + } else { + throw NotSerializableException("Property type $returnType for $name of $clazz differs from constructor parameter type ${param.type.javaType}") + } } else { - throw NotSerializableException("Property type $returnType for $name of $clazz differs from constructor parameter type ${param.type.javaType}") + try { + val field = (clazz.getDeclaredField(param.name)) + + rc += PropertySerializer.make(name, PrivatePropertyReader(field, type), field.genericType, factory) + } catch (e: NoSuchFieldException) { + throw NotSerializableException("No property matching constructor parameter named '$name' of '$clazz'. " + + "If using Java, check that you have the -parameters option specified in the Java compiler. " + + "Alternately, provide a proxy serializer (SerializationCustomSerializer) if recompiling isn't an option") + } } + } return ConstructorDestructorMethods(rc, emptyList()) @@ -130,11 +136,11 @@ private fun propertiesForSerializationFromConstructor( * and use those */ private fun propertiesForSerializationFromSetters( - properties : Map, + properties: Map, type: Type, factory: SerializerFactory): ConstructorDestructorMethods { - val getters : MutableList = ArrayList(properties.size) - val setters : MutableList = ArrayList(properties.size) + val getters: MutableList = ArrayList(properties.size) + val setters: MutableList = ArrayList(properties.size) properties.forEach { property -> val getter: Method? = property.value.readMethod @@ -146,8 +152,8 @@ private fun propertiesForSerializationFromSetters( // the getter / setter vs property as if there is a difference then that property isn't reported // by the BEAN inspector and thus we don't consider that case here - getters += PropertySerializer.make(property.key, getter, resolveTypeVariables(getter.genericReturnType, type), - factory) + getters += PropertySerializer.make(property.key, PublicPropertyReader(getter), + resolveTypeVariables(getter.genericReturnType, type), factory) setters += setter } @@ -159,15 +165,22 @@ private fun constructorParamTakesReturnTypeOfGetter(getterReturnType: Type, rawG return typeToken.isSupertypeOf(getterReturnType) || typeToken.isSupertypeOf(rawGetterReturnType) } -private fun propertiesForSerializationFromAbstract(clazz: Class<*>, type: Type, factory: SerializerFactory): ConstructorDestructorMethods { +private fun propertiesForSerializationFromAbstract( + clazz: Class<*>, + type: Type, + factory: SerializerFactory): ConstructorDestructorMethods { // Kotlin reflection doesn't work with Java getters the way you might expect, so we drop back to good ol' beans. - val properties = Introspector.getBeanInfo(clazz).propertyDescriptors.filter { it.name != "class" }.sortedBy { it.name }.filterNot { it is IndexedPropertyDescriptor } + val properties = Introspector.getBeanInfo(clazz).propertyDescriptors + .filter { it.name != "class" } + .sortedBy { it.name } + .filterNot { it is IndexedPropertyDescriptor } val rc: MutableList = ArrayList(properties.size) for (property in properties) { // Check that the method has a getter in java. - val getter = property.readMethod ?: throw NotSerializableException("Property has no getter method for ${property.name} of $clazz.") + val getter = property.readMethod ?: throw NotSerializableException( + "Property has no getter method for ${property.name} of $clazz.") val returnType = resolveTypeVariables(getter.genericReturnType, type) - rc += PropertySerializer.make(property.name, getter, returnType, factory) + rc += PropertySerializer.make(property.name, PublicPropertyReader(getter), returnType, factory) } return ConstructorDestructorMethods(rc, emptyList()) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt index 0dbc3f8be9..a6c587bc6e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt @@ -40,6 +40,7 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { val transformsCache = ConcurrentHashMap>>() open val classCarpenter = ClassCarpenter(cl, whitelist) + val classloader: ClassLoader get() = classCarpenter.classloader @@ -381,3 +382,4 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { override fun toString(): String = "?" } } + diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/ThrowableSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/ThrowableSerializer.kt index 5700c08be7..93d8b0fbed 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/ThrowableSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/ThrowableSerializer.kt @@ -25,7 +25,7 @@ class ThrowableSerializer(factory: SerializerFactory) : CustomSerializer.Proxy> serializersByDescriptor = + (ConcurrentHashMap>) f.get(factory); + + assertEquals(1, serializersByDescriptor.size()); + ObjectSerializer cSerializer = ((ObjectSerializer)serializersByDescriptor.values().toArray()[0]); + assertEquals(1, cSerializer.getPropertySerializers().component1().size()); + Object[] propertyReaders = cSerializer.getPropertySerializers().component1().toArray(); + assertTrue (((PropertySerializer)propertyReaders[0]).getPropertyReader() instanceof PrivatePropertyReader); + } + + @Test + public void singlePrivateWithConstructorAndGetter() + throws NotSerializableException, NoSuchFieldException, IllegalAccessException { + SerializerFactory factory = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + SerializationOutput ser = new SerializationOutput(factory); + DeserializationInput des = new DeserializationInput(factory); + + C2 c = new C2("dripping taps"); + C2 c2 = des.deserialize(ser.serialize(c), C2.class); + + assertEquals (c.a, c2.a); + + // + // Now ensure we actually got a private property serializer + // + Field f = SerializerFactory.class.getDeclaredField("serializersByDescriptor"); + f.setAccessible(true); + ConcurrentHashMap> serializersByDescriptor = + (ConcurrentHashMap>) f.get(factory); + + assertEquals(1, serializersByDescriptor.size()); + ObjectSerializer cSerializer = ((ObjectSerializer)serializersByDescriptor.values().toArray()[0]); + assertEquals(1, cSerializer.getPropertySerializers().component1().size()); + Object[] propertyReaders = cSerializer.getPropertySerializers().component1().toArray(); + assertTrue (((PropertySerializer)propertyReaders[0]).getPropertyReader() instanceof PublicPropertyReader); + } +} diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt index 08d7af6ebb..8bbe597b26 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt @@ -1,10 +1,8 @@ package net.corda.nodeapi.internal.serialization.amqp -import net.corda.core.serialization.SerializedBytes import org.apache.qpid.proton.codec.Data import net.corda.nodeapi.internal.serialization.AllWhitelist import net.corda.nodeapi.internal.serialization.EmptyWhitelist -import java.io.NotSerializableException fun testDefaultFactory() = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) fun testDefaultFactoryWithWhitelist() = SerializerFactory(EmptyWhitelist, ClassLoader.getSystemClassLoader()) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/ErrorMessagesTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/ErrorMessagesTests.kt index 15f543da0d..6aab88660a 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/ErrorMessagesTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/ErrorMessagesTests.kt @@ -1,6 +1,7 @@ package net.corda.nodeapi.internal.serialization.amqp import org.assertj.core.api.Assertions +import org.junit.Ignore import org.junit.Test import java.io.NotSerializableException @@ -12,6 +13,8 @@ class ErrorMessagesTests { private fun errMsg(property:String, testname: String) = "Property '$property' or its getter is non public, this renders class 'class $testname\$C' unserializable -> class $testname\$C" + // Java allows this to be set at the class level yet Kotlin doesn't for some reason + @Ignore("Current behaviour allows for the serialization of objects with private members, this will be disallowed at some point in the future") @Test fun privateProperty() { data class C(private val a: Int) @@ -25,6 +28,8 @@ class ErrorMessagesTests { }.isInstanceOf(NotSerializableException::class.java).hasMessage(errMsg("a", testname)) } + // Java allows this to be set at the class level yet Kotlin doesn't for some reason + @Ignore("Current behaviour allows for the serialization of objects with private members, this will be disallowed at some point in the future") @Test fun privateProperty2() { data class C(val a: Int, private val b: Int) @@ -38,6 +43,8 @@ class ErrorMessagesTests { }.isInstanceOf(NotSerializableException::class.java).hasMessage(errMsg("b", testname)) } + // Java allows this to be set at the class level yet Kotlin doesn't for some reason + @Ignore("Current behaviour allows for the serialization of objects with private members, this will be disallowed at some point in the future") @Test fun privateProperty3() { // despite b being private, the getter we've added is public and thus allows for the serialisation @@ -54,6 +61,8 @@ class ErrorMessagesTests { val c = DeserializationInput(sf).deserialize(bytes) } + // Java allows this to be set at the class level yet Kotlin doesn't for some reason + @Ignore("Current behaviour allows for the serialization of objects with private members, this will be disallowed at some point in the future") @Test fun protectedProperty() { data class C(protected val a: Int) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/PrivatePropertyTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/PrivatePropertyTests.kt new file mode 100644 index 0000000000..51405e8c26 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/PrivatePropertyTests.kt @@ -0,0 +1,117 @@ +package net.corda.nodeapi.internal.serialization.amqp + +import junit.framework.TestCase.assertTrue +import junit.framework.TestCase.assertEquals +import org.junit.Test +import org.apache.qpid.proton.amqp.Symbol +import java.util.concurrent.ConcurrentHashMap + +class PrivatePropertyTests { + private val factory = testDefaultFactory() + + @Test + fun testWithOnePrivateProperty() { + data class C(private val b: String) + + val c1 = C("Pants are comfortable sometimes") + val c2 = DeserializationInput(factory).deserialize(SerializationOutput(factory).serialize(c1)) + assertEquals(c1, c2) + } + + @Test + fun testWithOnePrivatePropertyNullableNotNull() { + data class C(private val b: String?) + + val c1 = C("Pants are comfortable sometimes") + val c2 = DeserializationInput(factory).deserialize(SerializationOutput(factory).serialize(c1)) + assertEquals(c1, c2) + } + + @Test + fun testWithOnePrivatePropertyNullableNull() { + data class C(private val b: String?) + + val c1 = C(null) + val c2 = DeserializationInput(factory).deserialize(SerializationOutput(factory).serialize(c1)) + assertEquals(c1, c2) + } + + @Test + fun testWithOnePublicOnePrivateProperty() { + data class C(val a: Int, private val b: Int) + + val c1 = C(1, 2) + val c2 = DeserializationInput(factory).deserialize(SerializationOutput(factory).serialize(c1)) + assertEquals(c1, c2) + } + + @Test + fun testWithOnePublicOnePrivateProperty2() { + data class C(val a: Int, private val b: Int) + + val c1 = C(1, 2) + val schemaAndBlob = SerializationOutput(factory).serializeAndReturnSchema(c1) + assertEquals(1, schemaAndBlob.schema.types.size) + + val field = SerializerFactory::class.java.getDeclaredField("serializersByDescriptor") + field.isAccessible = true + @Suppress("UNCHECKED_CAST") + val serializersByDescriptor = field.get(factory) as ConcurrentHashMap> + + val schemaDescriptor = schemaAndBlob.schema.types.first().descriptor.name + serializersByDescriptor.filterKeys { (it as Symbol) == schemaDescriptor }.values.apply { + assertEquals(1, this.size) + assertTrue(this.first() is ObjectSerializer) + val propertySerializers = (this.first() as ObjectSerializer).propertySerializers.getters.toList() + assertEquals(2, propertySerializers.size) + // a was public so should have a synthesised getter + assertTrue(propertySerializers[0].propertyReader is PublicPropertyReader) + + // b is private and thus won't have teh getter so we'll have reverted + // to using reflection to remove the inaccessible property + assertTrue(propertySerializers[1].propertyReader is PrivatePropertyReader) + } + } + + @Test + fun testGetterMakesAPublicReader() { + data class C(val a: Int, private val b: Int) { + @Suppress("UNUSED") + fun getB() = b + } + + val c1 = C(1, 2) + val schemaAndBlob = SerializationOutput(factory).serializeAndReturnSchema(c1) + assertEquals(1, schemaAndBlob.schema.types.size) + + val field = SerializerFactory::class.java.getDeclaredField("serializersByDescriptor") + field.isAccessible = true + @Suppress("UNCHECKED_CAST") + val serializersByDescriptor = field.get(factory) as ConcurrentHashMap> + + val schemaDescriptor = schemaAndBlob.schema.types.first().descriptor.name + serializersByDescriptor.filterKeys { (it as Symbol) == schemaDescriptor }.values.apply { + assertEquals(1, this.size) + assertTrue(this.first() is ObjectSerializer) + val propertySerializers = (this.first() as ObjectSerializer).propertySerializers.getters.toList() + assertEquals(2, propertySerializers.size) + + // as before, a is public so we'll use the getter method + assertTrue(propertySerializers[0].propertyReader is PublicPropertyReader) + + // the getB() getter explicitly added means we should use the "normal" public + // method reader rather than the private oen + assertTrue(propertySerializers[1].propertyReader is PublicPropertyReader) + } + } + + @Test + fun testNested() { + data class Inner(private val a: Int) + data class Outer(private val i: Inner) + + val c1 = Outer(Inner(1010101)) + val c2 = DeserializationInput(factory).deserialize(SerializationOutput(factory).serialize(c1)) + assertEquals(c1, c2) + } +} \ No newline at end of file diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenterTestUtils.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenterTestUtils.kt index 8773140f6d..a27b6440d3 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenterTestUtils.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/carpenter/ClassCarpenterTestUtils.kt @@ -35,9 +35,16 @@ fun Schema.mangleNames(names: List): Schema { return Schema(types = newTypes) } +/** + * Custom implementation of a [SerializerFactory] where we need to give it a class carpenter + * rather than have it create its own + */ +class SerializerFactoryExternalCarpenter(override val classCarpenter: ClassCarpenter) + : SerializerFactory (classCarpenter.whitelist, classCarpenter.classloader) + open class AmqpCarpenterBase(whitelist: ClassWhitelist) { var cc = ClassCarpenter(whitelist = whitelist) - var factory = SerializerFactory(AllWhitelist, cc.classloader) + var factory = SerializerFactoryExternalCarpenter(cc) fun serialise(clazz: Any) = SerializationOutput(factory).serialize(clazz) fun testName(): String = Thread.currentThread().stackTrace[2].methodName From c2bd7403a8dc560a1c9d1a731038c50a0233704e Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Wed, 10 Jan 2018 11:42:08 +0000 Subject: [PATCH 10/17] hibernate mapping changes (#2337) * add foreign key names and move the participants mapping to the subclass so that the table name can be configured * update api-current file * fix compilation errors * PR changes * PR changes --- .ci/api-current.txt | 8 ++++---- .../core/internal/schemas/NodeInfoSchema.kt | 6 +++--- .../net/corda/core/schemas/CommonSchema.kt | 13 ++++++------ docs/source/changelog.rst | 3 +++ docs/source/upgrade-notes.rst | 20 +++++++++++++++++++ .../finance/schemas/SampleCashSchemaV2.kt | 13 ++++++++---- .../schemas/SampleCommercialPaperSchemaV2.kt | 13 ++++++++---- .../corda/node/services/vault/VaultSchema.kt | 6 ++++-- .../internal/vault/DummyDealContract.kt | 2 +- .../internal/vault/DummyDealStateSchemaV1.kt | 14 +++++++------ .../internal/vault/DummyLinearContract.kt | 2 +- .../vault/DummyLinearStateSchemaV1.kt | 2 ++ .../vault/DummyLinearStateSchemaV2.kt | 18 +++++++++-------- 13 files changed, 80 insertions(+), 40 deletions(-) diff --git a/.ci/api-current.txt b/.ci/api-current.txt index f3eee165fa..493a31b91a 100644 --- a/.ci/api-current.txt +++ b/.ci/api-current.txt @@ -2613,22 +2613,22 @@ public final class net.corda.core.schemas.CommonSchemaV1 extends net.corda.core. @org.jetbrains.annotations.NotNull public final net.corda.core.identity.AbstractParty getIssuer() @org.jetbrains.annotations.NotNull public final byte[] getIssuerRef() @org.jetbrains.annotations.NotNull public final net.corda.core.identity.AbstractParty getOwner() - @org.jetbrains.annotations.Nullable public final Set getParticipants() + @org.jetbrains.annotations.Nullable public Set getParticipants() public final long getQuantity() public final void setIssuer(net.corda.core.identity.AbstractParty) public final void setIssuerRef(byte[]) public final void setOwner(net.corda.core.identity.AbstractParty) - public final void setParticipants(Set) + public void setParticipants(Set) public final void setQuantity(long) ## @javax.persistence.MappedSuperclass @net.corda.core.serialization.CordaSerializable public static class net.corda.core.schemas.CommonSchemaV1$LinearState extends net.corda.core.schemas.PersistentState public (Set, String, UUID) public (net.corda.core.contracts.UniqueIdentifier, Set) @org.jetbrains.annotations.Nullable public final String getExternalId() - @org.jetbrains.annotations.Nullable public final Set getParticipants() + @org.jetbrains.annotations.Nullable public Set getParticipants() @org.jetbrains.annotations.NotNull public final UUID getUuid() public final void setExternalId(String) - public final void setParticipants(Set) + public void setParticipants(Set) public final void setUuid(UUID) ## public class net.corda.core.schemas.MappedSchema extends java.lang.Object diff --git a/core/src/main/kotlin/net/corda/core/internal/schemas/NodeInfoSchema.kt b/core/src/main/kotlin/net/corda/core/internal/schemas/NodeInfoSchema.kt index 9cb84357b5..6e35a631c4 100644 --- a/core/src/main/kotlin/net/corda/core/internal/schemas/NodeInfoSchema.kt +++ b/core/src/main/kotlin/net/corda/core/internal/schemas/NodeInfoSchema.kt @@ -32,14 +32,14 @@ object NodeInfoSchemaV1 : MappedSchema( @Column(name = "addresses") @OneToMany(cascade = arrayOf(CascadeType.ALL), orphanRemoval = true) - @JoinColumn(name = "node_info_id") + @JoinColumn(name = "node_info_id", foreignKey = ForeignKey(name = "FK__info_hosts__infos")) val addresses: List, @Column(name = "legal_identities_certs") @ManyToMany(cascade = arrayOf(CascadeType.ALL)) @JoinTable(name = "node_link_nodeinfo_party", - joinColumns = arrayOf(JoinColumn(name = "node_info_id")), - inverseJoinColumns = arrayOf(JoinColumn(name = "party_name"))) + joinColumns = arrayOf(JoinColumn(name = "node_info_id", foreignKey = ForeignKey(name = "FK__link_nodeinfo_party__infos"))), + inverseJoinColumns = arrayOf(JoinColumn(name = "party_name", foreignKey = ForeignKey(name = "FK__link_ni_p__info_p_cert")))) val legalIdentitiesAndCerts: List, @Column(name = "platform_version") diff --git a/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt b/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt index a630a7b8e3..e7770667d3 100644 --- a/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt +++ b/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt @@ -5,8 +5,8 @@ import net.corda.core.identity.AbstractParty import org.hibernate.annotations.Type import java.util.* import javax.persistence.Column -import javax.persistence.ElementCollection import javax.persistence.MappedSuperclass +import javax.persistence.Transient /** * JPA representation of the common schema entities @@ -23,9 +23,8 @@ object CommonSchemaV1 : MappedSchema(schemaFamily = CommonSchema.javaClass, vers /** [ContractState] attributes */ /** X500Name of participant parties **/ - @ElementCollection - @Column(name = "participants") - var participants: MutableSet? = null, + @Transient + open var participants: MutableSet? = null, /** * Represents a [LinearState] [UniqueIdentifier] @@ -34,6 +33,7 @@ object CommonSchemaV1 : MappedSchema(schemaFamily = CommonSchema.javaClass, vers var externalId: String?, @Column(name = "uuid", nullable = false) + @Type(type = "uuid-char") var uuid: UUID ) : PersistentState() { @@ -48,9 +48,8 @@ object CommonSchemaV1 : MappedSchema(schemaFamily = CommonSchema.javaClass, vers /** [ContractState] attributes */ /** X500Name of participant parties **/ - @ElementCollection - @Column(name = "participants") - var participants: MutableSet? = null, + @Transient + open var participants: MutableSet? = null, /** [OwnableState] attributes */ diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index da570bd9ee..ac9093745b 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,6 +6,9 @@ from the previous milestone release. UNRELEASED ---------- +* JPA Mapping annotations for States extending ``CommonSchemaV1.LinearState`` and ``CommonSchemaV1.FungibleState`` on the + `participants` collection need to be moved to the actual class. This allows to properly specify the unique table name per a collection. + See: DummyDealStateSchemaV1.PersistentDummyDealState * X.509 certificates now have an extension that specifies the Corda role the certificate is used for, and the role hierarchy is now enforced in the validation code. See ``net.corda.core.internal.CertRole`` for the current implementation diff --git a/docs/source/upgrade-notes.rst b/docs/source/upgrade-notes.rst index baa2328b4e..da66072fbe 100644 --- a/docs/source/upgrade-notes.rst +++ b/docs/source/upgrade-notes.rst @@ -31,6 +31,26 @@ We also strongly recommend cross referencing with the :doc:`changelog` to confir UNRELEASED ---------- +* For existing contract ORM schemas that extend from `CommonSchemaV1.LinearState` or `CommonSchemaV1.FungibleState`, + you will need to explicitly map the `participants` collection to a database table. Previously this mapping was done in the + superclass, but that makes it impossible to properly configure the table name. + The required change is to add the ``override var participants: MutableSet? = null`` field to your class, and + add JPA mappings. For ex., see this example: + +.. sourcecode:: kotlin + + @Entity + @Table(name = "cash_states_v2", + indexes = arrayOf(Index(name = "ccy_code_idx2", columnList = "ccy_code"))) + class PersistentCashState( + + @ElementCollection + @Column(name = "participants") + @CollectionTable(name="cash_states_v2_participants", joinColumns = arrayOf( + JoinColumn(name = "output_index", referencedColumnName = "output_index"), + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + override var participants: MutableSet? = null, + Testing ~~~~~~~ diff --git a/finance/src/test/kotlin/net/corda/finance/schemas/SampleCashSchemaV2.kt b/finance/src/test/kotlin/net/corda/finance/schemas/SampleCashSchemaV2.kt index 8b548f89ce..bea02485cf 100644 --- a/finance/src/test/kotlin/net/corda/finance/schemas/SampleCashSchemaV2.kt +++ b/finance/src/test/kotlin/net/corda/finance/schemas/SampleCashSchemaV2.kt @@ -4,10 +4,7 @@ import net.corda.core.identity.AbstractParty import net.corda.core.schemas.CommonSchemaV1 import net.corda.core.schemas.MappedSchema import net.corda.core.utilities.OpaqueBytes -import javax.persistence.Column -import javax.persistence.Entity -import javax.persistence.Index -import javax.persistence.Table +import javax.persistence.* /** * Second version of a cash contract ORM schema that extends the common @@ -19,6 +16,14 @@ object SampleCashSchemaV2 : MappedSchema(schemaFamily = CashSchema.javaClass, ve @Table(name = "cash_states_v2", indexes = arrayOf(Index(name = "ccy_code_idx2", columnList = "ccy_code"))) class PersistentCashState( + + @ElementCollection + @Column(name = "participants") + @CollectionTable(name="cash_states_v2_participants", joinColumns = arrayOf( + JoinColumn(name = "output_index", referencedColumnName = "output_index"), + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + override var participants: MutableSet? = null, + /** product type */ @Column(name = "ccy_code", length = 3) var currency: String, diff --git a/finance/src/test/kotlin/net/corda/finance/schemas/SampleCommercialPaperSchemaV2.kt b/finance/src/test/kotlin/net/corda/finance/schemas/SampleCommercialPaperSchemaV2.kt index 2c7882c048..fb646d123b 100644 --- a/finance/src/test/kotlin/net/corda/finance/schemas/SampleCommercialPaperSchemaV2.kt +++ b/finance/src/test/kotlin/net/corda/finance/schemas/SampleCommercialPaperSchemaV2.kt @@ -8,10 +8,7 @@ import net.corda.core.utilities.MAX_HASH_HEX_SIZE import net.corda.core.utilities.OpaqueBytes import org.hibernate.annotations.Type import java.time.Instant -import javax.persistence.Column -import javax.persistence.Entity -import javax.persistence.Index -import javax.persistence.Table +import javax.persistence.* /** * Second version of a cash contract ORM schema that extends the common @@ -24,6 +21,14 @@ object SampleCommercialPaperSchemaV2 : MappedSchema(schemaFamily = CommercialPap indexes = arrayOf(Index(name = "ccy_code_index2", columnList = "ccy_code"), Index(name = "maturity_index2", columnList = "maturity_instant"))) class PersistentCommercialPaperState( + + @ElementCollection + @Column(name = "participants") + @CollectionTable(name="cp_states_v2_participants", joinColumns = arrayOf( + JoinColumn(name = "output_index", referencedColumnName = "output_index"), + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + override var participants: MutableSet? = null, + @Column(name = "maturity_instant") var maturity: Instant, diff --git a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt index 78d5a3ac60..a86585bc87 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt @@ -77,7 +77,8 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio @CollectionTable(name = "vault_linear_states_parts", joinColumns = arrayOf( JoinColumn(name = "output_index", referencedColumnName = "output_index"), - JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id")), + foreignKey = ForeignKey(name = "FK__lin_stat_parts__lin_stat")) @Column(name = "participants") var participants: MutableSet? = null, // Reason for not using Set is described here: @@ -109,7 +110,8 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio @CollectionTable(name = "vault_fungible_states_parts", joinColumns = arrayOf( JoinColumn(name = "output_index", referencedColumnName = "output_index"), - JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id")), + foreignKey = ForeignKey(name = "FK__fung_st_parts__fung_st")) @Column(name = "participants") var participants: MutableSet? = null, diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealContract.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealContract.kt index 453dac658c..1cd75a83b0 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealContract.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealContract.kt @@ -31,7 +31,7 @@ class DummyDealContract : Contract { override fun generateMappedObject(schema: MappedSchema): PersistentState { return when (schema) { is DummyDealStateSchemaV1 -> DummyDealStateSchemaV1.PersistentDummyDealState( - _participants = participants.toSet(), + participants = participants.toMutableSet(), uid = linearId ) else -> throw IllegalArgumentException("Unrecognised schema $schema") diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealStateSchemaV1.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealStateSchemaV1.kt index 493e4688a5..9ed95d29b7 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealStateSchemaV1.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyDealStateSchemaV1.kt @@ -4,9 +4,7 @@ import net.corda.core.contracts.UniqueIdentifier import net.corda.core.identity.AbstractParty import net.corda.core.schemas.CommonSchemaV1 import net.corda.core.schemas.MappedSchema -import javax.persistence.Entity -import javax.persistence.Table -import javax.persistence.Transient +import javax.persistence.* /** * An object used to fully qualify the [DummyDealStateSchema] family name (i.e. independent of version). @@ -22,11 +20,15 @@ object DummyDealStateSchemaV1 : MappedSchema(schemaFamily = DummyDealStateSchema @Table(name = "dummy_deal_states") class PersistentDummyDealState( /** parent attributes */ - @Transient - val _participants: Set, + @ElementCollection + @Column(name = "participants") + @CollectionTable(name = "dummy_deal_states_participants", joinColumns = arrayOf( + JoinColumn(name = "output_index", referencedColumnName = "output_index"), + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + override var participants: MutableSet? = null, @Transient val uid: UniqueIdentifier - ) : CommonSchemaV1.LinearState(uid, _participants) + ) : CommonSchemaV1.LinearState(uuid = uid.id, externalId = uid.externalId, participants = participants) } diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearContract.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearContract.kt index 0a2a962084..359e602180 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearContract.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearContract.kt @@ -49,7 +49,7 @@ class DummyLinearContract : Contract { linearBoolean = linearBoolean ) is DummyLinearStateSchemaV2 -> DummyLinearStateSchemaV2.PersistentDummyLinearState( - _participants = participants.toSet(), + participants = participants.toMutableSet(), uid = linearId, linearString = linearString, linearNumber = linearNumber, diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV1.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV1.kt index aa649337ff..d2ff099848 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV1.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV1.kt @@ -4,6 +4,7 @@ import net.corda.core.contracts.ContractState import net.corda.core.identity.AbstractParty import net.corda.core.schemas.MappedSchema import net.corda.core.schemas.PersistentState +import org.hibernate.annotations.Type import java.time.Instant import java.util.* import javax.persistence.* @@ -36,6 +37,7 @@ object DummyLinearStateSchemaV1 : MappedSchema(schemaFamily = DummyLinearStateSc var externalId: String?, @Column(name = "uuid", nullable = false) + @Type(type = "uuid-char") var uuid: UUID, /** diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV2.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV2.kt index 224d6fed87..6a25167c71 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV2.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/DummyLinearStateSchemaV2.kt @@ -4,9 +4,7 @@ import net.corda.core.contracts.UniqueIdentifier import net.corda.core.identity.AbstractParty import net.corda.core.schemas.CommonSchemaV1 import net.corda.core.schemas.MappedSchema -import javax.persistence.Column -import javax.persistence.Entity -import javax.persistence.Table +import javax.persistence.* /** * Second version of a cash contract ORM schema that extends the common @@ -17,6 +15,14 @@ object DummyLinearStateSchemaV2 : MappedSchema(schemaFamily = DummyLinearStateSc @Entity @Table(name = "dummy_linear_states_v2") class PersistentDummyLinearState( + + @ElementCollection + @Column(name = "participants") + @CollectionTable(name = "dummy_linear_states_v2_participants", joinColumns = arrayOf( + JoinColumn(name = "output_index", referencedColumnName = "output_index"), + JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))) + override var participants: MutableSet? = null, + @Column(name = "linear_string") var linearString: String, @Column(name = "linear_number") var linearNumber: Long, @@ -25,11 +31,7 @@ object DummyLinearStateSchemaV2 : MappedSchema(schemaFamily = DummyLinearStateSc @Column(name = "linear_boolean") var linearBoolean: Boolean, - /** parent attributes */ - @Transient - val _participants: Set, - @Transient val uid: UniqueIdentifier - ) : CommonSchemaV1.LinearState(uid, _participants) + ) : CommonSchemaV1.LinearState(uuid = uid.id, externalId = uid.externalId, participants = participants) } From 017f865fa388245bafc694d535df43a5b106b319 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Mon, 11 Dec 2017 18:32:12 +0000 Subject: [PATCH 11/17] CORDA-852 - Fix AMQP serialisation of nested generic --- .../corda/nodeapi/internal/serialization/amqp/GenericsTests.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index 9882c79b41..e9ab19db4f 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -111,4 +111,4 @@ class GenericsTests { } } } -} \ No newline at end of file +} From 41220de816ddd11832af36a9d5851447a00ad605 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Mon, 11 Dec 2017 20:01:46 +0000 Subject: [PATCH 12/17] CORDA-855 - Fix for fingerprinting generics in AMQP * Undo refactor --- .../amqp/EnumEvolutionSerializer.kt | 4 +- .../serialization/amqp/EvolutionSerializer.kt | 49 +++++++++++++ .../internal/serialization/amqp/Schema.kt | 67 +++++++++++------- .../serialization/amqp/SerializerFactory.kt | 69 ++++++++++--------- .../amqp/JavaSerialiseEnumTests.java | 4 +- .../amqp/JavaSerializationOutputTests.java | 7 +- .../amqp/ListsSerializationJavaTest.java | 4 +- .../serialization/amqp/AMQPTestUtils.kt | 2 + .../amqp/DeserializeAndReturnEnvelopeTests.kt | 4 +- .../serialization/amqp/DeserializeMapTests.kt | 2 +- .../DeserializeNeedingCarpentryOfEnumsTest.kt | 4 +- ...erializeNeedingCarpentrySimpleTypesTest.kt | 4 +- .../amqp/DeserializeNeedingCarpentryTests.kt | 2 +- .../amqp/DeserializeSimpleTypesTests.kt | 4 +- .../internal/serialization/amqp/EnumTests.kt | 2 +- .../serialization/amqp/GenericsTests.kt | 69 +++++++++++++++++-- .../amqp/SerializationOutputTests.kt | 6 +- .../amqp/SerializeAndReturnSchemaTest.kt | 2 +- ...ticInitialisationOfSerializedObjectTest.kt | 4 +- 19 files changed, 223 insertions(+), 86 deletions(-) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt index 88639d466b..6e96d4bad8 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt @@ -106,7 +106,7 @@ class EnumEvolutionSerializer( // to the name as it exists. We want to test any new constants have been added to the end // of the enum class val serialisedOrds = ((schemas.schema.types.find { it.name == old.name } as RestrictedType).choices - .associateBy ({ it.value.toInt() }, { conversions[it.name] })) + .associateBy({ it.value.toInt() }, { conversions[it.name] })) if (ordinals.filterNot { serialisedOrds[it.value] == it.key }.isNotEmpty()) { throw NotSerializableException("Constants have been reordered, additions must be appended to the end") @@ -133,4 +133,4 @@ class EnumEvolutionSerializer( override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } -} \ No newline at end of file +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt index 9293729306..e2d9ce671a 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt @@ -130,3 +130,52 @@ class EvolutionSerializer( } } +/** + * Instances of this type are injected into a [SerializerFactory] at creation time to dictate the + * behaviour of evolution within that factory. Under normal circumstances this will simply + * be an object that returns an [EvolutionSerializer]. Of course, any implementation that + * extends this class can be written to invoke whatever behaviour is desired. + */ +abstract class EvolutionSerializerGetterBase { + abstract fun getEvolutionSerializer( + factory: SerializerFactory, + typeNotation: TypeNotation, + newSerializer: AMQPSerializer, + schemas: SerializationSchemas): AMQPSerializer +} + +/** + * The normal use case for generating an [EvolutionSerializer]'s based on the differences + * between the received schema and the class as it exists now on the class path, + */ +class EvolutionSerializerGetter : EvolutionSerializerGetterBase() { + override fun getEvolutionSerializer(factory: SerializerFactory, + typeNotation: TypeNotation, + newSerializer: AMQPSerializer, + schemas: SerializationSchemas): AMQPSerializer = + factory.serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) { + when (typeNotation) { + is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, factory) + is RestrictedType -> EnumEvolutionSerializer.make(typeNotation, newSerializer, factory, schemas) + } + } +} + +/** + * An implementation of [EvolutionSerializerGetterBase] that disables all evolution within a + * [SerializerFactory]. This is most useful in testing where it is known that evolution should not be + * occurring and where bugs may be hidden by transparent invocation of an [EvolutionSerializer]. This + * prevents that by simply throwing an exception whenever such a serializer is requested. + */ +class EvolutionSerializerGetterTesting : EvolutionSerializerGetterBase() { + override fun getEvolutionSerializer(factory: SerializerFactory, + typeNotation: TypeNotation, + newSerializer: AMQPSerializer, + schemas: SerializationSchemas): AMQPSerializer { + throw NotSerializableException("No evolution should be occurring\n" + + " ${typeNotation.name}\n" + + " ${typeNotation.descriptor.name}\n" + + " ${newSerializer.type.typeName}\n" + + " ${newSerializer.typeDescriptor}\n\n${schemas.schema}") + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt index 79b2a57b93..dcfdfe9b7a 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt @@ -348,17 +348,50 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact // This method concatentates various elements of the types recursively as unencoded strings into the hasher, effectively // creating a unique string for a type which we then hash in the calling function above. -private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory): Hasher { - return if (type in alreadySeen) { +private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet, + hasher: Hasher, factory: SerializerFactory, offset: String = ""): Hasher { + // We don't include Example and Example where type is ? or T in this otherwise we + // generate different fingerprints for class Outer(val a: Inner) when serialising + // and deserializing (assuming deserialization is occurring in a factory that didn't + // serialise the object in the first place (and thus the cache lookup fails). This is also + // true of Any, where we need Example and Example to have the same fingerprint + return if (type in alreadySeen && (type !is SerializerFactory.AnyType) && (type !is TypeVariable<*>)) { hasher.putUnencodedChars(ALREADY_SEEN_HASH) } else { alreadySeen += type try { when (type) { - is SerializerFactory.AnyType -> hasher.putUnencodedChars(ANY_TYPE_HASH) + is ParameterizedType -> { + // Hash the rawType + params + val clazz = type.rawType as Class<*> + + val startingHash = if (isCollectionOrMap(clazz)) { + hasher.putUnencodedChars(clazz.name) + } else { + hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) { + fingerprintForObject(type, type, alreadySeen, hasher, factory, "$offset ") + } + } + + // ... and concatentate the type data for each parameter type. + type.actualTypeArguments.fold(startingHash) { orig, paramType -> + fingerprintForType(paramType, type, alreadySeen, orig, factory, "$offset ") + } + } + // Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when + // looking at A and B from Example (remember we call this function recursively). When + // serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but + // when deserializing we only have the wilcard placeholder ?, or AnyType + // + // Note, TypeVariable<*> used to be encided as TYPE_VARIABLE_HASH but that again produces a + // differing fingerprint on serialisation and deserialization + is SerializerFactory.AnyType, + is TypeVariable<*> -> { + hasher.putUnencodedChars("?").putUnencodedChars(ANY_TYPE_HASH) + } is Class<*> -> { if (type.isArray) { - fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory).putUnencodedChars(ARRAY_HASH) + fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, "$offset ").putUnencodedChars(ARRAY_HASH) } else if (SerializerFactory.isPrimitive(type)) { hasher.putUnencodedChars(type.name) } else if (isCollectionOrMap(type)) { @@ -377,31 +410,15 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta // to the CorDapp but maybe reference to the JAR in the short term. hasher.putUnencodedChars(type.name) } else { - fingerprintForObject(type, type, alreadySeen, hasher, factory) + fingerprintForObject(type, type, alreadySeen, hasher, factory, "$offset ") } } } } - is ParameterizedType -> { - // Hash the rawType + params - val clazz = type.rawType as Class<*> - val startingHash = if (isCollectionOrMap(clazz)) { - hasher.putUnencodedChars(clazz.name) - } else { - hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) { - fingerprintForObject(type, type, alreadySeen, hasher, factory) - } - } - // ... and concatentate the type data for each parameter type. - type.actualTypeArguments.fold(startingHash) { orig, paramType -> - fingerprintForType(paramType, type, alreadySeen, orig, factory) - } - } // Hash the element type + some array hash is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen, - hasher, factory).putUnencodedChars(ARRAY_HASH) + hasher, factory, "$offset ").putUnencodedChars(ARRAY_HASH) // TODO: include bounds - is TypeVariable<*> -> hasher.putUnencodedChars(type.name).putUnencodedChars(TYPE_VARIABLE_HASH) is WildcardType -> hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH) else -> throw NotSerializableException("Don't know how to hash") } @@ -416,15 +433,15 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta private fun isCollectionOrMap(type: Class<*>) = (Collection::class.java.isAssignableFrom(type) || Map::class.java.isAssignableFrom(type)) && !EnumSet::class.java.isAssignableFrom(type) -private fun fingerprintForObject(type: Type, contextType: Type?, alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory): Hasher { +private fun fingerprintForObject(type: Type, contextType: Type?, alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory, offset: String = ""): Hasher { // Hash the class + properties + interfaces val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type") propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory).getters .fold(hasher.putUnencodedChars(name)) { orig, prop -> - fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory) + fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory, "$offset ") .putUnencodedChars(prop.name) .putUnencodedChars(if (prop.mandatory) NOT_NULLABLE_HASH else NULLABLE_HASH) } - interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory) } + interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, "$offset ") } return hasher } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt index a6c587bc6e..2a7c6557bb 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt @@ -20,6 +20,10 @@ data class FactorySchemaAndDescriptor(val schemas: SerializationSchemas, val typ /** * Factory of serializers designed to be shared across threads and invocations. + * + * @property evolutionSerializerGetter controls how evolution serializers are generated by the factory. The normal + * use case is an [EvolutionSerializer] type is returned. However, in some scenarios, primarily testing, this + * can be altered to fit the requirements of the test. */ // TODO: support for intern-ing of deserialized objects for some core types (e.g. PublicKey) for memory efficiency // TODO: maybe support for caching of serialized form of some core types for performance @@ -33,9 +37,12 @@ data class FactorySchemaAndDescriptor(val schemas: SerializationSchemas, val typ // TODO: need to rethink matching of constructor to properties in relation to implementing interfaces and needing those properties etc. // TODO: need to support super classes as well as interfaces with our current code base... what's involved? If we continue to ban, what is the impact? @ThreadSafe -open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { +open class SerializerFactory( + val whitelist: ClassWhitelist, + cl: ClassLoader, + private val evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter()) { private val serializersByType = ConcurrentHashMap>() - private val serializersByDescriptor = ConcurrentHashMap>() + val serializersByDescriptor = ConcurrentHashMap>() private val customSerializers = CopyOnWriteArrayList() val transformsCache = ConcurrentHashMap>>() @@ -44,17 +51,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { val classloader: ClassLoader get() = classCarpenter.classloader - private fun getEvolutionSerializer( - typeNotation: TypeNotation, - newSerializer: AMQPSerializer, - schemas: SerializationSchemas): AMQPSerializer { - return serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) { - when (typeNotation) { - is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, this) - is RestrictedType -> EnumEvolutionSerializer.make(typeNotation, newSerializer, this, schemas) - } - } - } + private fun getEvolutionSerializer(typeNotation: TypeNotation, newSerializer: AMQPSerializer, + schemas: SerializationSchemas) + = evolutionSerializerGetter.getEvolutionSerializer(this, typeNotation, newSerializer, schemas) /** * Look up, and manufacture if necessary, a serializer for the given type. @@ -93,7 +92,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { whitelist.requireWhitelisted(actualType) EnumSerializer(actualType, actualClass ?: declaredClass, this) } - else -> makeClassSerializer(actualClass ?: declaredClass, actualType, declaredType) + else -> { + makeClassSerializer(actualClass ?: declaredClass, actualType, declaredType) + } } serializersByDescriptor.putIfAbsent(serializer.typeDescriptor, serializer) @@ -102,23 +103,23 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { } /** - * Try and infer concrete types for any generics type variables for the actual class encountered, based on the declared - * type. + * Try and infer concrete types for any generics type variables for the actual class encountered, + * based on the declared type. */ // TODO: test GenericArrayType - private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>, declaredType: Type): Type? = - when (declaredType) { - is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType) - // Nothing to infer, otherwise we'd have ParameterizedType - is Class<*> -> actualClass - is GenericArrayType -> { - val declaredComponent = declaredType.genericComponentType - inferTypeVariables(actualClass?.componentType, declaredComponent.asClass()!!, declaredComponent)?.asArray() - } - is TypeVariable<*> -> actualClass - is WildcardType -> actualClass - else -> null - } + private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>, + declaredType: Type) : Type? = when (declaredType) { + is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType) + // Nothing to infer, otherwise we'd have ParameterizedType + is Class<*> -> actualClass + is GenericArrayType -> { + val declaredComponent = declaredType.genericComponentType + inferTypeVariables(actualClass?.componentType, declaredComponent.asClass()!!, declaredComponent)?.asArray() + } + is TypeVariable<*> -> actualClass + is WildcardType -> actualClass + else -> null + } /** * Try and infer concrete types for any generics type variables for the actual class encountered, based on the declared @@ -214,9 +215,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { try { val serialiser = processSchemaEntry(typeNotation) - // if we just successfully built a serialiser for the type but the type fingerprint + // if we just successfully built a serializer for the type but the type fingerprint // doesn't match that of the serialised object then we are dealing with different - // instance of the class, as such we need to build an EvolutionSerialiser + // instance of the class, as such we need to build an EvolutionSerializer if (serialiser.typeDescriptor != typeNotation.descriptor.name) { getEvolutionSerializer(typeNotation, serialiser, schemaAndDescriptor.schemas) } @@ -331,13 +332,15 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) { private val namesOfPrimitiveTypes: Map> = primitiveTypeNames.map { it.value to it.key }.toMap() - fun nameForType(type: Type): String = when (type) { + fun nameForType(type: Type): String = when (type) { is Class<*> -> { primitiveTypeName(type) ?: if (type.isArray) { "${nameForType(type.componentType)}${if (type.componentType.isPrimitive) "[p]" else "[]"}" } else type.name } - is ParameterizedType -> "${nameForType(type.rawType)}<${type.actualTypeArguments.joinToString { nameForType(it) }}>" + is ParameterizedType -> { + "${nameForType(type.rawType)}<${type.actualTypeArguments.joinToString { nameForType(it) }}>" + } is GenericArrayType -> "${nameForType(type.genericComponentType)}[]" is WildcardType -> "?" is TypeVariable<*> -> "?" diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerialiseEnumTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerialiseEnumTests.java index e0b65ad27c..a64f9c3d9e 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerialiseEnumTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerialiseEnumTests.java @@ -29,7 +29,9 @@ public class JavaSerialiseEnumTests { public void testJavaConstructorAnnotations() throws NotSerializableException { Bra bra = new Bra(Bras.UNDERWIRE); - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); SerializationOutput ser = new SerializationOutput(factory1); SerializedBytes bytes = ser.serialize(bra); } diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerializationOutputTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerializationOutputTests.java index 1743c645b4..4379718003 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerializationOutputTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaSerializationOutputTests.java @@ -172,8 +172,11 @@ public class JavaSerializationOutputTests { } private Object serdes(Object obj) throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); - SerializerFactory factory2 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); + SerializerFactory factory2 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); SerializationOutput ser = new SerializationOutput(factory1); SerializedBytes bytes = ser.serialize(obj); diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ListsSerializationJavaTest.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ListsSerializationJavaTest.java index 6085546c37..c77f452b33 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ListsSerializationJavaTest.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ListsSerializationJavaTest.java @@ -125,7 +125,9 @@ public class ListsSerializationJavaTest { // Have to have own version as Kotlin inline functions cannot be easily called from Java private static void assertEqualAfterRoundTripSerialization(T container, Class clazz) throws Exception { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); SerializationOutput ser = new SerializationOutput(factory1); SerializedBytes bytes = ser.serialize(container); DeserializationInput des = new DeserializationInput(factory1); diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt index 8bbe597b26..d7701aed52 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPTestUtils.kt @@ -5,6 +5,8 @@ import net.corda.nodeapi.internal.serialization.AllWhitelist import net.corda.nodeapi.internal.serialization.EmptyWhitelist fun testDefaultFactory() = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) +fun testDefaultFactoryNoEvolution() = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader(), + EvolutionSerializerGetterTesting()) fun testDefaultFactoryWithWhitelist() = SerializerFactory(EmptyWhitelist, ClassLoader.getSystemClassLoader()) class TestSerializationOutput( diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeAndReturnEnvelopeTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeAndReturnEnvelopeTests.kt index 7b9c7a4e39..d793c9f0d2 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeAndReturnEnvelopeTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeAndReturnEnvelopeTests.kt @@ -11,13 +11,14 @@ class DeserializeAndReturnEnvelopeTests { @Suppress("NOTHING_TO_INLINE") inline private fun classTestName(clazz: String) = "${this.javaClass.name}\$${testName()}\$$clazz" + val factory = testDefaultFactoryNoEvolution() + @Test fun oneType() { data class A(val a: Int, val b: String) val a = A(10, "20") - val factory = testDefaultFactory() fun serialise(clazz: Any) = SerializationOutput(factory).serialize(clazz) val obj = DeserializationInput(factory).deserializeAndReturnEnvelope(serialise(a)) @@ -33,7 +34,6 @@ class DeserializeAndReturnEnvelopeTests { val b = B(A(10, "20"), 30.0F) - val factory = testDefaultFactory() fun serialise(clazz: Any) = SerializationOutput(factory).serialize(clazz) val obj = DeserializationInput(factory).deserializeAndReturnEnvelope(serialise(b)) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeMapTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeMapTests.kt index 68e106fd89..2e910bf19c 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeMapTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeMapTests.kt @@ -12,7 +12,7 @@ class DeserializeMapTests { private const val VERBOSE = false } - private val sf = testDefaultFactory() + private val sf = testDefaultFactoryNoEvolution() @Test fun mapTest() { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryOfEnumsTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryOfEnumsTest.kt index de33b7cf18..caa3e06c4f 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryOfEnumsTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryOfEnumsTest.kt @@ -18,7 +18,7 @@ class DeserializeNeedingCarpentryOfEnumsTest : AmqpCarpenterBase(AllWhitelist) { // // Setup the test // - val setupFactory = testDefaultFactory() + val setupFactory = testDefaultFactoryNoEvolution() val enumConstants = listOf("AAA", "BBB", "CCC", "DDD", "EEE", "FFF", "GGG", "HHH", "III", "JJJ").associateBy({ it }, { EnumField() }) @@ -57,7 +57,7 @@ class DeserializeNeedingCarpentryOfEnumsTest : AmqpCarpenterBase(AllWhitelist) { // // Setup the test // - val setupFactory = testDefaultFactory() + val setupFactory = testDefaultFactoryNoEvolution() val enumConstants = listOf("AAA", "BBB", "CCC", "DDD", "EEE", "FFF", "GGG", "HHH", "III", "JJJ").associateBy({ it }, { EnumField() }) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentrySimpleTypesTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentrySimpleTypesTest.kt index 2ed861152d..002adc5e24 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentrySimpleTypesTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentrySimpleTypesTest.kt @@ -17,8 +17,8 @@ class DeserializeNeedingCarpentrySimpleTypesTest : AmqpCarpenterBase(AllWhitelis private const val VERBOSE = false } - private val sf = testDefaultFactory() - private val sf2 = testDefaultFactory() + private val sf = testDefaultFactoryNoEvolution() + private val sf2 = testDefaultFactoryNoEvolution() @Test fun singleInt() { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryTests.kt index fdd981cd08..f54112b2dc 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeNeedingCarpentryTests.kt @@ -27,7 +27,7 @@ class DeserializeNeedingCarpentryTests : AmqpCarpenterBase(AllWhitelist) { private const val VERBOSE = false } - private val sf1 = testDefaultFactory() + private val sf1 = testDefaultFactoryNoEvolution() // Deserialize with whitelisting on to check that `CordaSerializable` annotation present. private val sf2 = testDefaultFactoryWithWhitelist() diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeSimpleTypesTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeSimpleTypesTests.kt index 4c5c9311ec..6f2dcd3bc2 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeSimpleTypesTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializeSimpleTypesTests.kt @@ -16,8 +16,8 @@ class DeserializeSimpleTypesTests { private const val VERBOSE = false } - val sf1 = testDefaultFactory() - val sf2 = testDefaultFactory() + val sf1 = testDefaultFactoryNoEvolution() + val sf2 = testDefaultFactoryNoEvolution() @Test fun testChar() { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumTests.kt index 60805c994f..cc46273f08 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumTests.kt @@ -65,7 +65,7 @@ class EnumTests { @Suppress("NOTHING_TO_INLINE") inline private fun classTestName(clazz: String) = "${this.javaClass.name}\$${testName()}\$$clazz" - private val sf1 = testDefaultFactory() + private val sf1 = testDefaultFactoryNoEvolution() @Test fun serialiseSimpleTest() { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index e9ab19db4f..001ea67929 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -3,25 +3,82 @@ package net.corda.nodeapi.internal.serialization.amqp import net.corda.core.serialization.SerializedBytes import net.corda.nodeapi.internal.serialization.AllWhitelist import org.junit.Test +import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals class GenericsTests { + companion object { + val VERBOSE = false + } + + private fun printSeparator() = if(VERBOSE) println ("\n\n-------------------------------------------\n\n") else Unit + + private fun BytesAndSchemas.printSchema() = if (VERBOSE) println ("${this.schema}\n") else Unit + + private fun ConcurrentHashMap>.printKeyToType() { + if (!VERBOSE) return + + forEach { + println ("Key = ${it.key} - ${it.value.type.typeName}") + } + println() + } + + @Test + fun twoDifferntTypesSameParameterizedOuter() { + data class G(val a: A) + + val factory = testDefaultFactoryNoEvolution() + + val bytes1 = SerializationOutput(factory).serializeAndReturnSchema(G("hi")).apply { printSchema() } + + factory.serializersByDescriptor.printKeyToType() + + val bytes2 = SerializationOutput(factory).serializeAndReturnSchema(G(121)).apply { printSchema() } + + factory.serializersByDescriptor.printKeyToType() + + listOf (factory, testDefaultFactory()).forEach { f -> + DeserializationInput(f).deserialize(bytes1.obj).apply { assertEquals("hi", this.a) } + DeserializationInput(f).deserialize(bytes2.obj).apply { assertEquals(121, this.a) } + } + } + + @Test + fun doWeIgnoreMultipleParams() { + data class G1(val a: T) + data class G2(val a: T) + data class Wrapper(val a: Int, val b: G1, val c: G2) + + val factory = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactoryNoEvolution() + + val bytes = SerializationOutput(factory).serializeAndReturnSchema(Wrapper(1, G1("hi"), G2("poop"))).apply { printSchema() } + printSeparator() + DeserializationInput(factory2).deserialize(bytes.obj) + } @Test fun nestedSerializationOfGenerics() { - data class G(val a: T) - data class Wrapper(val a: Int, val b: G) + data class G(val a: T) + data class Wrapper(val a: Int, val b: G) - val factory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) - val altContextFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) + val factory = testDefaultFactoryNoEvolution() + val altContextFactory = testDefaultFactoryNoEvolution() val ser = SerializationOutput(factory) - val bytes = ser.serializeAndReturnSchema(G("hi")) + val bytes = ser.serializeAndReturnSchema(G("hi")).apply { printSchema() } + + factory.serializersByDescriptor.printKeyToType() assertEquals("hi", DeserializationInput(factory).deserialize(bytes.obj).a) assertEquals("hi", DeserializationInput(altContextFactory).deserialize(bytes.obj).a) - val bytes2 = ser.serializeAndReturnSchema(Wrapper(1, G("hi"))) + val bytes2 = ser.serializeAndReturnSchema(Wrapper(1, G("hi"))).apply { printSchema() } + + factory.serializersByDescriptor.printKeyToType() + + printSeparator() DeserializationInput(factory).deserialize(bytes2.obj).apply { assertEquals(1, a) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt index 9871bff1c6..c6b1eeff4d 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt @@ -169,9 +169,11 @@ class SerializationOutputTests { private inline fun serdes(obj: T, factory: SerializerFactory = SerializerFactory( - AllWhitelist, ClassLoader.getSystemClassLoader()), + AllWhitelist, ClassLoader.getSystemClassLoader(), + EvolutionSerializerGetterTesting()), freshDeserializationFactory: SerializerFactory = SerializerFactory( - AllWhitelist, ClassLoader.getSystemClassLoader()), + AllWhitelist, ClassLoader.getSystemClassLoader(), + EvolutionSerializerGetterTesting()), expectedEqual: Boolean = true, expectDeserializedEqual: Boolean = true): T { val ser = SerializationOutput(factory) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializeAndReturnSchemaTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializeAndReturnSchemaTest.kt index acd8c5b8fe..51248a4d4e 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializeAndReturnSchemaTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializeAndReturnSchemaTest.kt @@ -10,7 +10,7 @@ class SerializeAndReturnSchemaTest { @Suppress("NOTHING_TO_INLINE") inline private fun classTestName(clazz: String) = "${this.javaClass.name}\$${testName()}\$$clazz" - val factory = testDefaultFactory() + val factory = testDefaultFactoryNoEvolution() // just a simple test to verify the internal test extension for serialize does // indeed give us the correct schema back. This is more useful in support of other diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/StaticInitialisationOfSerializedObjectTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/StaticInitialisationOfSerializedObjectTest.kt index 2533bdf182..1a85fc784b 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/StaticInitialisationOfSerializedObjectTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/StaticInitialisationOfSerializedObjectTest.kt @@ -45,7 +45,7 @@ class StaticInitialisationOfSerializedObjectTest { } @Test - fun KotlinObjectWithCompanionObject() { + fun kotlinObjectWithCompanionObject() { data class D(val c: C) val sf = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) @@ -104,7 +104,7 @@ class StaticInitialisationOfSerializedObjectTest { override val classCarpenter = ClassCarpenter(ClassLoader.getSystemClassLoader(), wl2) } - // This time have the serilization factory and the carpenter use different whitelists + // This time have the serialization factory and the carpenter use different whitelists @Test fun deserializeTest2() { data class D(val c: C2) From 97793447d50680230313bb27932be6b2432977c8 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Tue, 2 Jan 2018 18:19:20 +0000 Subject: [PATCH 13/17] CORDA-855 - Adding tests for wild card generics Can't actually get something to go through the serializer with a wild card in place as it seems that's an impossible situation * Review Changes * CORDA-855 - Review Comments * Review Comments * Review comments --- .gitignore | 1 + .../amqp/DeserializationInput.kt | 1 - .../serialization/amqp/EvolutionSerializer.kt | 20 +-- .../internal/serialization/amqp/Schema.kt | 38 +++-- .../serialization/amqp/SerializerFactory.kt | 10 +- .../serialization/amqp/TransformsSchema.kt | 2 +- .../serialization/amqp/ErrorMessageTests.java | 7 +- .../serialization/amqp/JavaGenericsTest.java | 93 ++++++++++++ .../amqp/JavaPrivatePropertyTests.java | 8 +- .../amqp/SetterConstructorTests.java | 31 +++- .../amqp/EnumEvolvabilityTests.kt | 22 +-- .../amqp/EvolutionSerializerGetterTesting.kt | 22 +++ .../serialization/amqp/GenericsTests.kt | 133 ++++++++++++++++-- .../amqp/GenericsTests.loadGenericFromFile | Bin 0 -> 265 bytes 14 files changed, 322 insertions(+), 66 deletions(-) create mode 100644 node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaGenericsTest.java create mode 100644 node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializerGetterTesting.kt create mode 100644 node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.loadGenericFromFile diff --git a/.gitignore b/.gitignore index 2bc98f9dfd..05182bed10 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ crashlytics-build.properties # docs related docs/virtualenv/ +virtualenv/ # bft-smart **/config/currentView diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt index e260f5a891..d34ee9c12e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt @@ -54,7 +54,6 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) { inline fun deserialize(bytes: SerializedBytes): T = deserialize(bytes, T::class.java) - @Throws(NotSerializableException::class) inline internal fun deserializeAndReturnEnvelope(bytes: SerializedBytes): ObjectAndEnvelope = deserializeAndReturnEnvelope(bytes, T::class.java) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt index e2d9ce671a..7335db0fa3 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt @@ -153,7 +153,7 @@ class EvolutionSerializerGetter : EvolutionSerializerGetterBase() { typeNotation: TypeNotation, newSerializer: AMQPSerializer, schemas: SerializationSchemas): AMQPSerializer = - factory.serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) { + factory.getSerializersByDescriptor().computeIfAbsent(typeNotation.descriptor.name!!) { when (typeNotation) { is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, factory) is RestrictedType -> EnumEvolutionSerializer.make(typeNotation, newSerializer, factory, schemas) @@ -161,21 +161,3 @@ class EvolutionSerializerGetter : EvolutionSerializerGetterBase() { } } -/** - * An implementation of [EvolutionSerializerGetterBase] that disables all evolution within a - * [SerializerFactory]. This is most useful in testing where it is known that evolution should not be - * occurring and where bugs may be hidden by transparent invocation of an [EvolutionSerializer]. This - * prevents that by simply throwing an exception whenever such a serializer is requested. - */ -class EvolutionSerializerGetterTesting : EvolutionSerializerGetterBase() { - override fun getEvolutionSerializer(factory: SerializerFactory, - typeNotation: TypeNotation, - newSerializer: AMQPSerializer, - schemas: SerializationSchemas): AMQPSerializer { - throw NotSerializableException("No evolution should be occurring\n" + - " ${typeNotation.name}\n" + - " ${typeNotation.descriptor.name}\n" + - " ${newSerializer.type.typeName}\n" + - " ${newSerializer.typeDescriptor}\n\n${schemas.schema}") - } -} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt index dcfdfe9b7a..e79336a731 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt @@ -346,10 +346,11 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact } } -// This method concatentates various elements of the types recursively as unencoded strings into the hasher, effectively +// This method concatenates various elements of the types recursively as unencoded strings into the hasher, effectively // creating a unique string for a type which we then hash in the calling function above. private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet, - hasher: Hasher, factory: SerializerFactory, offset: String = ""): Hasher { + hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher { + // We don't include Example and Example where type is ? or T in this otherwise we // generate different fingerprints for class Outer(val a: Inner) when serialising // and deserializing (assuming deserialization is occurring in a factory that didn't @@ -369,21 +370,21 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta hasher.putUnencodedChars(clazz.name) } else { hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) { - fingerprintForObject(type, type, alreadySeen, hasher, factory, "$offset ") + fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4) } } - // ... and concatentate the type data for each parameter type. + // ... and concatenate the type data for each parameter type. type.actualTypeArguments.fold(startingHash) { orig, paramType -> - fingerprintForType(paramType, type, alreadySeen, orig, factory, "$offset ") + fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4) } } // Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when // looking at A and B from Example (remember we call this function recursively). When // serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but - // when deserializing we only have the wilcard placeholder ?, or AnyType + // when deserializing we only have the wildcard placeholder ?, or AnyType // - // Note, TypeVariable<*> used to be encided as TYPE_VARIABLE_HASH but that again produces a + // Note, TypeVariable<*> used to be encoded as TYPE_VARIABLE_HASH but that again produces a // differing fingerprint on serialisation and deserialization is SerializerFactory.AnyType, is TypeVariable<*> -> { @@ -391,7 +392,8 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta } is Class<*> -> { if (type.isArray) { - fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, "$offset ").putUnencodedChars(ARRAY_HASH) + fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, offset+4) + .putUnencodedChars(ARRAY_HASH) } else if (SerializerFactory.isPrimitive(type)) { hasher.putUnencodedChars(type.name) } else if (isCollectionOrMap(type)) { @@ -410,16 +412,18 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta // to the CorDapp but maybe reference to the JAR in the short term. hasher.putUnencodedChars(type.name) } else { - fingerprintForObject(type, type, alreadySeen, hasher, factory, "$offset ") + fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4) } } } } // Hash the element type + some array hash is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen, - hasher, factory, "$offset ").putUnencodedChars(ARRAY_HASH) + hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH) // TODO: include bounds - is WildcardType -> hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH) + is WildcardType -> { + hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH) + } else -> throw NotSerializableException("Don't know how to hash") } } catch (e: NotSerializableException) { @@ -433,15 +437,21 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta private fun isCollectionOrMap(type: Class<*>) = (Collection::class.java.isAssignableFrom(type) || Map::class.java.isAssignableFrom(type)) && !EnumSet::class.java.isAssignableFrom(type) -private fun fingerprintForObject(type: Type, contextType: Type?, alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory, offset: String = ""): Hasher { +private fun fingerprintForObject( + type: Type, + contextType: Type?, + alreadySeen: MutableSet, + hasher: Hasher, + factory: SerializerFactory, + offset: Int = 0): Hasher { // Hash the class + properties + interfaces val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type") propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory).getters .fold(hasher.putUnencodedChars(name)) { orig, prop -> - fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory, "$offset ") + fingerprintForType(prop.resolvedType, type, alreadySeen, orig, factory, offset+4) .putUnencodedChars(prop.name) .putUnencodedChars(if (prop.mandatory) NOT_NULLABLE_HASH else NULLABLE_HASH) } - interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, "$offset ") } + interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, offset+4) } return hasher } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt index 2a7c6557bb..75f9bf0ded 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt @@ -42,9 +42,9 @@ open class SerializerFactory( cl: ClassLoader, private val evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter()) { private val serializersByType = ConcurrentHashMap>() - val serializersByDescriptor = ConcurrentHashMap>() + private val serializersByDescriptor = ConcurrentHashMap>() private val customSerializers = CopyOnWriteArrayList() - val transformsCache = ConcurrentHashMap>>() + private val transformsCache = ConcurrentHashMap>>() open val classCarpenter = ClassCarpenter(cl, whitelist) @@ -55,6 +55,10 @@ open class SerializerFactory( schemas: SerializationSchemas) = evolutionSerializerGetter.getEvolutionSerializer(this, typeNotation, newSerializer, schemas) + fun getSerializersByDescriptor() = serializersByDescriptor + + fun getTransformsCache() = transformsCache + /** * Look up, and manufacture if necessary, a serializer for the given type. * @@ -332,7 +336,7 @@ open class SerializerFactory( private val namesOfPrimitiveTypes: Map> = primitiveTypeNames.map { it.value to it.key }.toMap() - fun nameForType(type: Type): String = when (type) { + fun nameForType(type: Type): String = when (type) { is Class<*> -> { primitiveTypeName(type) ?: if (type.isArray) { "${nameForType(type.componentType)}${if (type.componentType.isPrimitive) "[p]" else "[]"}" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt index 1d01f91f4f..50c38be9f5 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/TransformsSchema.kt @@ -200,7 +200,7 @@ data class TransformsSchema(val types: Map>(TransformTypes::class.java) try { val clazz = sf.classloader.loadClass(name) diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ErrorMessageTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ErrorMessageTests.java index 258d7ed3b4..7cd21c4f21 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ErrorMessageTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/ErrorMessageTests.java @@ -31,7 +31,12 @@ public class ErrorMessageTests { @Test public void testJavaConstructorAnnotations() { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); + SerializationOutput ser = new SerializationOutput(factory1); Assertions.assertThatThrownBy(() -> ser.serialize(new C(1))) diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaGenericsTest.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaGenericsTest.java new file mode 100644 index 0000000000..feb416e991 --- /dev/null +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaGenericsTest.java @@ -0,0 +1,93 @@ +package net.corda.nodeapi.internal.serialization.amqp; + +import net.corda.core.serialization.SerializedBytes; +import net.corda.nodeapi.internal.serialization.AllWhitelist; +import org.junit.Test; +import java.io.NotSerializableException; + +import static org.jgroups.util.Util.assertEquals; + +public class JavaGenericsTest { + private static class Inner { + private final Integer v; + + private Inner(Integer v) { this.v = v; } + public Integer getV() { return v; } + } + + private static class A { + private final T t; + + private A(T t) { this.t = t; } + public T getT() { return t; } + } + + @Test + public void basicGeneric() throws NotSerializableException { + A a1 = new A(1); + + SerializerFactory factory = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + new EvolutionSerializerGetter()); + + SerializationOutput ser = new SerializationOutput(factory); + SerializedBytes bytes = ser.serialize(a1); + + DeserializationInput des = new DeserializationInput(factory); + A a2 = des.deserialize(bytes, A.class); + + assertEquals(1, a2.getT()); + } + + private SerializedBytes forceWildcardSerialize(A a) throws NotSerializableException { + SerializerFactory factory = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + new EvolutionSerializerGetter()); + + return (new SerializationOutput(factory)).serialize(a); + } + + private SerializedBytes forceWildcardSerializeFactory( + A a, + SerializerFactory factory) throws NotSerializableException { + return (new SerializationOutput(factory)).serialize(a); + } + + private A forceWildcardDeserialize(SerializedBytes bytes) throws NotSerializableException { + SerializerFactory factory = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + new EvolutionSerializerGetter()); + + DeserializationInput des = new DeserializationInput(factory); + return des.deserialize(bytes, A.class); + } + + private A forceWildcardDeserializeFactory( + SerializedBytes bytes, + SerializerFactory factory) throws NotSerializableException { + return (new DeserializationInput(factory)).deserialize(bytes, A.class); + } + + @Test + public void forceWildcard() throws NotSerializableException { + SerializedBytes bytes = forceWildcardSerialize(new A(new Inner(29))); + Inner i = (Inner)forceWildcardDeserialize(bytes).getT(); + assertEquals(29, i.getV()); + } + + @Test + public void forceWildcardSharedFactory() throws NotSerializableException { + SerializerFactory factory = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + new EvolutionSerializerGetter()); + + SerializedBytes bytes = forceWildcardSerializeFactory(new A(new Inner(29)), factory); + Inner i = (Inner)forceWildcardDeserializeFactory(bytes, factory).getT(); + + assertEquals(29, i.getV()); + } +} diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java index 8c495766d0..29f1b949d5 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/JavaPrivatePropertyTests.java @@ -25,7 +25,9 @@ public class JavaPrivatePropertyTests { @Test public void singlePrivateWithConstructor() throws NotSerializableException, NoSuchFieldException, IllegalAccessException { - SerializerFactory factory = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerializerGetter = new EvolutionSerializerGetter(); + SerializerFactory factory = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader(), + evolutionSerializerGetter); SerializationOutput ser = new SerializationOutput(factory); DeserializationInput des = new DeserializationInput(factory); @@ -53,7 +55,9 @@ public class JavaPrivatePropertyTests { @Test public void singlePrivateWithConstructorAndGetter() throws NotSerializableException, NoSuchFieldException, IllegalAccessException { - SerializerFactory factory = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerializerGetter = new EvolutionSerializerGetter(); + SerializerFactory factory = new SerializerFactory(AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), evolutionSerializerGetter); SerializationOutput ser = new SerializationOutput(factory); DeserializationInput des = new DeserializationInput(factory); diff --git a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/SetterConstructorTests.java b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/SetterConstructorTests.java index eefdd0957b..bb710a841e 100644 --- a/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/SetterConstructorTests.java +++ b/node-api/src/test/java/net/corda/nodeapi/internal/serialization/amqp/SetterConstructorTests.java @@ -109,7 +109,12 @@ public class SetterConstructorTests { // despite having no constructor we should still be able to serialise an instance of C @Test public void serialiseC() throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); + SerializationOutput ser = new SerializationOutput(factory1); C c1 = new C(); @@ -178,7 +183,11 @@ public class SetterConstructorTests { @Test public void deserialiseC() throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); C cPre1 = new C(); @@ -241,7 +250,11 @@ public class SetterConstructorTests { @Test public void serialiseOuterAndInner() throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); Inner1 i1 = new Inner1("Hello"); Inner2 i2 = new Inner2(); @@ -263,7 +276,11 @@ public class SetterConstructorTests { @Test public void typeMistmatch() throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); TypeMismatch tm = new TypeMismatch(); tm.setA(10); @@ -279,7 +296,11 @@ public class SetterConstructorTests { @Test public void typeMistmatch2() throws NotSerializableException { - SerializerFactory factory1 = new SerializerFactory(AllWhitelist.INSTANCE, ClassLoader.getSystemClassLoader()); + EvolutionSerializerGetterBase evolutionSerialiserGetter = new EvolutionSerializerGetter(); + SerializerFactory factory1 = new SerializerFactory( + AllWhitelist.INSTANCE, + ClassLoader.getSystemClassLoader(), + evolutionSerialiserGetter); TypeMismatch2 tm = new TypeMismatch2(); tm.setA("10"); diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolvabilityTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolvabilityTests.kt index d18b970cfa..2769c31798 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolvabilityTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolvabilityTests.kt @@ -6,6 +6,8 @@ import org.assertj.core.api.Assertions import org.junit.Test import java.io.File import java.io.NotSerializableException +import java.util.* +import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -387,21 +389,25 @@ class EnumEvolvabilityTests { data class C1(val annotatedEnum: AnnotatedEnumOnce) val sf = testDefaultFactory() + val f = sf.javaClass.getDeclaredField("transformsCache") + f.isAccessible = true - assertEquals(0, sf.transformsCache.size) + val transformsCache = f.get(sf) as ConcurrentHashMap>> + + assertEquals(0, transformsCache.size) val sb1 = TestSerializationOutput(VERBOSE, sf).serializeAndReturnSchema(C1(AnnotatedEnumOnce.D)) - assertEquals(2, sf.transformsCache.size) - assertTrue(sf.transformsCache.containsKey(C1::class.java.name)) - assertTrue(sf.transformsCache.containsKey(AnnotatedEnumOnce::class.java.name)) + assertEquals(2, transformsCache.size) + assertTrue(transformsCache.containsKey(C1::class.java.name)) + assertTrue(transformsCache.containsKey(AnnotatedEnumOnce::class.java.name)) val sb2 = TestSerializationOutput(VERBOSE, sf).serializeAndReturnSchema(C2(AnnotatedEnumOnce.D)) - assertEquals(3, sf.transformsCache.size) - assertTrue(sf.transformsCache.containsKey(C1::class.java.name)) - assertTrue(sf.transformsCache.containsKey(C2::class.java.name)) - assertTrue(sf.transformsCache.containsKey(AnnotatedEnumOnce::class.java.name)) + assertEquals(3, transformsCache.size) + assertTrue(transformsCache.containsKey(C1::class.java.name)) + assertTrue(transformsCache.containsKey(C2::class.java.name)) + assertTrue(transformsCache.containsKey(AnnotatedEnumOnce::class.java.name)) assertEquals(sb1.transformsSchema.types[AnnotatedEnumOnce::class.java.name], sb2.transformsSchema.types[AnnotatedEnumOnce::class.java.name]) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializerGetterTesting.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializerGetterTesting.kt new file mode 100644 index 0000000000..fc0d636c21 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializerGetterTesting.kt @@ -0,0 +1,22 @@ +package net.corda.nodeapi.internal.serialization.amqp + +import java.io.NotSerializableException + +/** + * An implementation of [EvolutionSerializerGetterBase] that disables all evolution within a + * [SerializerFactory]. This is most useful in testing where it is known that evolution should not be + * occurring and where bugs may be hidden by transparent invocation of an [EvolutionSerializer]. This + * prevents that by simply throwing an exception whenever such a serializer is requested. + */ +class EvolutionSerializerGetterTesting : EvolutionSerializerGetterBase() { + override fun getEvolutionSerializer(factory: SerializerFactory, + typeNotation: TypeNotation, + newSerializer: AMQPSerializer, + schemas: SerializationSchemas): AMQPSerializer { + throw NotSerializableException("No evolution should be occurring\n" + + " ${typeNotation.name}\n" + + " ${typeNotation.descriptor.name}\n" + + " ${newSerializer.type.typeName}\n" + + " ${newSerializer.typeDescriptor}\n\n${schemas.schema}") + } +} diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index 001ea67929..bd858bb103 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -2,43 +2,50 @@ package net.corda.nodeapi.internal.serialization.amqp import net.corda.core.serialization.SerializedBytes import net.corda.nodeapi.internal.serialization.AllWhitelist +import net.corda.testing.common.internal.ProjectStructure.projectRootDir import org.junit.Test +import java.io.File +import java.net.URI import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals class GenericsTests { companion object { val VERBOSE = false + + @Suppress("UNUSED") + var localPath = projectRootDir.toUri().resolve( + "node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp") } - private fun printSeparator() = if(VERBOSE) println ("\n\n-------------------------------------------\n\n") else Unit + private fun printSeparator() = if (VERBOSE) println("\n\n-------------------------------------------\n\n") else Unit - private fun BytesAndSchemas.printSchema() = if (VERBOSE) println ("${this.schema}\n") else Unit + private fun BytesAndSchemas.printSchema() = if (VERBOSE) println("${this.schema}\n") else Unit private fun ConcurrentHashMap>.printKeyToType() { if (!VERBOSE) return forEach { - println ("Key = ${it.key} - ${it.value.type.typeName}") + println("Key = ${it.key} - ${it.value.type.typeName}") } println() } @Test - fun twoDifferntTypesSameParameterizedOuter() { + fun twoDifferentTypesSameParameterizedOuter() { data class G(val a: A) val factory = testDefaultFactoryNoEvolution() val bytes1 = SerializationOutput(factory).serializeAndReturnSchema(G("hi")).apply { printSchema() } - factory.serializersByDescriptor.printKeyToType() + factory.getSerializersByDescriptor().printKeyToType() val bytes2 = SerializationOutput(factory).serializeAndReturnSchema(G(121)).apply { printSchema() } - factory.serializersByDescriptor.printKeyToType() + factory.getSerializersByDescriptor().printKeyToType() - listOf (factory, testDefaultFactory()).forEach { f -> + listOf(factory, testDefaultFactory()).forEach { f -> DeserializationInput(f).deserialize(bytes1.obj).apply { assertEquals("hi", this.a) } DeserializationInput(f).deserialize(bytes2.obj).apply { assertEquals(121, this.a) } } @@ -69,14 +76,14 @@ class GenericsTests { val bytes = ser.serializeAndReturnSchema(G("hi")).apply { printSchema() } - factory.serializersByDescriptor.printKeyToType() + factory.getSerializersByDescriptor().printKeyToType() assertEquals("hi", DeserializationInput(factory).deserialize(bytes.obj).a) assertEquals("hi", DeserializationInput(altContextFactory).deserialize(bytes.obj).a) val bytes2 = ser.serializeAndReturnSchema(Wrapper(1, G("hi"))).apply { printSchema() } - factory.serializersByDescriptor.printKeyToType() + factory.getSerializersByDescriptor().printKeyToType() printSeparator() @@ -93,7 +100,7 @@ class GenericsTests { @Test fun nestedGenericsReferencesByteArrayViaSerializedBytes() { - data class G(val a : Int) + data class G(val a: Int) data class Wrapper(val a: Int, val b: SerializedBytes) val factory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) @@ -128,27 +135,30 @@ class GenericsTests { ser.serialize(Wrapper(Container(InnerA(1)))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals(1, c.b.a_a) } + it.getSerializersByDescriptor().printKeyToType(); printSeparator() } } ser.serialize(Wrapper(Container(InnerB(1)))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals(1, c.b.a_b) } + it.getSerializersByDescriptor().printKeyToType(); printSeparator() } } ser.serialize(Wrapper(Container(InnerC("Ho ho ho")))).apply { factories.forEach { DeserializationInput(it).deserialize(this).apply { assertEquals("Ho ho ho", c.b.a_c) } + it.getSerializersByDescriptor().printKeyToType(); printSeparator() } } } @Test fun nestedSerializationWhereGenericDoesntImpactFingerprint() { - data class Inner(val a : Int) + data class Inner(val a: Int) data class Container(val b: Inner) - data class Wrapper(val c: Container) + data class Wrapper(val c: Container) val factorys = listOf( SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()), @@ -168,4 +178,103 @@ class GenericsTests { } } } + + data class ForceWildcard(val t: T) + + private fun forceWildcardSerialize( + a: ForceWildcard<*>, + factory: SerializerFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())): SerializedBytes<*> { + val bytes = SerializationOutput(factory).serializeAndReturnSchema(a) + factory.getSerializersByDescriptor().printKeyToType() + bytes.printSchema() + return bytes.obj + } + + @Suppress("UNCHECKED_CAST") + private fun forceWildcardDeserializeString( + bytes: SerializedBytes<*>, + factory: SerializerFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())) { + DeserializationInput(factory).deserialize(bytes as SerializedBytes>) + } + + @Suppress("UNCHECKED_CAST") + private fun forceWildcardDeserializeDouble( + bytes: SerializedBytes<*>, + factory: SerializerFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())) { + DeserializationInput(factory).deserialize(bytes as SerializedBytes>) + } + + @Suppress("UNCHECKED_CAST") + private fun forceWildcardDeserialize( + bytes: SerializedBytes<*>, + factory: SerializerFactory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())) { + DeserializationInput(factory).deserialize(bytes as SerializedBytes>) + } + + @Test + fun forceWildcard() { + forceWildcardDeserializeString(forceWildcardSerialize(ForceWildcard("hello"))) + forceWildcardDeserializeDouble(forceWildcardSerialize(ForceWildcard(3.0))) + } + + @Test + fun forceWildcardSharedFactory() { + val f = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) + forceWildcardDeserializeString(forceWildcardSerialize(ForceWildcard("hello"), f), f) + forceWildcardDeserializeDouble(forceWildcardSerialize(ForceWildcard(3.0), f), f) + } + + @Test + fun forceWildcardDeserialize() { + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard("hello"))) + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard(10))) + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard(20.0))) + } + + @Test + fun forceWildcardDeserializeSharedFactory() { + val f = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard("hello"), f), f) + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard(10), f), f) + forceWildcardDeserialize(forceWildcardSerialize(ForceWildcard(20.0), f), f) + } + + @Test + fun loadGenericFromFile() { + val resource = "${javaClass.simpleName}.${testName()}" + val sf = testDefaultFactory() + + // Uncomment to re-generate test files, needs to be done in three stages + // File(URI("$localPath/$resource")).writeBytes(forceWildcardSerialize(ForceWildcard("wibble")).bytes) + + assertEquals("wibble", + DeserializationInput(sf).deserialize(SerializedBytes>( + File(GenericsTests::class.java.getResource(resource).toURI()).readBytes())).t) + } + + interface DifferentBounds { + fun go() + } + + @Test + fun differentBounds() { + data class A (val a: Int): DifferentBounds { + override fun go() { + println(a) + } + } + + data class G(val b: T) + + val factorys = listOf( + SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()), + SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())) + + val ser = SerializationOutput(factorys[0]) + + ser.serialize(G(A(10))).apply { + factorys.forEach { + } + } + } } diff --git a/node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.loadGenericFromFile b/node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.loadGenericFromFile new file mode 100644 index 0000000000000000000000000000000000000000..5c467153f860402ddf0ce2b596ee7869a97d8a51 GIT binary patch literal 265 zcmYe!FG@*dWME)uIGO|`85kH3d}U@>tdy5pqL&PkvvM&>ba4vJb;-#KOxG?c%5?Ou zaMupBwLQSexR9+pGbt%26|R}-z-q<=D;VK?)&sLy7ka|Y(#y+FNlh%s)XU5(NiE7t z%+V`OEy_&H$*f8&$;{8wOUx}S&~s1C0}3V=holyl6sx%97bT~LXXd0NCl;kRxWY|l lKA^@#j4K@uNFtoWa)5_@A!ErxMlDwd2S+EkVVnmU82~ZBQoH~F literal 0 HcmV?d00001 From 8b5ad9d5c0e351638087bd4db165a178194e052f Mon Sep 17 00:00:00 2001 From: igor nitto Date: Thu, 11 Jan 2018 10:45:33 +0000 Subject: [PATCH 14/17] Remove unused class DefaultCordaRpcPermissions (#2328) --- .../internal/DefaultCordaRpcPermissions.kt | 25 ------------------- .../node/internal/RpcAuthorisationProxy.kt | 2 +- .../corda/node/internal/SecureCordaRPCOps.kt | 2 +- .../security/RPCSecurityManagerImpl.kt | 9 +++---- 4 files changed, 6 insertions(+), 32 deletions(-) delete mode 100644 node/src/main/kotlin/net/corda/node/internal/DefaultCordaRpcPermissions.kt diff --git a/node/src/main/kotlin/net/corda/node/internal/DefaultCordaRpcPermissions.kt b/node/src/main/kotlin/net/corda/node/internal/DefaultCordaRpcPermissions.kt deleted file mode 100644 index dccec894e7..0000000000 --- a/node/src/main/kotlin/net/corda/node/internal/DefaultCordaRpcPermissions.kt +++ /dev/null @@ -1,25 +0,0 @@ -package net.corda.node.internal - -import net.corda.core.flows.FlowLogic -import net.corda.core.messaging.CordaRPCOps -import net.corda.node.services.Permissions.Companion.all -import net.corda.node.services.Permissions.Companion.startFlow -import net.corda.node.services.Permissions.Companion.invokeRpc -import kotlin.reflect.KVisibility -import kotlin.reflect.full.declaredMemberFunctions - -object DefaultCordaRpcPermissions { - - private val invokePermissions = CordaRPCOps::class.declaredMemberFunctions.filter { it.visibility == KVisibility.PUBLIC }.associate { it.name to setOf(invokeRpc(it), all()) } - private val startFlowPermissions = setOf("startFlow", "startFlowDynamic", "startTrackedFlow", "startTrackedFlowDynamic").associate { it to this::startFlowPermission } - - fun permissionsAllowing(methodName: String, args: List): Set { - - val invoke = invokePermissions[methodName] ?: emptySet() - val start = startFlowPermissions[methodName]?.invoke(args) - return if (start != null) invoke + start else invoke - } - - @Suppress("UNCHECKED_CAST") - private fun startFlowPermission(args: List): String = if (args[0] is Class<*>) startFlow(args[0] as Class>) else startFlow(args[0] as String) -} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/internal/RpcAuthorisationProxy.kt b/node/src/main/kotlin/net/corda/node/internal/RpcAuthorisationProxy.kt index d2f0f8afc1..1264935c26 100644 --- a/node/src/main/kotlin/net/corda/node/internal/RpcAuthorisationProxy.kt +++ b/node/src/main/kotlin/net/corda/node/internal/RpcAuthorisationProxy.kt @@ -19,7 +19,7 @@ import java.io.InputStream import java.security.PublicKey // TODO change to KFunction reference after Kotlin fixes https://youtrack.jetbrains.com/issue/KT-12140 -class RpcAuthorisationProxy(private val implementation: CordaRPCOps, private val context: () -> RpcAuthContext, private val permissionsAllowing: (methodName: String, args: List) -> Set) : CordaRPCOps { +class RpcAuthorisationProxy(private val implementation: CordaRPCOps, private val context: () -> RpcAuthContext) : CordaRPCOps { override fun uploadAttachmentWithMetadata(jar: InputStream, uploader: String, filename: String): SecureHash = guard("uploadAttachmentWithMetadata") { implementation.uploadAttachmentWithMetadata(jar, uploader, filename) diff --git a/node/src/main/kotlin/net/corda/node/internal/SecureCordaRPCOps.kt b/node/src/main/kotlin/net/corda/node/internal/SecureCordaRPCOps.kt index 8394c35ae9..0a7d9e9ce1 100644 --- a/node/src/main/kotlin/net/corda/node/internal/SecureCordaRPCOps.kt +++ b/node/src/main/kotlin/net/corda/node/internal/SecureCordaRPCOps.kt @@ -14,7 +14,7 @@ class SecureCordaRPCOps(services: ServiceHubInternal, smm: StateMachineManager, database: CordaPersistence, flowStarter: FlowStarter, - val unsafe: CordaRPCOps = CordaRPCOpsImpl(services, smm, database, flowStarter)) : CordaRPCOps by RpcAuthorisationProxy(unsafe, ::rpcContext, DefaultCordaRpcPermissions::permissionsAllowing) { + val unsafe: CordaRPCOps = CordaRPCOpsImpl(services, smm, database, flowStarter)) : CordaRPCOps by RpcAuthorisationProxy(unsafe, ::rpcContext) { /** * Returns the RPC protocol version, which is the same the node's Platform Version. Exists since version 1 so guaranteed diff --git a/node/src/main/kotlin/net/corda/node/internal/security/RPCSecurityManagerImpl.kt b/node/src/main/kotlin/net/corda/node/internal/security/RPCSecurityManagerImpl.kt index e667d673b7..2237cc1547 100644 --- a/node/src/main/kotlin/net/corda/node/internal/security/RPCSecurityManagerImpl.kt +++ b/node/src/main/kotlin/net/corda/node/internal/security/RPCSecurityManagerImpl.kt @@ -103,7 +103,7 @@ class RPCSecurityManagerImpl(config: AuthServiceConfig) : RPCSecurityManager { } } -/** +/* * Provide a representation of RPC permissions based on Apache Shiro permissions framework. * A permission represents a set of actions: for example, the set of all RPC invocations, or the set * of RPC invocations acting on a given class of Flows in input. A permission `implies` another one if @@ -128,7 +128,7 @@ private class RPCPermission : DomainPermission { constructor() : super() } -/** +/* * A [org.apache.shiro.authz.permission.PermissionResolver] implementation for RPC permissions. * Provides a method to construct an [RPCPermission] instance from its string representation * in the form used by a Node admin. @@ -141,7 +141,6 @@ private class RPCPermission : DomainPermission { * * - `StartFlow.$FlowClassName`: allowing to call a `startFlow*` RPC method targeting a Flow instance * of a given class - * */ private object RPCPermissionResolver : PermissionResolver { @@ -253,7 +252,7 @@ private class NodeJdbcRealm(config: SecurityConfiguration.AuthService.DataSource private typealias ShiroCache = org.apache.shiro.cache.Cache -/** +/* * Adapts a [com.google.common.cache.Cache] to a [org.apache.shiro.cache.Cache] implementation. */ private fun Cache.toShiroCache(name: String) = object : ShiroCache { @@ -285,7 +284,7 @@ private fun Cache.toShiroCache(name: String) = object : ShiroCache< override fun toString() = "Guava cache adapter [$impl]" } -/** +/* * Implementation of [org.apache.shiro.cache.CacheManager] based on * cache implementation in [com.google.common.cache] */ From 0a56c75543139f0a5b54dc696215549f03382b4c Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Wed, 10 Jan 2018 17:06:54 +0000 Subject: [PATCH 15/17] Minor: AuthDBTests - close RPC connections to prevent thread leaks --- .../kotlin/net/corda/node/AuthDBTests.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt b/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt index 4015265da8..50689e84f5 100644 --- a/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt @@ -97,21 +97,21 @@ class AuthDBTests : NodeBasedTest() { @Test fun `login with correct credentials`() { - client.start("user", "foo") + client.start("user", "foo").close() } @Test fun `login with wrong credentials`() { - client.start("user", "foo") + client.start("user", "foo").close() assertFailsWith( ActiveMQSecurityException::class, "Login with incorrect password should fail") { - client.start("user", "bar") + client.start("user", "bar").close() } assertFailsWith( ActiveMQSecurityException::class, "Login with unknown username should fail") { - client.start("X", "foo") + client.start("X", "foo").close() } } @@ -153,7 +153,7 @@ class AuthDBTests : NodeBasedTest() { assertFailsWith( ActiveMQSecurityException::class, "Login with incorrect password should fail") { - client.start("user2", "bar") + client.start("user2", "bar").close() } db.insert(UserAndRoles( @@ -161,7 +161,7 @@ class AuthDBTests : NodeBasedTest() { password = encodePassword("bar"), roles = listOf("default"))) - client.start("user2", "bar") + client.start("user2", "bar").close() } @Test From bbfbb08c432b6eca1115ec4f2c25c836a159c48b Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Fri, 12 Jan 2018 07:59:08 +0000 Subject: [PATCH 16/17] CORDA-881: Signed network parameters has the network map cert attached to it instead of just the public key. (#2346) Introduced DigitalSignatureWithCert and SignedDataWithCert as internal APIs, with the expectation that they will become public; renamed the network parameters end-point to network-parameters; updated the network-map.rst doc; and did some refactoring. --- .gitignore | 1 + .../kotlin/net/corda/core/crypto/Crypto.kt | 2 +- .../core/internal/DigitalSignatureWithCert.kt | 27 ++++ .../net/corda/core/internal/InternalUtils.kt | 24 ++++ docs/source/network-map.rst | 126 +++++++++--------- .../nodeapi/internal/DevIdentityGenerator.kt | 17 +-- .../nodeapi/internal/KeyStoreConfigHelpers.kt | 31 ++++- .../corda/nodeapi/internal/SignedNodeInfo.kt | 2 + .../nodeapi/internal/network/NetworkMap.kt | 38 +----- .../network/NetworkParametersCopier.kt | 23 ++-- .../internal/crypto/X509UtilitiesTest.kt | 4 +- .../node/services/network/NetworkMapTest.kt | 8 +- .../registration/NodeRegistrationTest.kt | 6 +- .../messaging/MQSecurityAsNodeTest.kt | 24 ++-- .../net/corda/node/internal/AbstractNode.kt | 20 ++- .../node/services/config/ConfigUtilities.kt | 5 +- .../node/services/network/NetworkMapClient.kt | 47 +++---- .../HTTPNetworkRegistrationService.kt | 4 +- .../services/network/NetworkMapClientTest.kt | 6 +- .../node/internal/network/NetworkMapServer.kt | 67 +++------- .../kotlin/net/corda/testing/TestConstants.kt | 16 +-- 21 files changed, 236 insertions(+), 262 deletions(-) create mode 100644 core/src/main/kotlin/net/corda/core/internal/DigitalSignatureWithCert.kt diff --git a/.gitignore b/.gitignore index 05182bed10..f267b20469 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ lib/quasar.jar .idea/markdown-navigator .idea/runConfigurations .idea/dictionaries +.idea/codeStyles/ /gradle-plugins/.idea/ # Include the -parameters compiler option by default in IntelliJ required for serialization. diff --git a/core/src/main/kotlin/net/corda/core/crypto/Crypto.kt b/core/src/main/kotlin/net/corda/core/crypto/Crypto.kt index 1657eb2d4a..ef365c7a4f 100644 --- a/core/src/main/kotlin/net/corda/core/crypto/Crypto.kt +++ b/core/src/main/kotlin/net/corda/core/crypto/Crypto.kt @@ -407,7 +407,7 @@ object Crypto { */ @JvmStatic @Throws(InvalidKeyException::class, SignatureException::class) - fun doSign(privateKey: PrivateKey, clearData: ByteArray) = doSign(findSignatureScheme(privateKey), privateKey, clearData) + fun doSign(privateKey: PrivateKey, clearData: ByteArray): ByteArray = doSign(findSignatureScheme(privateKey), privateKey, clearData) /** * Generic way to sign [ByteArray] data with a [PrivateKey] and a known schemeCodeName [String]. diff --git a/core/src/main/kotlin/net/corda/core/internal/DigitalSignatureWithCert.kt b/core/src/main/kotlin/net/corda/core/internal/DigitalSignatureWithCert.kt new file mode 100644 index 0000000000..04ca25c6cb --- /dev/null +++ b/core/src/main/kotlin/net/corda/core/internal/DigitalSignatureWithCert.kt @@ -0,0 +1,27 @@ +package net.corda.core.internal + +import net.corda.core.crypto.DigitalSignature +import net.corda.core.crypto.SignedData +import net.corda.core.crypto.verify +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.deserialize +import net.corda.core.utilities.OpaqueBytes +import java.security.cert.X509Certificate + +// TODO: Rename this to DigitalSignature.WithCert once we're happy for it to be public API. The methods will need documentation +// and the correct exceptions will be need to be annotated +/** A digital signature with attached certificate of the public key. */ +class DigitalSignatureWithCert(val by: X509Certificate, bytes: ByteArray) : DigitalSignature(bytes) { + fun verify(content: ByteArray): Boolean = by.publicKey.verify(content, this) + fun verify(content: OpaqueBytes): Boolean = verify(content.bytes) +} + +/** Similar to [SignedData] but instead of just attaching the public key, the certificate for the key is attached instead. */ +@CordaSerializable +class SignedDataWithCert(val raw: SerializedBytes, val sig: DigitalSignatureWithCert) { + fun verified(): T { + sig.verify(raw) + return uncheckedCast(raw.deserialize()) + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt index 1ad9e16630..70724cb884 100644 --- a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt @@ -3,11 +3,14 @@ package net.corda.core.internal import net.corda.core.cordapp.CordappProvider +import net.corda.core.crypto.Crypto import net.corda.core.crypto.SecureHash import net.corda.core.crypto.sha256 import net.corda.core.identity.CordaX500Name import net.corda.core.node.ServicesForResolution import net.corda.core.serialization.SerializationContext +import net.corda.core.serialization.deserialize +import net.corda.core.serialization.serialize import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction import org.bouncycastle.asn1.x500.X500Name @@ -27,6 +30,8 @@ import java.nio.charset.Charset import java.nio.charset.StandardCharsets.UTF_8 import java.nio.file.* import java.nio.file.attribute.FileAttribute +import java.security.PrivateKey +import java.security.cert.X509Certificate import java.time.Duration import java.time.temporal.Temporal import java.util.* @@ -308,6 +313,19 @@ fun TransactionBuilder.toLedgerTransaction(services: ServicesForResolution, seri val KClass<*>.packageName: String get() = java.`package`.name fun URL.openHttpConnection(): HttpURLConnection = openConnection() as HttpURLConnection + +fun HttpURLConnection.checkOkResponse() { + if (responseCode != 200) { + val message = errorStream.use { it.reader().readText() } + throw IOException("Response Code $responseCode: $message") + } +} + +inline fun HttpURLConnection.responseAs(): T { + checkOkResponse() + return inputStream.use { it.readBytes() }.deserialize() +} + /** Analogous to [Thread.join]. */ fun ExecutorService.join() { shutdown() // Do not change to shutdownNow, tests use this method to assert the executor has no more tasks. @@ -336,3 +354,9 @@ val CordaX500Name.x500Name: X500Name @VisibleForTesting val CordaX500Name.Companion.unspecifiedCountry get() = "ZZ" + +fun T.signWithCert(privateKey: PrivateKey, certificate: X509Certificate): SignedDataWithCert { + val serialised = serialize() + val signature = Crypto.doSign(privateKey, serialised.bytes) + return SignedDataWithCert(serialised, DigitalSignatureWithCert(certificate, signature)) +} diff --git a/docs/source/network-map.rst b/docs/source/network-map.rst index efe8110843..f6eca336f4 100644 --- a/docs/source/network-map.rst +++ b/docs/source/network-map.rst @@ -1,84 +1,78 @@ Network Map =========== -The network map stores a collection of ``NodeInfo`` objects, each representing another node with which the node can interact. -There are two sources from which a Corda node can retrieve ``NodeInfo`` objects: +The network map is a collection of signed ``NodeInfo`` objects (signed by the node it represents and thus tamper-proof) +forming the set of reachable nodes in a compatbility zone. A node can receive these objects from two sources: -1. the REST protocol with the network map service, which also provides a publishing API, +1. The HTTP network map service if the ``compatibilityZoneURL`` config key is specified. +2. The ``additional-node-infos`` directory within the node's directory. -2. the ``additional-node-infos`` directory. +HTTP network map service +------------------------ +If the node is configured with the ``compatibilityZoneURL`` config then it first uploads its own signed ``NodeInfo`` +to the server (and each time it changes on startup) and then proceeds to download the entire network map. The network map +consists of a list of ``NodeInfo`` hashes. The node periodically polls for the network map (based on the HTTP cache expiry +header) and any new hash entries are downloaded and cached. Entries which no longer exist are deleted from the node's cache. -Protocol Design ---------------- -The node info publishing protocol: +The set of REST end-points for the network map service are as follows. -* Create a ``NodeInfo`` object, and sign it to create a ``SignedNodeInfo`` object. - -* Serialise the signed data and POST the data to the network map server. - -* The network map server validates the signature and acknowledges the registration with a HTTP 200 response, it will return HTTP 400 "Bad Request" if the data failed validation or if the public key wasn't registered with the network. - -* The network map server will sign and distribute the new network map periodically. - -Node side network map update protocol: - -* The Corda node will query the network map service periodically according to the ``Expires`` attribute in the HTTP header. - -* The network map service returns a signed ``NetworkMap`` object which looks as follows: - -.. container:: codeset - - .. sourcecode:: kotlin - - data class NetworkMap { - val nodeInfoHashes: List, - val networkParametersHash: SecureHash - } - -The object contains list of node info hashes and hash of the network parameters data structure (without the signatures). - -* The node updates its local copy of ``NodeInfos`` if it is different from the newly downloaded ``NetworkMap``. - -Network Map service REST API: - -+----------------+-----------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Request method | Path | Description | -+================+===================================+========================================================================================================================================================+ -| POST | /network-map/publish | Publish new ``NodeInfo`` to the network map service, the legal identity in ``NodeInfo`` must match with the identity registered with the doorman. | -+----------------+-----------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| GET | /network-map | Retrieve ``NetworkMap`` from the server, the ``NetworkMap`` object contains list of node info hashes and ``NetworkParameters`` hash. | -+----------------+-----------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| GET | /network-map/node-info/{hash} | Retrieve ``NodeInfo`` object with the same hash. | -+----------------+-----------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ -| GET | /network-map/parameters/{hash} | Retrieve ``NetworkParameters`` object with the same hash. | -+----------------+-----------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------+ - -TODO: Access control of the network map will be added in the future. ++----------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| Request method | Path | Description | ++================+=========================================+==============================================================================================================================================+ +| POST | /network-map/publish | For the node to upload its signed ``NodeInfo`` object to the network map. | ++----------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| GET | /network-map | Retrieve the current signed network map object. The entire object is signed with the network map certificate which is also attached. | ++----------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| GET | /network-map/node-info/{hash} | Retrieve a signed ``NodeInfo`` as specified in the network map object. | ++----------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ +| GET | /network-map/network-parameters/{hash} | Retrieve the signed network parameters (see below). The entire object is signed with the network map certificate which is also attached. | ++----------------+-----------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+ The ``additional-node-infos`` directory --------------------------------------- -Each Corda node reads, and continuously polls, the files contained in a directory named ``additional-node-infos`` inside the node base directory. -Nodes expect to find a serialized ``SignedNodeInfo`` object, the same object which is sent to network map server. - -Whenever a node starts it writes on disk a file containing its own ``NodeInfo``, this file is called ``nodeInfo-XXX`` where ``XXX`` is a long string. - -Hence if an operator wants node A to see node B they can pick B's ``NodeInfo`` file from B base directory and drop it into A's ``additional-node-infos`` directory. +Alongside the HTTP network map service, or as a replacement if the node isn't connected to one, the node polls the +contents of the ``additional-node-infos`` directory located in its base directory. Each file is expected to be the same +signed ``NodeInfo`` object that the network map service vends. These are automtically added to the node's cache and can +be used to supplement or replace the HTTP network map. If the same node is advertised through both mechanisms then the +latest one is taken. +On startup the node generates its own signed node info file, filename of the format ``nodeInfo-${hash}``. To create a simple +network without the HTTP network map service then simply place this file in the ``additional-node-infos`` directory +of every node that's part of this network. Network parameters ------------------ -Network parameters are constants that every node participating in the network needs to agree on and use for interop purposes. -The structure is distributed as a file containing serialized ``SignedData`` with a signature from -a sub-key of the compatibility zone root cert. Network map advertises the hash of currently used network parameters. -The ``NetworkParameters`` structure contains: - * ``minimumPlatformVersion`` - minimum version of Corda platform that is required for nodes in the network. - * ``notaries`` - list of well known and trusted notary identities with information on validation type. - * ``maxMessageSize`` - maximum P2P message size sent over the wire in bytes. - * ``maxTransactionSize`` - maximum permitted transaction size in bytes. - * ``modifiedTime`` - the time the network parameters were created by the CZ operator. - * ``epoch`` - version number of the network parameters. Starting from 1, this will always increment on each new set of parameters. -The set of parameters is still under development and we may find the need to add additional fields. +Network parameters are a set of values that every node participating in the network needs to agree on and use to +correctly interoperate with each other. If the node is using the HTTP network map service then on first startup it will +download the signed network parameters, cache it in a ``network-parameters`` file and apply them on the node. + +.. warning:: If the ``network-parameters`` file is changed and no longer matches what the network map service is advertising + then the node will automatically shutdown. Resolution to this is to delete the incorrect file and restart the node so + that the parameters can be downloaded again. + +.. note:: A future release will support the notion of network parameters changes. + +If the node isn't using a HTTP network map service then it's expected the signed file is provided by some other means. +For such a scenario there is the network bootstrapper tool which in addition to generating the network parameters file +also distributes the node info files to the node directories. More information can be found in :doc:`setting-up-a-corda-network`. + +The current set of network parameters: + +:minimumPlatformVersion: The minimum platform version that the nodes must be running. Any node which is below this will + not start. +:notaries: List of identity and validation type (either validating or non-validating) of the notaries which are permitted + in the compatibility zone. +:maxMessageSize: Maximum allowed P2P message size sent over the wire in bytes. Any message larger than this will be + split up. +:maxTransactionSize: Maximum permitted transaction size in bytes. +:modifiedTime: The time when the network parameters were last modified by the compatibility zone operator. +:epoch: Version number of the network parameters. Starting from 1, this will always increment whenever any of the + parameters change. + +.. note:: ``maxTransactionSize`` is currently not enforced in the node, but will be in a later release. + +More parameters may be added in future releases. diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt index 0cb8472f89..9ccc286f4f 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/DevIdentityGenerator.kt @@ -34,13 +34,8 @@ object DevIdentityGenerator { override val trustStorePassword get() = throw NotImplementedError("Not expected to be called") } - // TODO The passwords for the dev key stores are spread everywhere and should be constants in a single location - val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) - nodeSslConfig.certificatesDirectory.createDirectories() - nodeSslConfig.createDevKeyStores(rootCert, intermediateCa, legalName) + nodeSslConfig.createDevKeyStores(legalName) val keyStoreWrapper = KeyStoreWrapper(nodeSslConfig.nodeKeystore, nodeSslConfig.keyStorePassword) val identity = keyStoreWrapper.storeLegalIdentity(legalName, "$NODE_IDENTITY_ALIAS_PREFIX-private-key", Crypto.generateKeyPair()) @@ -54,16 +49,12 @@ object DevIdentityGenerator { val keyPairs = (1..dirs.size).map { generateKeyPair() } val compositeKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public }).build(threshold) - val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val rootCert = caKeyStore.getCertificate(X509Utilities.CORDA_ROOT_CA) - keyPairs.zip(dirs) { keyPair, nodeDir -> val (serviceKeyCert, compositeKeyCert) = listOf(keyPair.public, compositeKey).map { publicKey -> X509Utilities.createCertificate( CertificateType.SERVICE_IDENTITY, - intermediateCa.certificate, - intermediateCa.keyPair, + DEV_INTERMEDIATE_CA.certificate, + DEV_INTERMEDIATE_CA.keyPair, notaryName.x500Principal, publicKey) } @@ -74,7 +65,7 @@ object DevIdentityGenerator { "$DISTRIBUTED_NOTARY_ALIAS_PREFIX-private-key", keyPair.private, "cordacadevkeypass".toCharArray(), - arrayOf(serviceKeyCert, intermediateCa.certificate, rootCert)) + arrayOf(serviceKeyCert, DEV_INTERMEDIATE_CA.certificate, DEV_ROOT_CA.certificate)) keystore.save(distServKeyStoreFile, "cordacadevpass") } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/KeyStoreConfigHelpers.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/KeyStoreConfigHelpers.kt index 374a4cdf24..2ba6192454 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/KeyStoreConfigHelpers.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/KeyStoreConfigHelpers.kt @@ -9,12 +9,17 @@ import org.bouncycastle.asn1.x509.GeneralName import org.bouncycastle.asn1.x509.GeneralSubtree import org.bouncycastle.asn1.x509.NameConstraints import java.security.cert.X509Certificate +import javax.security.auth.x500.X500Principal + +// TODO Merge this file and DevIdentityGenerator /** * Create the node and SSL key stores needed by a node. The node key store will be populated with a node CA cert (using * the given legal name), and the SSL key store will store the TLS cert which is a sub-cert of the node CA. */ -fun SSLConfiguration.createDevKeyStores(rootCert: X509Certificate, intermediateCa: CertificateAndKeyPair, legalName: CordaX500Name) { +fun SSLConfiguration.createDevKeyStores(legalName: CordaX500Name, + rootCert: X509Certificate = DEV_ROOT_CA.certificate, + intermediateCa: CertificateAndKeyPair = DEV_INTERMEDIATE_CA) { val (nodeCaCert, nodeCaKeyPair) = createDevNodeCa(intermediateCa, legalName) loadOrCreateKeyStore(nodeKeystore, keyStorePassword).apply { @@ -39,6 +44,17 @@ fun SSLConfiguration.createDevKeyStores(rootCert: X509Certificate, intermediateC } } +fun createDevNetworkMapCa(rootCa: CertificateAndKeyPair = DEV_ROOT_CA): CertificateAndKeyPair { + val keyPair = Crypto.generateKeyPair() + val cert = X509Utilities.createCertificate( + CertificateType.NETWORK_MAP, + rootCa.certificate, + rootCa.keyPair, + X500Principal("CN=Network Map,O=R3 Ltd,L=London,C=GB"), + keyPair.public) + return CertificateAndKeyPair(cert, keyPair) +} + /** * Create a dev node CA cert, as a sub-cert of the given [intermediateCa], and matching key pair using the given * [CordaX500Name] as the cert subject. @@ -55,3 +71,16 @@ fun createDevNodeCa(intermediateCa: CertificateAndKeyPair, legalName: CordaX500N nameConstraints = nameConstraints) return CertificateAndKeyPair(cert, keyPair) } + +val DEV_INTERMEDIATE_CA: CertificateAndKeyPair get() = DevCaHelper.loadDevCa(X509Utilities.CORDA_INTERMEDIATE_CA) + +val DEV_ROOT_CA: CertificateAndKeyPair get() = DevCaHelper.loadDevCa(X509Utilities.CORDA_ROOT_CA) + +// We need a class so that we can get hold of the class loader +internal object DevCaHelper { + fun loadDevCa(alias: String): CertificateAndKeyPair { + // TODO: Should be identity scheme + val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") + return caKeyStore.getCertificateAndKeyPair(alias, "cordacadevkeypass") + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/SignedNodeInfo.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/SignedNodeInfo.kt index 1b0ed15348..ea7de12aa2 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/SignedNodeInfo.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/SignedNodeInfo.kt @@ -13,12 +13,14 @@ import java.security.SignatureException * A signed [NodeInfo] object containing a signature for each identity. The list of signatures is expected * to be in the same order as the identities. */ +// TODO Move this to net.corda.nodeapi.internal.network // TODO Add signatures for composite keys. The current thinking is to make sure there is a signature for each leaf key // that the node owns. This check can only be done by the network map server as it can check with the doorman if a node // is part of a composite identity. This of course further requires the doorman being able to issue CSRs for composite // public keys. @CordaSerializable class SignedNodeInfo(val raw: SerializedBytes, val signatures: List) { + // TODO Add root cert param (or TrustAnchor) to make sure all the identities belong to the same root fun verified(): NodeInfo { val nodeInfo = raw.deserialize() val identities = nodeInfo.legalIdentities.filterNot { it.owningKey is CompositeKey } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt index f784a4297b..118683f753 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt @@ -1,16 +1,12 @@ package net.corda.nodeapi.internal.network -import net.corda.core.crypto.DigitalSignature import net.corda.core.crypto.SecureHash -import net.corda.core.crypto.verify import net.corda.core.identity.Party +import net.corda.core.internal.CertRole +import net.corda.core.internal.SignedDataWithCert import net.corda.core.node.NodeInfo import net.corda.core.serialization.CordaSerializable -import net.corda.core.serialization.SerializedBytes -import net.corda.core.serialization.deserialize import net.corda.nodeapi.internal.crypto.X509Utilities -import java.security.SignatureException -import java.security.cert.CertPathValidatorException import java.security.cert.X509Certificate import java.time.Instant @@ -55,28 +51,8 @@ data class NetworkParameters( @CordaSerializable data class NotaryInfo(val identity: Party, val validating: Boolean) -/** - * A serialized [NetworkMap] and its signature and certificate. Enforces signature validity in order to deserialize the data - * contained within. - */ -@CordaSerializable -class SignedNetworkMap(val raw: SerializedBytes, val signature: DigitalSignatureWithCert) { - /** - * Return the deserialized NetworkMap if the signature and certificate can be verified. - * - * @throws CertPathValidatorException if the certificate path is invalid. - * @throws SignatureException if the signature is invalid. - */ - @Throws(SignatureException::class, CertPathValidatorException::class) - fun verified(trustedRoot: X509Certificate): NetworkMap { - signature.by.publicKey.verify(raw.bytes, signature) - // Assume network map cert is under the default trust root. - X509Utilities.validateCertificateChain(trustedRoot, signature.by, trustedRoot) - return raw.deserialize() - } -} - -// TODO: This class should reside in the [DigitalSignature] class. -// TODO: Removing the val from signatureBytes causes serialisation issues -/** A digital signature that identifies who the public key is owned by, and the certificate which provides prove of the identity */ -class DigitalSignatureWithCert(val by: X509Certificate, val signatureBytes: ByteArray) : DigitalSignature(signatureBytes) +fun SignedDataWithCert.verifiedNetworkMapCert(rootCert: X509Certificate): T { + require(CertRole.extract(sig.by) == CertRole.NETWORK_MAP) { "Incorrect cert role: ${CertRole.extract(sig.by)}" } + X509Utilities.validateCertificateChain(rootCert, sig.by, rootCert) + return verified() +} \ No newline at end of file diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt index fe9b88a24e..fdcb28fef2 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkParametersCopier.kt @@ -1,35 +1,32 @@ package net.corda.nodeapi.internal.network -import net.corda.core.crypto.Crypto -import net.corda.core.crypto.SignedData -import net.corda.core.crypto.sign import net.corda.core.internal.copyTo import net.corda.core.internal.div +import net.corda.core.internal.signWithCert import net.corda.core.serialization.serialize -import net.corda.nodeapi.internal.crypto.X509Utilities +import net.corda.nodeapi.internal.createDevNetworkMapCa +import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair import java.nio.file.FileAlreadyExistsException import java.nio.file.Path import java.nio.file.StandardCopyOption -import java.security.KeyPair class NetworkParametersCopier( networkParameters: NetworkParameters, - signingKeyPair: KeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME), + networkMapCa: CertificateAndKeyPair = createDevNetworkMapCa(), overwriteFile: Boolean = false ) { private val copyOptions = if (overwriteFile) arrayOf(StandardCopyOption.REPLACE_EXISTING) else emptyArray() - private val serializedNetworkParameters = networkParameters.let { - val serialize = it.serialize() - val signature = signingKeyPair.sign(serialize) - SignedData(serialize, signature).serialize() - } + private val serialisedSignedNetParams = networkParameters.signWithCert( + networkMapCa.keyPair.private, + networkMapCa.certificate + ).serialize() fun install(nodeDir: Path) { try { - serializedNetworkParameters.open().copyTo(nodeDir / NETWORK_PARAMS_FILE_NAME, *copyOptions) + serialisedSignedNetParams.open().copyTo(nodeDir / NETWORK_PARAMS_FILE_NAME, *copyOptions) } catch (e: FileAlreadyExistsException) { // This is only thrown if the file already exists and we didn't specify to overwrite it. In that case we // ignore this exception as we're happy with the existing file. } } -} \ No newline at end of file +} diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/X509UtilitiesTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/X509UtilitiesTest.kt index af38597bfe..05ec9db2e5 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/X509UtilitiesTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/X509UtilitiesTest.kt @@ -168,7 +168,7 @@ class X509UtilitiesTest { val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() // Generate server cert and private key and populate another keystore suitable for SSL - sslConfig.createDevKeyStores(rootCa.certificate, intermediateCa, MEGA_CORP.name) + sslConfig.createDevKeyStores(MEGA_CORP.name, rootCa.certificate, intermediateCa) // Load back server certificate val serverKeyStore = loadKeyStore(sslConfig.nodeKeystore, sslConfig.keyStorePassword) @@ -203,7 +203,7 @@ class X509UtilitiesTest { val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() // Generate server cert and private key and populate another keystore suitable for SSL - sslConfig.createDevKeyStores(rootCa.certificate, intermediateCa, MEGA_CORP.name) + sslConfig.createDevKeyStores(MEGA_CORP.name, rootCa.certificate, intermediateCa) sslConfig.createTrustStore(rootCa.certificate) val keyStore = loadKeyStore(sslConfig.sslKeystore, sslConfig.keyStorePassword) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/network/NetworkMapTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/network/NetworkMapTest.kt index 5530ba7850..67b5d35a76 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/network/NetworkMapTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/network/NetworkMapTest.kt @@ -1,13 +1,9 @@ package net.corda.node.services.network import net.corda.cordform.CordformNode -import net.corda.core.crypto.SignedData import net.corda.core.crypto.random63BitValue +import net.corda.core.internal.* import net.corda.core.internal.concurrent.transpose -import net.corda.core.internal.div -import net.corda.core.internal.exists -import net.corda.core.internal.list -import net.corda.core.internal.readAll import net.corda.core.node.NodeInfo import net.corda.core.serialization.deserialize import net.corda.core.utilities.getOrThrow @@ -69,7 +65,7 @@ class NetworkMapTest { val alice = startNode(providedName = ALICE_NAME).getOrThrow() val networkParameters = (alice.configuration.baseDirectory / NETWORK_PARAMS_FILE_NAME) .readAll() - .deserialize>() + .deserialize>() .verified() // We use a random modified time above to make the network parameters unqiue so that we're sure they came // from the server diff --git a/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt b/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt index 17544c8bd0..7b77464ed9 100644 --- a/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt @@ -64,7 +64,11 @@ class NodeRegistrationTest { @Before fun startServer() { - server = NetworkMapServer(1.minutes, portAllocation.nextHostAndPort(), DEV_ROOT_CA, "localhost", registrationHandler) + server = NetworkMapServer( + cacheTimeout = 1.minutes, + hostAndPort = portAllocation.nextHostAndPort(), + myHostNameValue = "localhost", + additionalServices = registrationHandler) serverHostAndPort = server.start() } diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt index 8252173ca0..e8aef9bffd 100644 --- a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt @@ -2,10 +2,15 @@ package net.corda.services.messaging import net.corda.core.crypto.Crypto import net.corda.core.identity.CordaX500Name -import net.corda.core.internal.* +import net.corda.core.internal.copyTo +import net.corda.core.internal.createDirectories +import net.corda.core.internal.exists +import net.corda.core.internal.x500Name +import net.corda.nodeapi.RPCApi import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_USER import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEER_USER -import net.corda.nodeapi.RPCApi +import net.corda.nodeapi.internal.DEV_INTERMEDIATE_CA +import net.corda.nodeapi.internal.DEV_ROOT_CA import net.corda.nodeapi.internal.config.SSLConfiguration import net.corda.nodeapi.internal.crypto.* import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration @@ -90,20 +95,13 @@ class MQSecurityAsNodeTest : MQSecurityTest() { javaClass.classLoader.getResourceAsStream("certificates/cordatruststore.jks").copyTo(trustStoreFile) } - val caKeyStore = loadKeyStore( - javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), - "cordacadevpass") - - val rootCACert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) - val intermediateCA = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - val clientKeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) // Set name constrain to the legal name. val nameConstraints = NameConstraints(arrayOf(GeneralSubtree(GeneralName(GeneralName.directoryName, legalName.x500Name))), arrayOf()) val clientCACert = X509Utilities.createCertificate( CertificateType.INTERMEDIATE_CA, - intermediateCA.certificate, - intermediateCA.keyPair, + DEV_INTERMEDIATE_CA.certificate, + DEV_INTERMEDIATE_CA.keyPair, legalName.x500Principal, clientKeyPair.public, nameConstraints = nameConstraints) @@ -123,7 +121,7 @@ class MQSecurityAsNodeTest : MQSecurityTest() { X509Utilities.CORDA_CLIENT_CA, clientKeyPair.private, keyPass, - arrayOf(clientCACert, intermediateCA.certificate, rootCACert)) + arrayOf(clientCACert, DEV_INTERMEDIATE_CA.certificate, DEV_ROOT_CA.certificate)) clientCAKeystore.save(nodeKeystore, keyStorePassword) val tlsKeystore = loadOrCreateKeyStore(sslKeystore, keyStorePassword) @@ -131,7 +129,7 @@ class MQSecurityAsNodeTest : MQSecurityTest() { X509Utilities.CORDA_CLIENT_TLS, tlsKeyPair.private, keyPass, - arrayOf(clientTLSCert, clientCACert, intermediateCA.certificate, rootCACert)) + arrayOf(clientTLSCert, clientCACert, DEV_INTERMEDIATE_CA.certificate, DEV_ROOT_CA.certificate)) tlsKeystore.save(sslKeystore, keyStorePassword) } } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 0edc5a1624..0f988ff0a1 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -12,7 +12,6 @@ import net.corda.core.concurrent.CordaFuture import net.corda.core.context.InvocationContext import net.corda.core.crypto.CompositeKey import net.corda.core.crypto.DigitalSignature -import net.corda.core.crypto.SignedData import net.corda.core.crypto.sign import net.corda.core.flows.* import net.corda.core.identity.CordaX500Name @@ -67,6 +66,7 @@ import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.crypto.loadKeyStore import net.corda.nodeapi.internal.network.NETWORK_PARAMS_FILE_NAME import net.corda.nodeapi.internal.network.NetworkParameters +import net.corda.nodeapi.internal.network.verifiedNetworkMapCert import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.HibernateConfiguration @@ -208,7 +208,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, val (identity, identityKeyPair) = obtainIdentity(notaryConfig = null) val identityService = makeIdentityService(identity.certificate) networkMapClient = configuration.compatibilityZoneURL?.let { NetworkMapClient(it, identityService.trustRoot) } - retrieveNetworkParameters() + retrieveNetworkParameters(identityService.trustRoot) // Do all of this in a database transaction so anything that might need a connection has one. val (startedImpl, schedulerService) = initialiseDatabasePersistence(schemaService, identityService) { database -> val networkMapCache = NetworkMapCacheImpl(PersistentNetworkMapCache(database, networkParameters.notaries), identityService) @@ -643,23 +643,19 @@ abstract class AbstractNode(val configuration: NodeConfiguration, return PersistentKeyManagementService(identityService, keyPairs) } - private fun retrieveNetworkParameters() { - val networkParamsFile = configuration.baseDirectory.list { paths -> - paths.filter { it.fileName.toString() == NETWORK_PARAMS_FILE_NAME }.findFirst().orElse(null) - } + private fun retrieveNetworkParameters(trustRoot: X509Certificate) { + val networkParamsFile = configuration.baseDirectory / NETWORK_PARAMS_FILE_NAME - networkParameters = if (networkParamsFile != null) { - networkParamsFile.readAll().deserialize>().verified() + networkParameters = if (networkParamsFile.exists()) { + networkParamsFile.readAll().deserialize>().verifiedNetworkMapCert(trustRoot) } else { log.info("No network-parameters file found. Expecting network parameters to be available from the network map.") val networkMapClient = checkNotNull(networkMapClient) { "Node hasn't been configured to connect to a network map from which to get the network parameters" } val (networkMap, _) = networkMapClient.getNetworkMap() - val signedParams = checkNotNull(networkMapClient.getNetworkParameter(networkMap.networkParameterHash)) { - "Failed loading network parameters from network map server" - } - val verifiedParams = signedParams.verified() + val signedParams = networkMapClient.getNetworkParameters(networkMap.networkParameterHash) + val verifiedParams = signedParams.verifiedNetworkMapCert(trustRoot) signedParams.serialize().open().copyTo(configuration.baseDirectory / NETWORK_PARAMS_FILE_NAME) verifiedParams } diff --git a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt index f48e8d4022..be5428675a 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt @@ -52,10 +52,7 @@ fun SSLConfiguration.configureDevKeyAndTrustStores(myLegalName: CordaX500Name) { loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordatruststore.jks"), "trustpass").save(trustStoreFile, trustStorePassword) } if (!sslKeystore.exists() || !nodeKeystore.exists()) { - val caKeyStore = loadKeyStore(javaClass.classLoader.getResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - val rootCert = caKeyStore.getX509Certificate(X509Utilities.CORDA_ROOT_CA) - val intermediateCa = caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") - createDevKeyStores(rootCert, intermediateCa, myLegalName) + createDevKeyStores(myLegalName) // Move distributed service composite key (generated by IdentityGenerator.generateToDisk) to keystore if exists. val distributedServiceKeystore = certificatesDirectory / "distributedService.jks" diff --git a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt index 5064361c7b..4e1e2c821c 100644 --- a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt @@ -2,28 +2,26 @@ package net.corda.node.services.network import com.google.common.util.concurrent.MoreExecutors import net.corda.core.crypto.SecureHash -import net.corda.core.crypto.SignedData +import net.corda.core.internal.SignedDataWithCert +import net.corda.core.internal.checkOkResponse import net.corda.core.internal.openHttpConnection +import net.corda.core.internal.responseAs import net.corda.core.node.NodeInfo -import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.utilities.contextLogger import net.corda.core.utilities.minutes import net.corda.core.utilities.seconds import net.corda.node.services.api.NetworkMapCacheInternal import net.corda.node.utilities.NamedThreadFactory +import net.corda.nodeapi.internal.SignedNodeInfo import net.corda.nodeapi.internal.network.NetworkMap import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap -import net.corda.nodeapi.internal.SignedNodeInfo +import net.corda.nodeapi.internal.network.verifiedNetworkMapCert import okhttp3.CacheControl import okhttp3.Headers -import org.apache.commons.io.IOUtils import rx.Subscription import java.io.BufferedReader import java.io.Closeable -import java.io.IOException -import java.net.HttpURLConnection import java.net.URL import java.security.cert.X509Certificate import java.time.Duration @@ -40,42 +38,29 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val trustedRoot: X509C requestMethod = "POST" setRequestProperty("Content-Type", "application/octet-stream") outputStream.use { signedNodeInfo.serialize().open().copyTo(it) } - if (responseCode != 200) { - throw IOException("Response Code $responseCode: ${IOUtils.toString(errorStream)}") - } + checkOkResponse() } } fun getNetworkMap(): NetworkMapResponse { - val conn = networkMapUrl.openHttpConnection() - val signedNetworkMap = conn.inputStream.use { it.readBytes() }.deserialize() - val networkMap = signedNetworkMap.verified(trustedRoot) - val timeout = CacheControl.parse(Headers.of(conn.headerFields.filterKeys { it != null }.mapValues { it.value.first() })).maxAgeSeconds().seconds + val connection = networkMapUrl.openHttpConnection() + val signedNetworkMap = connection.responseAs>() + val networkMap = signedNetworkMap.verifiedNetworkMapCert(trustedRoot) + val timeout = CacheControl.parse(Headers.of(connection.headerFields.filterKeys { it != null }.mapValues { it.value[0] })).maxAgeSeconds().seconds return NetworkMapResponse(networkMap, timeout) } - fun getNodeInfo(nodeInfoHash: SecureHash): NodeInfo? { - val conn = URL("$networkMapUrl/node-info/$nodeInfoHash").openHttpConnection() - return if (conn.responseCode == HttpURLConnection.HTTP_NOT_FOUND) { - null - } else { - val signedNodeInfo = conn.inputStream.use { it.readBytes() }.deserialize() - signedNodeInfo.verified() - } + fun getNodeInfo(nodeInfoHash: SecureHash): NodeInfo { + return URL("$networkMapUrl/node-info/$nodeInfoHash").openHttpConnection().responseAs().verified() } - fun getNetworkParameter(networkParameterHash: SecureHash): SignedData? { - val conn = URL("$networkMapUrl/network-parameter/$networkParameterHash").openHttpConnection() - return if (conn.responseCode == HttpURLConnection.HTTP_NOT_FOUND) { - null - } else { - conn.inputStream.use { it.readBytes() }.deserialize() - } + fun getNetworkParameters(networkParameterHash: SecureHash): SignedDataWithCert { + return URL("$networkMapUrl/network-parameters/$networkParameterHash").openHttpConnection().responseAs() } fun myPublicHostname(): String { - val conn = URL("$networkMapUrl/my-hostname").openHttpConnection() - return conn.inputStream.bufferedReader().use(BufferedReader::readLine) + val connection = URL("$networkMapUrl/my-hostname").openHttpConnection() + return connection.inputStream.bufferedReader().use(BufferedReader::readLine) } } diff --git a/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt b/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt index 88d9230a61..49dde51b6d 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt @@ -24,9 +24,7 @@ class HTTPNetworkRegistrationService(compatibilityZoneURL: URL) : NetworkRegistr @Throws(CertificateRequestException::class) override fun retrieveCertificates(requestId: String): Array? { // Poll server to download the signed certificate once request has been approved. - val url = URL("$registrationURL/$requestId") - - val conn = url.openConnection() as HttpURLConnection + val conn = URL("$registrationURL/$requestId").openHttpConnection() conn.requestMethod = "GET" return when (conn.responseCode) { diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt index fa0a67fe15..da4784d060 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt @@ -1,6 +1,5 @@ package net.corda.node.services.network -import net.corda.core.crypto.SecureHash import net.corda.core.crypto.sha256 import net.corda.core.serialization.serialize import net.corda.core.utilities.seconds @@ -22,7 +21,6 @@ import org.junit.Test import java.io.IOException import java.net.URL import kotlin.test.assertEquals -import kotlin.test.assertNotNull class NetworkMapClientTest { @Rule @@ -83,8 +81,8 @@ class NetworkMapClientTest { @Test fun `download NetworkParameter correctly`() { // The test server returns same network parameter for any hash. - val networkParameter = networkMapClient.getNetworkParameter(SecureHash.randomSHA256())?.verified() - assertNotNull(networkParameter) + val parametersHash = server.networkParameters.serialize().hash + val networkParameter = networkMapClient.getNetworkParameters(parametersHash).verified() assertEquals(server.networkParameters, networkParameter) } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt index 810fcedf77..52975bbdb3 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt @@ -1,20 +1,16 @@ package net.corda.testing.node.internal.network -import net.corda.core.crypto.* -import net.corda.core.identity.CordaX500Name +import net.corda.core.crypto.SecureHash +import net.corda.core.internal.signWithCert import net.corda.core.node.NodeInfo import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.utilities.NetworkHostAndPort import net.corda.nodeapi.internal.SignedNodeInfo +import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair -import net.corda.nodeapi.internal.crypto.CertificateType -import net.corda.nodeapi.internal.crypto.X509Utilities -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert import net.corda.nodeapi.internal.network.NetworkMap import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap -import net.corda.testing.DEV_ROOT_CA import org.eclipse.jetty.server.Server import org.eclipse.jetty.server.ServerConnector import org.eclipse.jetty.server.handler.HandlerCollection @@ -28,34 +24,19 @@ import java.net.InetSocketAddress import java.security.SignatureException import java.time.Duration import java.time.Instant -import javax.security.auth.x500.X500Principal import javax.ws.rs.* import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response import javax.ws.rs.core.Response.ok import javax.ws.rs.core.Response.status -class NetworkMapServer(cacheTimeout: Duration, +class NetworkMapServer(private val cacheTimeout: Duration, hostAndPort: NetworkHostAndPort, - rootCa: CertificateAndKeyPair = DEV_ROOT_CA, + private val networkMapCa: CertificateAndKeyPair = createDevNetworkMapCa(), private val myHostNameValue: String = "test.host.name", vararg additionalServices: Any) : Closeable { companion object { private val stubNetworkParameters = NetworkParameters(1, emptyList(), 10485760, 40000, Instant.now(), 10) - - private fun networkMapKeyAndCert(rootCAKeyAndCert: CertificateAndKeyPair): CertificateAndKeyPair { - val networkMapKey = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) - val networkMapCert = X509Utilities.createCertificate( - CertificateType.NETWORK_MAP, - rootCAKeyAndCert.certificate, - rootCAKeyAndCert.keyPair, - X500Principal("CN=Corda Network Map,O=R3 Ltd,L=London,C=GB"), - networkMapKey.public) - // Check that the certificate validates. Nodes will perform this check upon receiving a network map, - // it's better to fail here than there. - X509Utilities.validateCertificateChain(rootCAKeyAndCert.certificate, networkMapCert) - return CertificateAndKeyPair(networkMapCert, networkMapKey) - } } private val server: Server @@ -64,9 +45,7 @@ class NetworkMapServer(cacheTimeout: Duration, check(field == stubNetworkParameters) { "Network parameters can be set only once" } field = networkParameters } - private val serializedParameters get() = networkParameters.serialize() - private val service = InMemoryNetworkMapService(cacheTimeout, networkMapKeyAndCert(rootCa)) - + private val service = InMemoryNetworkMapService() init { server = Server(InetSocketAddress(hostAndPort.host, hostAndPort.port)).apply { @@ -106,14 +85,10 @@ class NetworkMapServer(cacheTimeout: Duration, } @Path("network-map") - inner class InMemoryNetworkMapService(private val cacheTimeout: Duration, - private val networkMapKeyAndCert: CertificateAndKeyPair) { + inner class InMemoryNetworkMapService { private val nodeInfoMap = mutableMapOf() - private val parametersHash by lazy { serializedParameters.hash } - private val signedParameters by lazy { - SignedData( - serializedParameters, - DigitalSignature.WithKey(networkMapKeyAndCert.keyPair.public, Crypto.doSign(networkMapKeyAndCert.keyPair.private, serializedParameters.bytes))) + private val signedNetParams by lazy { + networkParameters.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) } @POST @@ -121,10 +96,9 @@ class NetworkMapServer(cacheTimeout: Duration, @Consumes(MediaType.APPLICATION_OCTET_STREAM) fun publishNodeInfo(input: InputStream): Response { return try { - val registrationData = input.readBytes().deserialize() - val nodeInfo = registrationData.verified() - val nodeInfoHash = nodeInfo.serialize().sha256() - nodeInfoMap.put(nodeInfoHash, registrationData) + val signedNodeInfo = input.readBytes().deserialize() + signedNodeInfo.verified() + nodeInfoMap[signedNodeInfo.raw.hash] = signedNodeInfo ok() } catch (e: Exception) { when (e) { @@ -137,10 +111,8 @@ class NetworkMapServer(cacheTimeout: Duration, @GET @Produces(MediaType.APPLICATION_OCTET_STREAM) fun getNetworkMap(): Response { - val networkMap = NetworkMap(nodeInfoMap.keys.toList(), parametersHash) - val serializedNetworkMap = networkMap.serialize() - val signature = Crypto.doSign(networkMapKeyAndCert.keyPair.private, serializedNetworkMap.bytes) - val signedNetworkMap = SignedNetworkMap(networkMap.serialize(), DigitalSignatureWithCert(networkMapKeyAndCert.certificate, signature)) + val networkMap = NetworkMap(nodeInfoMap.keys.toList(), signedNetParams.raw.hash) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) return Response.ok(signedNetworkMap.serialize().bytes).header("Cache-Control", "max-age=${cacheTimeout.seconds}").build() } @@ -162,16 +134,15 @@ class NetworkMapServer(cacheTimeout: Duration, } @GET - @Path("network-parameter/{var}") + @Path("network-parameters/{var}") @Produces(MediaType.APPLICATION_OCTET_STREAM) - fun getNetworkParameter(@PathParam("var") networkParameterHash: String): Response { - return Response.ok(signedParameters.serialize().bytes).build() + fun getNetworkParameter(@PathParam("var") hash: String): Response { + require(signedNetParams.raw.hash == SecureHash.parse(hash)) + return Response.ok(signedNetParams.serialize().bytes).build() } @GET @Path("my-hostname") - fun getHostName(): Response { - return Response.ok(myHostNameValue).build() - } + fun getHostName(): Response = Response.ok(myHostNameValue).build() } } diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/TestConstants.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/TestConstants.kt index 406477c52b..17fb7e1294 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/TestConstants.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/TestConstants.kt @@ -7,9 +7,6 @@ import net.corda.core.contracts.TypeOnlyCommandData import net.corda.core.crypto.generateKeyPair import net.corda.core.identity.CordaX500Name import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair -import net.corda.nodeapi.internal.crypto.X509Utilities -import net.corda.nodeapi.internal.crypto.getCertificateAndKeyPair -import net.corda.nodeapi.internal.crypto.loadKeyStore import java.security.PublicKey import java.time.Instant @@ -32,17 +29,10 @@ val ALICE_NAME = CordaX500Name("Alice Corp", "Madrid", "ES") val BOB_NAME = CordaX500Name("Bob Plc", "Rome", "IT") @JvmField val CHARLIE_NAME = CordaX500Name("Charlie Ltd", "Athens", "GR") -val DEV_INTERMEDIATE_CA: CertificateAndKeyPair by lazy { - // TODO: Should be identity scheme - val caKeyStore = loadKeyStore(ClassLoader.getSystemResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, "cordacadevkeypass") -} -val DEV_ROOT_CA: CertificateAndKeyPair by lazy { - // TODO: Should be identity scheme - val caKeyStore = loadKeyStore(ClassLoader.getSystemResourceAsStream("certificates/cordadevcakeys.jks"), "cordacadevpass") - caKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_ROOT_CA, "cordacadevkeypass") -} +val DEV_INTERMEDIATE_CA: CertificateAndKeyPair by lazy { net.corda.nodeapi.internal.DEV_INTERMEDIATE_CA } + +val DEV_ROOT_CA: CertificateAndKeyPair by lazy { net.corda.nodeapi.internal.DEV_ROOT_CA } fun dummyCommand(vararg signers: PublicKey = arrayOf(generateKeyPair().public)) = Command(DummyCommandData, signers.toList()) From b1509607cb22c038a7a2a676de2b2f37d1c6fd03 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Fri, 12 Jan 2018 15:15:38 +0000 Subject: [PATCH 17/17] Updating network-management to use DigitalSignatureWithCert, network map certs and other relevant changes from O/S. --- network-management/README.md | 8 +- .../doorman/NodeRegistrationTest.kt | 22 ++- .../r3/corda/networkmanage/hsm/DemoMain.kt | 43 ------ .../hsm/SigningServiceIntegrationTest.kt | 63 +------- .../common/persistence/NetworkMapStorage.kt | 6 +- .../PersistentNetworkMapStorage.kt | 15 +- .../persistence/entity/NetworkMapEntity.kt | 4 +- .../common/signer/NetworkMapSigner.kt | 12 +- .../networkmanage/common/signer/Signer.kt | 15 +- .../corda/networkmanage/common/utils/Utils.kt | 16 +-- .../doorman/DoormanParameters.kt | 3 +- .../r3/corda/networkmanage/doorman/Main.kt | 136 +++++++++++------- .../doorman/signer/LocalSigner.kt | 19 +-- .../doorman/webservice/NodeInfoWebService.kt | 8 +- .../hsm/configuration/Configuration.kt | 2 +- .../hsm/signer/HsmNetworkMapSigner.kt | 21 +-- .../networkmanage/hsm/utils/X509Utils.kt | 18 +-- .../PersistentNetworkMapStorageTest.kt | 42 +++--- .../common/signer/NetworkMapSignerTest.kt | 44 +++--- .../doorman/NodeInfoWebServiceTest.kt | 64 ++++----- 20 files changed, 244 insertions(+), 317 deletions(-) delete mode 100644 network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/DemoMain.kt diff --git a/network-management/README.md b/network-management/README.md index 33929abdd0..4f278c83df 100644 --- a/network-management/README.md +++ b/network-management/README.md @@ -134,8 +134,8 @@ networkMapConfig { ### 1. Create keystore for local signer - If local signer is enabled, the server will look for keystores in the certificate folder on start up. - The keystores can be created using `--mode` flag. + If local signer is enabled, the server will look for key stores in the certificate folder on start up. + The key stores can be created using `--mode` flag. ``` java -jar doorman-.jar --mode ROOT_KEYGEN ``` @@ -180,8 +180,8 @@ networkMapConfig { Save the parameters to `network-parameters.conf` ### 5. Load initial network parameters file for network map service -A network parameters file is required to start the network map service for the first time. The initial network parameters file can be loaded using the `--update-network-parameter` flag. +A network parameters file is required to start the network map service for the first time. The initial network parameters file can be loaded using the `--update-network-parameters` flag. We can now restart the network management server with both doorman and network map service. ``` -java -jar doorman-.jar --update-network-parameter network-parameters.conf +java -jar doorman-.jar --update-network-parameters network-parameters.conf ``` diff --git a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/doorman/NodeRegistrationTest.kt b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/doorman/NodeRegistrationTest.kt index 7c6ef580f1..4ece2a3a21 100644 --- a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/doorman/NodeRegistrationTest.kt +++ b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/doorman/NodeRegistrationTest.kt @@ -15,6 +15,7 @@ import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.seconds import net.corda.finance.DOLLARS import net.corda.finance.flows.CashIssueAndPaymentFlow +import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair import net.corda.nodeapi.internal.network.NetworkParameters import net.corda.nodeapi.internal.persistence.DatabaseConfig @@ -60,15 +61,17 @@ class NodeRegistrationTest : IntegrationTest() { private val dbId = random63BitValue().toString() private lateinit var rootCaCert: X509Certificate - private lateinit var intermediateCa: CertificateAndKeyPair + private lateinit var csrCa: CertificateAndKeyPair + private lateinit var networkMapCa: CertificateAndKeyPair private var server: NetworkManagementServer? = null @Before fun init() { - val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() + val (rootCa, doormanCa) = createDevIntermediateCaCertPath() rootCaCert = rootCa.certificate - this.intermediateCa = intermediateCa + this.csrCa = doormanCa + networkMapCa = createDevNetworkMapCa(rootCa) } @After @@ -138,10 +141,15 @@ class NodeRegistrationTest : IntegrationTest() { start( serverAddress, configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(runMigration = true)), - LocalSigner(intermediateCa.keyPair, arrayOf(intermediateCa.certificate, rootCaCert)), - networkParameters, - networkParameters?.let { NetworkMapConfig(cacheTimeout = timeoutMillis, signInterval = timeoutMillis) }, - DoormanConfig(approveAll = true, jiraConfig = null, approveInterval = timeoutMillis) + LocalSigner(csrCa.keyPair, arrayOf(csrCa.certificate, rootCaCert)), + DoormanConfig(approveAll = true, jiraConfig = null, approveInterval = timeoutMillis), + networkParameters?.let { + NetworkMapStartParams( + LocalSigner(networkMapCa.keyPair, arrayOf(networkMapCa.certificate, rootCaCert)), + networkParameters, + NetworkMapConfig(cacheTimeout = timeoutMillis, signInterval = timeoutMillis) + ) + } ) } } diff --git a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/DemoMain.kt b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/DemoMain.kt deleted file mode 100644 index bfc4366eeb..0000000000 --- a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/DemoMain.kt +++ /dev/null @@ -1,43 +0,0 @@ -package com.r3.corda.networkmanage.hsm - -import com.r3.corda.networkmanage.hsm.SigningServiceIntegrationTest.Companion.DB_NAME -import com.r3.corda.networkmanage.hsm.SigningServiceIntegrationTest.Companion.H2_TCP_PORT -import com.r3.corda.networkmanage.hsm.SigningServiceIntegrationTest.Companion.HOST -import com.r3.corda.networkmanage.hsm.configuration.Parameters -import java.util.* - -/** - * The main method for an interactive HSM signing service test/demo. It is supposed to be executed with the - * `DEMO - Create CSR and poll` method located in the [SigningServiceIntegrationTest], which is responsible for simulating - * CSR creation on the Doorman side. - * Execution instructions: - * 1) It is assumed that the HSM simulator is installed locally (or via means of the VM) and accessible under the address - * configured under the 'device' parameter (defaults to 3001@127.0.0.1). If that is not the case please specify - * a correct 'device' parameter value. Also, it is assumed that the HSM setup consists of a cryptographic user eligible to - * sign the CSRs (and potentially to generate new root and intermediate certificates). - * 2) Run the `DEMO - Create CSR and poll` as a regular test from your IntelliJ. - * The method starts the doorman, creates 3 CSRs for ALICE, BOB and CHARLIE - * and then polls the doorman until all 3 requests are signed. - * 3) Once the `DEMO - Create CSR and poll` is started, execute the following main method - * and interact with console menu options presented. - */ -fun main(args: Array) { - run(Parameters( - dataSourceProperties = makeTestDataSourceProperties(), - databaseConfig = makeNotInitialisingTestDatabaseProperties(), - csrPrivateKeyPassword = "", - networkMapPrivateKeyPassword = "", - rootPrivateKeyPassword = "", - keyGroup = "DEV.DOORMAN", - validDays = 3650 - )) -} - -private fun makeTestDataSourceProperties(): Properties { - val props = Properties() - props.setProperty("dataSourceClassName", "org.h2.jdbcx.JdbcDataSource") - props.setProperty("dataSource.url", "jdbc:h2:tcp://$HOST:$H2_TCP_PORT/mem:$DB_NAME;DB_CLOSE_DELAY=-1") - props.setProperty("dataSource.user", "sa") - props.setProperty("dataSource.password", "") - return props -} diff --git a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt index bcbb91fec9..765d47d5f0 100644 --- a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt +++ b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt @@ -22,27 +22,25 @@ import net.corda.nodeapi.internal.createDevNodeCa import net.corda.nodeapi.internal.crypto.* import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.testing.ALICE_NAME -import net.corda.testing.BOB_NAME -import net.corda.testing.CHARLIE_NAME import net.corda.testing.SerializationEnvironmentRule import net.corda.testing.internal.createDevIntermediateCaCertPath import net.corda.testing.internal.rigorousMock import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest -import org.h2.tools.Server -import org.junit.* +import org.junit.After +import org.junit.Before +import org.junit.Rule +import org.junit.Test import org.junit.rules.TemporaryFolder import java.net.URL import java.security.cert.X509Certificate import java.util.* import javax.persistence.PersistenceException import kotlin.concurrent.scheduleAtFixedRate -import kotlin.concurrent.thread class SigningServiceIntegrationTest { companion object { - val H2_TCP_PORT = "8092" - val HOST = "localhost" - val DB_NAME = "test_db" + private val HOST = "localhost" + private val DB_NAME = "test_db" } @Rule @@ -92,7 +90,7 @@ class SigningServiceIntegrationTest { val database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(runMigration = true)) NetworkManagementServer().use { server -> - server.start(NetworkHostAndPort(HOST, 0), database, networkMapServiceParameter = null, doormanServiceParameter = DoormanConfig(approveAll = true, approveInterval = 2.seconds.toMillis(), jiraConfig = null), updateNetworkParameters = null) + server.start(NetworkHostAndPort(HOST, 0), database, doormanServiceParameter = DoormanConfig(approveAll = true, approveInterval = 2.seconds.toMillis(), jiraConfig = null), startNetworkMap = null) val doormanHostAndPort = server.hostAndPort // Start Corda network registration. val config = createConfig().also { @@ -131,51 +129,6 @@ class SigningServiceIntegrationTest { } } - /* - * Piece of code is purely for demo purposes and should not be considered as actual test (therefore it is ignored). - * Its purpose is to produce 3 CSRs and wait (polling Doorman) for external signature. - * The use of the jUnit testing framework was chosen due to the convenience reasons: mocking, tempFolder storage. - * It is meant to be run together with the [DemoMain.main] method, which executes HSM signing service. - * The split is done due to the limited console support while executing tests and inability to capture user's input there. - * - */ - @Ignore - @Test - fun `DEMO - Create CSR and poll`() { - //Start doorman server - val database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(runMigration = true)) - - NetworkManagementServer().use { server -> - server.start(NetworkHostAndPort(HOST, 0), database, networkMapServiceParameter = null, doormanServiceParameter = DoormanConfig(approveAll = true, approveInterval = 2.seconds.toMillis(), jiraConfig = null), updateNetworkParameters = null) - thread(start = true, isDaemon = true) { - val h2ServerArgs = arrayOf("-tcpPort", H2_TCP_PORT, "-tcpAllowOthers") - Server.createTcpServer(*h2ServerArgs).start() - } - - // Start Corda network registration. - (1..3).map { num -> - thread(start = true) { - // Start Corda network registration. - val config = createConfig().also { - doReturn(when (num) { - 1 -> ALICE_NAME - 2 -> BOB_NAME - 3 -> CHARLIE_NAME - else -> throw IllegalArgumentException("Unrecognised option") - }).whenever(it).myLegalName - doReturn(URL("http://$HOST:${server.hostAndPort.port}")).whenever(it).compatibilityZoneURL - } - config.certificatesDirectory.createDirectories() - loadOrCreateKeyStore(config.trustStoreFile, config.trustStorePassword).also { - it.addOrReplaceCertificate(X509Utilities.CORDA_ROOT_CA, rootCaCert) - it.save(config.trustStoreFile, config.trustStorePassword) - } - NetworkRegistrationHelper(config, HTTPNetworkRegistrationService(config.compatibilityZoneURL!!)).buildKeystore() - } - }.map { it.join() } - } - } - private fun createConfig(): NodeConfiguration { return rigorousMock().also { doReturn(tempFolder.root.toPath()).whenever(it).baseDirectory @@ -198,5 +151,3 @@ class SigningServiceIntegrationTest { return props } } - -internal fun makeNotInitialisingTestDatabaseProperties() = DatabaseConfig(runMigration = false) diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/NetworkMapStorage.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/NetworkMapStorage.kt index 0dea37f7b8..732f049443 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/NetworkMapStorage.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/NetworkMapStorage.kt @@ -1,9 +1,9 @@ package com.r3.corda.networkmanage.common.persistence +import com.r3.corda.networkmanage.common.utils.SignedNetworkMap +import com.r3.corda.networkmanage.common.utils.SignedNetworkParameters import net.corda.core.crypto.SecureHash -import net.corda.core.crypto.SignedData import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap /** * Data access object interface for NetworkMap persistence layer @@ -34,7 +34,7 @@ interface NetworkMapStorage { * Return the signed network parameters object which matches the given hash. The hash is that of the underlying * [NetworkParameters] object and not the `SignedData` object that's returned. */ - fun getSignedNetworkParameters(hash: SecureHash): SignedData? + fun getSignedNetworkParameters(hash: SecureHash): SignedNetworkParameters? /** * Retrieve network map parameters. diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorage.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorage.kt index 314d2b9f83..740885dee0 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorage.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorage.kt @@ -1,17 +1,16 @@ package com.r3.corda.networkmanage.common.persistence import com.r3.corda.networkmanage.common.persistence.entity.* +import com.r3.corda.networkmanage.common.utils.SignedNetworkMap +import com.r3.corda.networkmanage.common.utils.SignedNetworkParameters import com.r3.corda.networkmanage.doorman.signer.LocalSigner -import net.corda.core.crypto.DigitalSignature import net.corda.core.crypto.SecureHash -import net.corda.core.crypto.SignedData import net.corda.core.crypto.sha256 import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.nodeapi.internal.network.NetworkMap import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap import net.corda.nodeapi.internal.persistence.CordaPersistence /** @@ -40,8 +39,8 @@ class PersistentNetworkMapStorage(private val database: CordaPersistence, privat database.transaction { val networkMapEntity = NetworkMapEntity( networkMap = signedNetworkMap.raw.bytes, - signature = signedNetworkMap.signature.signatureBytes, - certificate = signedNetworkMap.signature.by.encoded + signature = signedNetworkMap.sig.bytes, + certificate = signedNetworkMap.sig.by.encoded ) session.save(networkMapEntity) } @@ -49,10 +48,10 @@ class PersistentNetworkMapStorage(private val database: CordaPersistence, privat // TODO The signing cannot occur here as it won't work with an HSM. The signed network parameters needs to be persisted // into the database. - override fun getSignedNetworkParameters(hash: SecureHash): SignedData? { + override fun getSignedNetworkParameters(hash: SecureHash): SignedNetworkParameters? { val netParamsBytes = getNetworkParametersEntity(hash.toString())?.parametersBytes ?: return null - val sigWithCert = localSigner!!.sign(netParamsBytes) - return SignedData(SerializedBytes(netParamsBytes), DigitalSignature.WithKey(sigWithCert.by.publicKey, sigWithCert.signatureBytes)) + val sigWithCert = localSigner!!.signBytes(netParamsBytes) + return SignedNetworkParameters(SerializedBytes(netParamsBytes), sigWithCert) } override fun getNodeInfoHashes(certificateStatus: CertificateStatus): List { diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/entity/NetworkMapEntity.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/entity/NetworkMapEntity.kt index 019bed926e..f71a252213 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/entity/NetworkMapEntity.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/entity/NetworkMapEntity.kt @@ -1,7 +1,7 @@ package com.r3.corda.networkmanage.common.persistence.entity +import net.corda.core.internal.DigitalSignatureWithCert import net.corda.nodeapi.internal.crypto.X509CertificateFactory -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert import javax.persistence.* @Entity @@ -24,7 +24,7 @@ class NetworkMapEntity( val certificate: ByteArray ) { /** - * Deserializes NetworkMapEntity.signatureBytes into the [SignatureAndCertPath] instance + * Deserializes NetworkMapEntity.signatureBytes into the [DigitalSignatureWithCert] instance */ fun signatureAndCertificate(): DigitalSignatureWithCert { return DigitalSignatureWithCert(X509CertificateFactory().generateCertificate(certificate.inputStream()), signature) diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSigner.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSigner.kt index 05f0111d42..cafc5e7469 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSigner.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSigner.kt @@ -2,9 +2,9 @@ package com.r3.corda.networkmanage.common.signer import com.r3.corda.networkmanage.common.persistence.CertificateStatus import com.r3.corda.networkmanage.common.persistence.NetworkMapStorage +import net.corda.core.internal.SignedDataWithCert import net.corda.core.serialization.serialize import net.corda.nodeapi.internal.network.NetworkMap -import net.corda.nodeapi.internal.network.SignedNetworkMap class NetworkMapSigner(private val networkMapStorage: NetworkMapStorage, private val signer: Signer) { /** @@ -14,12 +14,10 @@ class NetworkMapSigner(private val networkMapStorage: NetworkMapStorage, private val currentSignedNetworkMap = networkMapStorage.getCurrentNetworkMap() val nodeInfoHashes = networkMapStorage.getNodeInfoHashes(CertificateStatus.VALID) val networkParameters = networkMapStorage.getLatestNetworkParameters() - val networkMap = NetworkMap(nodeInfoHashes, networkParameters.serialize().hash) - // We wan only check if the data structure is same. - if (networkMap != currentSignedNetworkMap?.verified(null)) { - val digitalSignature = signer.sign(networkMap.serialize().bytes) - val signedHashedNetworkMap = SignedNetworkMap(networkMap.serialize(), digitalSignature) - networkMapStorage.saveNetworkMap(signedHashedNetworkMap) + val serialisedNetworkMap = NetworkMap(nodeInfoHashes, networkParameters.serialize().hash).serialize() + if (serialisedNetworkMap != currentSignedNetworkMap?.raw) { + val newSignedNetworkMap = SignedDataWithCert(serialisedNetworkMap, signer.signBytes(serialisedNetworkMap.bytes)) + networkMapStorage.saveNetworkMap(newSignedNetworkMap) } } } \ No newline at end of file diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/Signer.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/Signer.kt index e00ee54335..40a030c2b0 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/Signer.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/Signer.kt @@ -1,17 +1,24 @@ package com.r3.corda.networkmanage.common.signer -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert +import net.corda.core.internal.DigitalSignatureWithCert +import net.corda.core.internal.SignedDataWithCert +import net.corda.core.serialization.serialize /** * An interface for arbitrary data signing functionality. */ interface Signer { /** - * Signs given [data]. The signing key selction strategy is left to the implementing class. - * @return [SignatureAndCertPath] that encapsulates the signature and the certificate path used in the signing process. + * Signs given bytes. The signing key selction strategy is left to the implementing class. + * @return [DigitalSignatureWithCert] that encapsulates the signature and the certificate path used in the signing process. * @throws [AuthenticationException] if fails authentication */ - fun sign(data: ByteArray): DigitalSignatureWithCert + fun signBytes(data: ByteArray): DigitalSignatureWithCert + + fun signObject(obj: T): SignedDataWithCert { + val serialised = obj.serialize() + return SignedDataWithCert(serialised, signBytes(serialised.bytes)) + } } class AuthenticationException : Exception() diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/utils/Utils.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/utils/Utils.kt index f1647de6bd..8ac987e03a 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/utils/Utils.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/utils/Utils.kt @@ -5,15 +5,18 @@ import com.typesafe.config.Config import com.typesafe.config.ConfigFactory import joptsimple.ArgumentAcceptingOptionSpec import joptsimple.OptionParser -import net.corda.core.crypto.DigitalSignature import net.corda.core.crypto.sha256 +import net.corda.core.internal.SignedDataWithCert import net.corda.nodeapi.internal.crypto.X509CertificateFactory -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert -import org.bouncycastle.cert.X509CertificateHolder +import net.corda.nodeapi.internal.network.NetworkMap +import net.corda.nodeapi.internal.network.NetworkParameters import java.security.PublicKey import java.security.cert.CertPath import java.security.cert.Certificate -import java.security.cert.X509Certificate + +// TODO These should be defined in node-api +typealias SignedNetworkParameters = SignedDataWithCert +typealias SignedNetworkMap = SignedDataWithCert // TODO: replace this with Crypto.hash when its available. /** @@ -39,15 +42,10 @@ fun Array.toConfigWithOptions(registerOptions: OptionParser.() -> Un class ShowHelpException(val parser: OptionParser, val errorMessage: String? = null) : Exception() -// TODO Remove this as we already have InternalUtils.cert -fun X509CertificateHolder.toX509Certificate(): X509Certificate = X509CertificateFactory().generateCertificate(encoded.inputStream()) - fun buildCertPath(vararg certificates: Certificate): CertPath = X509CertificateFactory().delegate.generateCertPath(certificates.asList()) fun buildCertPath(certPathBytes: ByteArray): CertPath = X509CertificateFactory().delegate.generateCertPath(certPathBytes.inputStream()) -fun DigitalSignature.WithKey.withCert(cert: X509Certificate): DigitalSignatureWithCert = DigitalSignatureWithCert(cert, bytes) - private fun String.toCamelcase(): String { return if (contains('_') || contains('-')) { CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, this.replace("-", "_")) diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/DoormanParameters.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/DoormanParameters.kt index 3d4b47a107..4ffaf397ce 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/DoormanParameters.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/DoormanParameters.kt @@ -53,10 +53,11 @@ data class DoormanConfig(val approveAll: Boolean = false, val approveInterval: Long = NetworkManagementServerParameters.DEFAULT_APPROVE_INTERVAL.toMillis()) data class NetworkMapConfig(val cacheTimeout: Long, - // TODO: Move signing to signing server. + // TODO: Move signing to signing server. val signInterval: Long = NetworkManagementServerParameters.DEFAULT_SIGN_INTERVAL.toMillis()) enum class Mode { + // TODO CA_KEYGEN now also generates the nework map cert, so it should be renamed. DOORMAN, CA_KEYGEN, ROOT_KEYGEN } diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/Main.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/Main.kt index 6b5f3636f3..9cce45f4ec 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/Main.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/Main.kt @@ -11,7 +11,10 @@ import com.r3.corda.networkmanage.doorman.signer.LocalSigner import com.r3.corda.networkmanage.doorman.webservice.MonitoringWebService import com.r3.corda.networkmanage.doorman.webservice.NodeInfoWebService import com.r3.corda.networkmanage.doorman.webservice.RegistrationWebService +import com.r3.corda.networkmanage.hsm.configuration.Parameters.Companion.DEFAULT_CSR_CERTIFICATE_NAME +import com.r3.corda.networkmanage.hsm.configuration.Parameters.Companion.DEFAULT_NETWORK_MAP_CERTIFICATE_NAME import net.corda.core.crypto.Crypto +import net.corda.core.crypto.SignatureScheme import net.corda.core.identity.CordaX500Name import net.corda.core.internal.createDirectories import net.corda.core.internal.div @@ -34,6 +37,7 @@ import java.time.Instant import java.util.* import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import javax.security.auth.x500.X500Principal import kotlin.concurrent.thread import kotlin.system.exitProcess @@ -130,17 +134,16 @@ class NetworkManagementServer : Closeable { fun start(hostAndPort: NetworkHostAndPort, database: CordaPersistence, - signer: LocalSigner? = null, - updateNetworkParameters: NetworkParameters?, - networkMapServiceParameter: NetworkMapConfig?, - doormanServiceParameter: DoormanConfig?) { - + doormanSigner: LocalSigner? = null, + doormanServiceParameter: DoormanConfig?, // TODO Doorman config shouldn't be optional as the doorman is always required to run + startNetworkMap: NetworkMapStartParams? + ) { val services = mutableListOf() val serverStatus = NetworkManagementServerStatus() // TODO: move signing to signing server. - networkMapServiceParameter?.let { services += getNetworkMapService(it, database, signer, updateNetworkParameters) } - doormanServiceParameter?.let { services += getDoormanService(it, database, signer, serverStatus) } + startNetworkMap?.let { services += getNetworkMapService(it.config, database, it.signer, it.updateNetworkParameters) } + doormanServiceParameter?.let { services += getDoormanService(it, database, doormanSigner, serverStatus) } require(services.isNotEmpty()) { "No service created, please provide at least one service config." } @@ -150,11 +153,13 @@ class NetworkManagementServer : Closeable { val webServer = NetworkManagementWebServer(hostAndPort, *services.toTypedArray()) webServer.start() - doOnClose += { webServer.close() } + doOnClose += webServer::close this.hostAndPort = webServer.hostAndPort } } +data class NetworkMapStartParams(val signer: LocalSigner?, val updateNetworkParameters: NetworkParameters?, val config: NetworkMapConfig) + data class NetworkManagementServerStatus(var serverStartTime: Instant = Instant.now(), var lastRequestCheckTime: Instant? = null) /** Read password from console, do a readLine instead if console is null (e.g. when debugging in IDE). */ @@ -203,58 +208,77 @@ fun generateRootKeyPair(rootStoreFile: Path, rootKeystorePass: String?, rootPriv println(loadKeyStore(rootStoreFile, rootKeystorePassword).getCertificate(X509Utilities.CORDA_ROOT_CA).publicKey) } -fun generateCAKeyPair(keystoreFile: Path, rootStoreFile: Path, rootKeystorePass: String?, rootPrivateKeyPass: String?, keystorePass: String?, caPrivateKeyPass: String?) { - println("Generating Intermediate CA keypair and certificate using root keystore $rootStoreFile.") +fun generateSigningKeyPairs(keystoreFile: Path, rootStoreFile: Path, rootKeystorePass: String?, rootPrivateKeyPass: String?, keystorePass: String?, caPrivateKeyPass: String?) { + println("Generating intermediate and network map key pairs and certificates using root key store $rootStoreFile.") // Get password from console if not in config. - val rootKeystorePassword = rootKeystorePass ?: readPassword("Root Keystore Password: ") - val rootPrivateKeyPassword = rootPrivateKeyPass ?: readPassword("Root Private Key Password: ") + val rootKeystorePassword = rootKeystorePass ?: readPassword("Root key store password: ") + val rootPrivateKeyPassword = rootPrivateKeyPass ?: readPassword("Root private key password: ") val rootKeyStore = loadKeyStore(rootStoreFile, rootKeystorePassword) - val rootKeyAndCert = rootKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_ROOT_CA, rootPrivateKeyPassword) + val rootKeyPairAndCert = rootKeyStore.getCertificateAndKeyPair(X509Utilities.CORDA_ROOT_CA, rootPrivateKeyPassword) - val keystorePassword = keystorePass ?: readPassword("Keystore Password: ") - val caPrivateKeyPassword = caPrivateKeyPass ?: readPassword("CA Private Key Password: ") + val keyStorePassword = keystorePass ?: readPassword("Key store Password: ") + val privateKeyPassword = caPrivateKeyPass ?: readPassword("Private key Password: ") // Ensure folder exists. keystoreFile.parent.createDirectories() - val keyStore = loadOrCreateKeyStore(keystoreFile, keystorePassword) + val keyStore = loadOrCreateKeyStore(keystoreFile, keyStorePassword) - if (keyStore.containsAlias(X509Utilities.CORDA_INTERMEDIATE_CA)) { - val oldKey = loadOrCreateKeyStore(keystoreFile, rootKeystorePassword).getCertificate(X509Utilities.CORDA_INTERMEDIATE_CA).publicKey - println("Key ${X509Utilities.CORDA_INTERMEDIATE_CA} already exists in keystore, process will now terminate.") - println(oldKey) - exitProcess(1) + fun storeCertIfAbsent(alias: String, certificateType: CertificateType, subject: X500Principal, signatureScheme: SignatureScheme) { + if (keyStore.containsAlias(alias)) { + println("$alias already exists in keystore:") + println(keyStore.getCertificate(alias)) + return + } + + val keyPair = Crypto.generateKeyPair(signatureScheme) + val cert = X509Utilities.createCertificate( + certificateType, + rootKeyPairAndCert.certificate, + rootKeyPairAndCert.keyPair, + subject, + keyPair.public + ) + keyStore.addOrReplaceKey( + alias, + keyPair.private, + privateKeyPassword.toCharArray(), + arrayOf(cert, rootKeyPairAndCert.certificate) + ) + keyStore.save(keystoreFile, keyStorePassword) + + println("$certificateType key pair and certificate stored in $keystoreFile.") + println(cert) } - val intermediateKeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) - val intermediateCert = X509Utilities.createCertificate( + storeCertIfAbsent( + DEFAULT_CSR_CERTIFICATE_NAME, CertificateType.INTERMEDIATE_CA, - rootKeyAndCert.certificate, - rootKeyAndCert.keyPair, - CordaX500Name(commonName = "Corda Intermediate CA", organisation = "R3 Ltd", organisationUnit = "Corda", locality = "London", country = "GB", state = null).x500Principal, - intermediateKeyPair.public - ) - keyStore.addOrReplaceKey( - X509Utilities.CORDA_INTERMEDIATE_CA, - intermediateKeyPair.private, - caPrivateKeyPassword.toCharArray(), - arrayOf(intermediateCert, rootKeyAndCert.certificate) - ) - keyStore.save(keystoreFile, keystorePassword) - println("Intermediate CA keypair and certificate stored in $keystoreFile.") - println(loadKeyStore(keystoreFile, keystorePassword).getCertificate(X509Utilities.CORDA_INTERMEDIATE_CA).publicKey) + X500Principal("CN=Corda Intermediate CA,OU=Corda,O=R3 Ltd,L=London,C=GB"), + X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) + + storeCertIfAbsent( + DEFAULT_NETWORK_MAP_CERTIFICATE_NAME, + CertificateType.NETWORK_MAP, + X500Principal("CN=Corda Network Map,OU=Corda,O=R3 Ltd,L=London,C=GB"), + Crypto.EDDSA_ED25519_SHA512) } -private fun buildLocalSigner(parameters: NetworkManagementServerParameters): LocalSigner? { - return parameters.keystorePath?.let { - // Get password from console if not in config. - val keystorePassword = parameters.keystorePassword ?: readPassword("Keystore Password: ") - val caPrivateKeyPassword = parameters.caPrivateKeyPassword ?: readPassword("CA Private Key Password: ") - val keystore = loadOrCreateKeyStore(parameters.keystorePath, keystorePassword) - val caKeyPair = keystore.getKeyPair(X509Utilities.CORDA_INTERMEDIATE_CA, caPrivateKeyPassword) - val caCertPath = keystore.getCertificateChain(X509Utilities.CORDA_INTERMEDIATE_CA).map { it as X509Certificate } - LocalSigner(caKeyPair, caCertPath.toTypedArray()) +private fun buildLocalSigners(parameters: NetworkManagementServerParameters): Pair? { + if (parameters.keystorePath == null) return null + + // Get password from console if not in config. + val keyStorePassword = parameters.keystorePassword ?: readPassword("Key store password: ") + val privateKeyPassword = parameters.caPrivateKeyPassword ?: readPassword("Private key password: ") + val keyStore = loadOrCreateKeyStore(parameters.keystorePath, keyStorePassword) + + val (doormanSigner, networkMapSigner) = listOf(DEFAULT_CSR_CERTIFICATE_NAME, DEFAULT_NETWORK_MAP_CERTIFICATE_NAME).map { + val keyPair = keyStore.getKeyPair(it, privateKeyPassword) + val certPath = keyStore.getCertificateChain(it).map { it as X509Certificate } + LocalSigner(keyPair, certPath.toTypedArray()) } + + return Pair(doormanSigner, networkMapSigner) } /** @@ -278,7 +302,7 @@ fun main(args: Array) { rootStorePath ?: throw IllegalArgumentException("The 'rootStorePath' parameter must be specified when generating keys!"), rootKeystorePassword, rootPrivateKeyPassword) - Mode.CA_KEYGEN -> generateCAKeyPair( + Mode.CA_KEYGEN -> generateSigningKeyPairs( keystorePath ?: throw IllegalArgumentException("The 'keystorePath' parameter must be specified when generating keys!"), rootStorePath ?: throw IllegalArgumentException("The 'rootStorePath' parameter must be specified when generating keys!"), rootKeystorePassword, @@ -289,18 +313,24 @@ fun main(args: Array) { initialiseSerialization() val database = configureDatabase(dataSourceProperties) // TODO: move signing to signing server. - val signer = buildLocalSigner(this) + val localSigners = buildLocalSigners(this) - if (signer != null) { - println("Starting network management services with local signer.") + if (localSigners != null) { + println("Starting network management services with local signing") } val networkManagementServer = NetworkManagementServer() - val networkParameter = updateNetworkParameters?.let { - println("Parsing network parameter from '${it.fileName}'...") + val networkParameters = updateNetworkParameters?.let { + // TODO This check shouldn't be needed. Fix up the config design. + requireNotNull(networkMapConfig) { "'networkMapConfig' config is required for applying network parameters" } + println("Parsing network parameters from '${it.toAbsolutePath()}'...") parseNetworkParametersFrom(it) } - networkManagementServer.start(NetworkHostAndPort(host, port), database, signer, networkParameter, networkMapConfig, doormanConfig) + val networkMapStartParams = networkMapConfig?.let { + NetworkMapStartParams(localSigners?.second, networkParameters, it) + } + + networkManagementServer.start(NetworkHostAndPort(host, port), database, localSigners?.first, doormanConfig, networkMapStartParams) Runtime.getRuntime().addShutdownHook(thread(start = false) { networkManagementServer.close() diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/signer/LocalSigner.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/signer/LocalSigner.kt index aa7ceaa742..18343d7e44 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/signer/LocalSigner.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/signer/LocalSigner.kt @@ -2,11 +2,10 @@ package com.r3.corda.networkmanage.doorman.signer import com.r3.corda.networkmanage.common.signer.Signer import com.r3.corda.networkmanage.common.utils.buildCertPath -import com.r3.corda.networkmanage.common.utils.withCert -import net.corda.core.crypto.sign +import net.corda.core.crypto.Crypto +import net.corda.core.internal.DigitalSignatureWithCert import net.corda.nodeapi.internal.crypto.CertificateType import net.corda.nodeapi.internal.crypto.X509Utilities -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert import org.bouncycastle.asn1.x509.GeneralName import org.bouncycastle.asn1.x509.GeneralSubtree import org.bouncycastle.asn1.x509.NameConstraints @@ -21,7 +20,9 @@ import javax.security.auth.x500.X500Principal * The [LocalSigner] class signs [PKCS10CertificationRequest] using provided CA key pair and certificate path. * This is intended to be used in testing environment where hardware signing module is not available. */ -class LocalSigner(private val caKeyPair: KeyPair, private val caCertPath: Array) : Signer { +//TODO Use a list instead of array +class LocalSigner(private val signingKeyPair: KeyPair, private val signingCertPath: Array) : Signer { + // TODO This doesn't belong in this class fun createSignedClientCertificate(certificationRequest: PKCS10CertificationRequest): CertPath { // The sub certs issued by the client must satisfy this directory name (or legal name in Corda) constraints, sub certs' directory name must be within client CA's name's subtree, // please see [sun.security.x509.X500Name.isWithinSubtree()] for more information. @@ -33,15 +34,15 @@ class LocalSigner(private val caKeyPair: KeyPair, private val caCertPath: Array< arrayOf()) val nodeCaCert = X509Utilities.createCertificate( CertificateType.NODE_CA, - caCertPath[0], - caKeyPair, + signingCertPath[0], + signingKeyPair, X500Principal(request.subject.encoded), request.publicKey, nameConstraints = nameConstraints) - return buildCertPath(nodeCaCert, *caCertPath) + return buildCertPath(nodeCaCert, *signingCertPath) } - override fun sign(data: ByteArray): DigitalSignatureWithCert { - return caKeyPair.sign(data).withCert(caCertPath.first()) + override fun signBytes(data: ByteArray): DigitalSignatureWithCert { + return DigitalSignatureWithCert(signingCertPath[0], Crypto.doSign(signingKeyPair.private, data)) } } diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/webservice/NodeInfoWebService.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/webservice/NodeInfoWebService.kt index 2ac26743e2..0b1f1e57f6 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/webservice/NodeInfoWebService.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/webservice/NodeInfoWebService.kt @@ -6,6 +6,7 @@ import com.google.common.cache.LoadingCache import com.r3.corda.networkmanage.common.persistence.NetworkMapStorage import com.r3.corda.networkmanage.common.persistence.NodeInfoStorage import com.r3.corda.networkmanage.common.persistence.NodeInfoWithSigned +import com.r3.corda.networkmanage.common.utils.SignedNetworkMap import com.r3.corda.networkmanage.doorman.NetworkMapConfig import com.r3.corda.networkmanage.doorman.webservice.NodeInfoWebService.Companion.NETWORK_MAP_PATH import net.corda.core.crypto.SecureHash @@ -15,7 +16,6 @@ import net.corda.core.serialization.serialize import net.corda.core.utilities.contextLogger import net.corda.nodeapi.internal.SignedNodeInfo import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap import java.io.InputStream import java.security.InvalidKeyException import java.security.SignatureException @@ -77,9 +77,9 @@ class NodeInfoWebService(private val nodeInfoStorage: NodeInfoStorage, } @GET - @Path("network-parameter/{netParamsHash}") // TODO Fix path to be /network-parameters - fun getNetworkParameters(@PathParam("netParamsHash") netParamsHash: String): Response { - val signedNetParams = networkMapStorage.getSignedNetworkParameters(SecureHash.parse(netParamsHash)) + @Path("network-parameters/{hash}") + fun getNetworkParameters(@PathParam("hash") hash: String): Response { + val signedNetParams = networkMapStorage.getSignedNetworkParameters(SecureHash.parse(hash)) return createResponse(signedNetParams) } diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/configuration/Configuration.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/configuration/Configuration.kt index 6986f3a862..09475b4910 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/configuration/Configuration.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/configuration/Configuration.kt @@ -48,7 +48,7 @@ data class Parameters(val dataSourceProperties: Properties, val DEFAULT_KEY_FILE_PATH: Path? = null //Paths.get("/Users/michalkit/WinDev1706Eval/Shared/TEST4.key") val DEFAULT_KEY_FILE_PASSWORD: String? = null val DEFAULT_AUTO_USERNAME: String? = null - val DEFAULT_NETWORK_MAP_CERTIFICATE_NAME = "cordaintermediateca_nm" + val DEFAULT_NETWORK_MAP_CERTIFICATE_NAME = "cordaintermediateca_nm" // TODO Change the value to "cordanetworkmap" since this is not a CA val DEFAULT_SIGN_INTERVAL = 600L // in seconds (10 minutes) } } diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/signer/HsmNetworkMapSigner.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/signer/HsmNetworkMapSigner.kt index 0465a3621a..322233220f 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/signer/HsmNetworkMapSigner.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/signer/HsmNetworkMapSigner.kt @@ -4,16 +4,15 @@ import com.google.common.util.concurrent.MoreExecutors import com.r3.corda.networkmanage.common.persistence.NetworkMapStorage import com.r3.corda.networkmanage.common.signer.NetworkMapSigner import com.r3.corda.networkmanage.common.signer.Signer -import com.r3.corda.networkmanage.common.utils.withCert import com.r3.corda.networkmanage.hsm.authentication.Authenticator +import com.r3.corda.networkmanage.hsm.utils.X509Utilities import com.r3.corda.networkmanage.hsm.utils.X509Utilities.getAndInitializeKeyStore -import com.r3.corda.networkmanage.hsm.utils.X509Utilities.signData import com.r3.corda.networkmanage.hsm.utils.X509Utilities.verify +import net.corda.core.internal.DigitalSignatureWithCert import net.corda.core.utilities.loggerFor import net.corda.core.utilities.minutes -import net.corda.nodeapi.internal.network.DigitalSignatureWithCert -import java.security.KeyPair import java.security.PrivateKey +import java.security.Signature import java.security.cert.X509Certificate import java.time.Duration import java.util.concurrent.Executors @@ -24,6 +23,7 @@ import java.util.concurrent.TimeUnit * Encapsulates logic for periodic network map signing execution. * It uses HSM as the signing entity with keys and certificates specified at the construction time. */ +// TODO Rename this to HsmSigner class HsmNetworkMapSigner(networkMapStorage: NetworkMapStorage, private val caCertificateKeyName: String, private val caPrivateKeyPass: String, @@ -40,6 +40,7 @@ class HsmNetworkMapSigner(networkMapStorage: NetworkMapStorage, private val networkMapSigner = NetworkMapSigner(networkMapStorage, this) private lateinit var scheduledExecutor: ScheduledExecutorService + // TODO This doesn't belong in this class fun start(): HsmNetworkMapSigner { val signingPeriodMillis = signingPeriod.toMillis() scheduledExecutor = Executors.newSingleThreadScheduledExecutor() @@ -60,14 +61,18 @@ class HsmNetworkMapSigner(networkMapStorage: NetworkMapStorage, /** * Signs given data using [CryptoServerJCE.CryptoServerProvider], which connects to the underlying HSM. */ - override fun sign(data: ByteArray): DigitalSignatureWithCert { + override fun signBytes(data: ByteArray): DigitalSignatureWithCert { return authenticator.connectAndAuthenticate { provider, _ -> val keyStore = getAndInitializeKeyStore(provider) val caCertificateChain = keyStore.getCertificateChain(caCertificateKeyName) val caKey = keyStore.getKey(caCertificateKeyName, caPrivateKeyPass.toCharArray()) as PrivateKey - val signature = signData(data, KeyPair(caCertificateChain.first().publicKey, caKey), provider) - verify(data, signature, caCertificateChain.first().publicKey) - signature.withCert(caCertificateChain[0] as X509Certificate) + val signature = Signature.getInstance(X509Utilities.SIGNATURE_ALGORITHM, provider).run { + initSign(caKey) + update(data) + sign() + } + verify(data, signature, caCertificateChain[0].publicKey) + DigitalSignatureWithCert(caCertificateChain[0] as X509Certificate, signature) } } } diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/utils/X509Utils.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/utils/X509Utils.kt index dc4354fe0b..9bdd397e46 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/utils/X509Utils.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/hsm/utils/X509Utils.kt @@ -1,7 +1,6 @@ package com.r3.corda.networkmanage.hsm.utils import CryptoServerJCE.CryptoServerProvider -import net.corda.core.crypto.DigitalSignature import net.corda.core.identity.CordaX500Name import net.corda.core.internal.x500Name import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair @@ -268,27 +267,14 @@ object X509Utilities { } } - /** - * Sign data with the given private key - */ - fun signData(data: ByteArray, - keyPair: KeyPair, - provider: Provider, - signatureAlgorithm: String = SIGNATURE_ALGORITHM): DigitalSignature.WithKey { - val signer = Signature.getInstance(signatureAlgorithm, provider) - signer.initSign(keyPair.private) - signer.update(data) - return DigitalSignature.WithKey(keyPair.public, signer.sign()) - } - fun verify(data: ByteArray, - signature: DigitalSignature, + signature: ByteArray, publicKey: PublicKey, signatureAlgorithm: String = SIGNATURE_ALGORITHM) { val verify = Signature.getInstance(signatureAlgorithm) verify.initVerify(publicKey) verify.update(data) - require(verify.verify(signature.bytes)) { "Signature didn't independently verify" } + require(verify.verify(signature)) { "Signature didn't independently verify" } } /** diff --git a/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorageTest.kt b/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorageTest.kt index 1ca4abd931..01e95a7277 100644 --- a/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorageTest.kt +++ b/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorageTest.kt @@ -1,16 +1,14 @@ package com.r3.corda.networkmanage.common.persistence import com.r3.corda.networkmanage.TestBase -import com.r3.corda.networkmanage.common.utils.withCert import com.r3.corda.networkmanage.doorman.signer.LocalSigner -import net.corda.core.crypto.sign import net.corda.core.identity.CordaX500Name -import net.corda.core.serialization.serialize -import net.corda.nodeapi.internal.SignedNodeInfo +import net.corda.core.internal.signWithCert +import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair import net.corda.nodeapi.internal.crypto.X509CertificateFactory import net.corda.nodeapi.internal.network.NetworkMap -import net.corda.nodeapi.internal.network.SignedNetworkMap +import net.corda.nodeapi.internal.network.verifiedNetworkMapCert import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.testing.common.internal.testNetworkParameters @@ -31,15 +29,15 @@ class PersistentNetworkMapStorageTest : TestBase() { private lateinit var requestStorage: PersistentCertificateRequestStorage private lateinit var rootCaCert: X509Certificate - private lateinit var intermediateCa: CertificateAndKeyPair + private lateinit var networkMapCa: CertificateAndKeyPair @Before fun startDb() { - val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() + val (rootCa) = createDevIntermediateCaCertPath() rootCaCert = rootCa.certificate - this.intermediateCa = intermediateCa + networkMapCa = createDevNetworkMapCa(rootCa) persistence = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(runMigration = true)) - networkMapStorage = PersistentNetworkMapStorage(persistence, LocalSigner(intermediateCa.keyPair, arrayOf(intermediateCa.certificate, rootCaCert))) + networkMapStorage = PersistentNetworkMapStorage(persistence, LocalSigner(networkMapCa.keyPair, arrayOf(networkMapCa.certificate, rootCaCert))) nodeInfoStorage = PersistentNodeInfoStorage(persistence) requestStorage = PersistentCertificateRequestStorage(persistence) } @@ -60,9 +58,7 @@ class PersistentNetworkMapStorageTest : TestBase() { val networkParametersHash = networkMapStorage.saveNetworkParameters(testNetworkParameters(emptyList())) val networkMap = NetworkMap(listOf(nodeInfoHash), networkParametersHash) - val serializedNetworkMap = networkMap.serialize() - val signatureData = intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate) - val signedNetworkMap = SignedNetworkMap(serializedNetworkMap, signatureData) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) // when networkMapStorage.saveNetworkMap(signedNetworkMap) @@ -70,8 +66,8 @@ class PersistentNetworkMapStorageTest : TestBase() { // then val persistedSignedNetworkMap = networkMapStorage.getCurrentNetworkMap() - assertEquals(signedNetworkMap.signature, persistedSignedNetworkMap?.signature) - assertEquals(signedNetworkMap.verified(rootCaCert), persistedSignedNetworkMap?.verified(rootCaCert)) + assertEquals(signedNetworkMap.sig, persistedSignedNetworkMap?.sig) + assertEquals(signedNetworkMap.verifiedNetworkMapCert(rootCaCert), persistedSignedNetworkMap?.verifiedNetworkMapCert(rootCaCert)) } @Test @@ -96,9 +92,7 @@ class PersistentNetworkMapStorageTest : TestBase() { // Sign network map making it current network map val networkMap = NetworkMap(emptyList(), networkParametersHash) - val serializedNetworkMap = networkMap.serialize() - val signatureData = intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate) - val signedNetworkMap = SignedNetworkMap(serializedNetworkMap, signatureData) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) networkMapStorage.saveNetworkMap(signedNetworkMap) // Create new network parameters @@ -114,11 +108,11 @@ class PersistentNetworkMapStorageTest : TestBase() { // This test will probably won't be needed when we remove the explicit use of LocalSigner @Test fun `getSignedNetworkParameters uses the local signer to return a signed object`() { - val netParams = testNetworkParameters(emptyList()) - val netParamsHash = networkMapStorage.saveNetworkParameters(netParams) - val signedNetParams = networkMapStorage.getSignedNetworkParameters(netParamsHash) - assertThat(signedNetParams?.verified()).isEqualTo(netParams) - assertThat(signedNetParams?.sig?.by).isEqualTo(intermediateCa.keyPair.public) + val networkParameters = testNetworkParameters(emptyList()) + val netParamsHash = networkMapStorage.saveNetworkParameters(networkParameters) + val signedNetworkParameters = networkMapStorage.getSignedNetworkParameters(netParamsHash) + assertThat(signedNetworkParameters?.verifiedNetworkMapCert(rootCaCert)).isEqualTo(networkParameters) + assertThat(signedNetworkParameters?.sig?.by).isEqualTo(networkMapCa.certificate) } @Test @@ -135,9 +129,7 @@ class PersistentNetworkMapStorageTest : TestBase() { // Create network parameters val networkParametersHash = networkMapStorage.saveNetworkParameters(testNetworkParameters(emptyList())) val networkMap = NetworkMap(listOf(nodeInfoHashA), networkParametersHash) - val serializedNetworkMap = networkMap.serialize() - val signatureData = intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate) - val signedNetworkMap = SignedNetworkMap(serializedNetworkMap, signatureData) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) // Sign network map networkMapStorage.saveNetworkMap(signedNetworkMap) diff --git a/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSignerTest.kt b/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSignerTest.kt index 71926c7fac..014fd0895e 100644 --- a/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSignerTest.kt +++ b/network-management/src/test/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSignerTest.kt @@ -3,21 +3,24 @@ package com.r3.corda.networkmanage.common.signer import com.nhaarman.mockito_kotlin.* import com.r3.corda.networkmanage.TestBase import com.r3.corda.networkmanage.common.persistence.NetworkMapStorage -import com.r3.corda.networkmanage.common.utils.withCert +import com.r3.corda.networkmanage.common.utils.SignedNetworkMap +import net.corda.core.crypto.Crypto import net.corda.core.crypto.SecureHash import net.corda.core.crypto.sha256 -import net.corda.core.crypto.sign +import net.corda.core.internal.DigitalSignatureWithCert +import net.corda.core.internal.signWithCert import net.corda.core.serialization.serialize +import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair import net.corda.nodeapi.internal.network.NetworkMap -import net.corda.nodeapi.internal.network.SignedNetworkMap +import net.corda.nodeapi.internal.network.verifiedNetworkMapCert import net.corda.testing.common.internal.testNetworkParameters import net.corda.testing.internal.createDevIntermediateCaCertPath +import org.assertj.core.api.Assertions.assertThat import org.junit.Before import org.junit.Test import java.security.cert.X509Certificate import kotlin.test.assertEquals -import kotlin.test.assertTrue class NetworkMapSignerTest : TestBase() { private lateinit var signer: Signer @@ -25,13 +28,13 @@ class NetworkMapSignerTest : TestBase() { private lateinit var networkMapSigner: NetworkMapSigner private lateinit var rootCaCert: X509Certificate - private lateinit var intermediateCa: CertificateAndKeyPair + private lateinit var networkMapCa: CertificateAndKeyPair @Before fun setUp() { - val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() + val (rootCa) = createDevIntermediateCaCertPath() rootCaCert = rootCa.certificate - this.intermediateCa = intermediateCa + networkMapCa = createDevNetworkMapCa(rootCa) signer = mock() networkMapStorage = mock() networkMapSigner = NetworkMapSigner(networkMapStorage, signer) @@ -42,13 +45,13 @@ class NetworkMapSignerTest : TestBase() { // given val signedNodeInfoHashes = listOf(SecureHash.randomSHA256(), SecureHash.randomSHA256()) val networkParameters = testNetworkParameters(emptyList()) - val serializedNetworkMap = NetworkMap(signedNodeInfoHashes, SecureHash.randomSHA256()).serialize() - whenever(networkMapStorage.getCurrentNetworkMap()) - .thenReturn(SignedNetworkMap(serializedNetworkMap, intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate))) + val networkMap = NetworkMap(signedNodeInfoHashes, SecureHash.randomSHA256()) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) + whenever(networkMapStorage.getCurrentNetworkMap()).thenReturn(signedNetworkMap) whenever(networkMapStorage.getNodeInfoHashes(any())).thenReturn(signedNodeInfoHashes) whenever(networkMapStorage.getLatestNetworkParameters()).thenReturn(networkParameters) - whenever(signer.sign(any())).then { - intermediateCa.keyPair.sign(it.arguments[0] as ByteArray).withCert(intermediateCa.certificate) + whenever(signer.signBytes(any())).then { + DigitalSignatureWithCert(networkMapCa.certificate, Crypto.doSign(networkMapCa.keyPair.private, it.arguments[0] as ByteArray)) } // when @@ -60,10 +63,10 @@ class NetworkMapSignerTest : TestBase() { verify(networkMapStorage).getLatestNetworkParameters() argumentCaptor().apply { verify(networkMapStorage).saveNetworkMap(capture()) - val networkMap = firstValue.verified(rootCaCert) - assertEquals(networkParameters.serialize().hash, networkMap.networkParameterHash) - assertEquals(signedNodeInfoHashes.size, networkMap.nodeInfoHashes.size) - assertTrue(networkMap.nodeInfoHashes.containsAll(signedNodeInfoHashes)) + val capturedNetworkMap = firstValue.verifiedNetworkMapCert(rootCaCert) + assertEquals(networkParameters.serialize().hash, capturedNetworkMap.networkParameterHash) + assertEquals(signedNodeInfoHashes.size, capturedNetworkMap.nodeInfoHashes.size) + assertThat(capturedNetworkMap.nodeInfoHashes).containsAll(signedNodeInfoHashes) } } @@ -73,8 +76,7 @@ class NetworkMapSignerTest : TestBase() { val networkParameters = testNetworkParameters(emptyList()) val networkMapParametersHash = networkParameters.serialize().bytes.sha256() val networkMap = NetworkMap(emptyList(), networkMapParametersHash) - val serializedNetworkMap = networkMap.serialize() - val signedNetworkMap = SignedNetworkMap(serializedNetworkMap, intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate)) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) whenever(networkMapStorage.getCurrentNetworkMap()).thenReturn(signedNetworkMap) whenever(networkMapStorage.getNodeInfoHashes(any())).thenReturn(emptyList()) whenever(networkMapStorage.getLatestNetworkParameters()).thenReturn(networkParameters) @@ -94,8 +96,8 @@ class NetworkMapSignerTest : TestBase() { whenever(networkMapStorage.getCurrentNetworkMap()).thenReturn(null) whenever(networkMapStorage.getNodeInfoHashes(any())).thenReturn(emptyList()) whenever(networkMapStorage.getLatestNetworkParameters()).thenReturn(networkParameters) - whenever(signer.sign(any())).then { - intermediateCa.keyPair.sign(it.arguments[0] as ByteArray).withCert(intermediateCa.certificate) + whenever(signer.signBytes(any())).then { + DigitalSignatureWithCert(networkMapCa.certificate, Crypto.doSign(networkMapCa.keyPair.private, it.arguments[0] as ByteArray)) } // when networkMapSigner.signNetworkMap() @@ -106,7 +108,7 @@ class NetworkMapSignerTest : TestBase() { verify(networkMapStorage).getLatestNetworkParameters() argumentCaptor().apply { verify(networkMapStorage).saveNetworkMap(capture()) - val networkMap = firstValue.verified(rootCaCert) + val networkMap = firstValue.verifiedNetworkMapCert(rootCaCert) assertEquals(networkParameters.serialize().hash, networkMap.networkParameterHash) } } diff --git a/network-management/src/test/kotlin/com/r3/corda/networkmanage/doorman/NodeInfoWebServiceTest.kt b/network-management/src/test/kotlin/com/r3/corda/networkmanage/doorman/NodeInfoWebServiceTest.kt index b1f0e320db..f615d8d7ff 100644 --- a/network-management/src/test/kotlin/com/r3/corda/networkmanage/doorman/NodeInfoWebServiceTest.kt +++ b/network-management/src/test/kotlin/com/r3/corda/networkmanage/doorman/NodeInfoWebServiceTest.kt @@ -5,35 +5,33 @@ import com.nhaarman.mockito_kotlin.times import com.nhaarman.mockito_kotlin.verify import com.r3.corda.networkmanage.common.persistence.NetworkMapStorage import com.r3.corda.networkmanage.common.persistence.NodeInfoStorage -import com.r3.corda.networkmanage.common.utils.withCert +import com.r3.corda.networkmanage.common.utils.SignedNetworkMap +import com.r3.corda.networkmanage.common.utils.SignedNetworkParameters import com.r3.corda.networkmanage.doorman.webservice.NodeInfoWebService import net.corda.core.crypto.SecureHash.Companion.randomSHA256 -import net.corda.core.crypto.SignedData -import net.corda.core.crypto.sign import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.checkOkResponse import net.corda.core.internal.openHttpConnection -import net.corda.core.serialization.deserialize +import net.corda.core.internal.responseAs +import net.corda.core.internal.signWithCert import net.corda.core.serialization.serialize import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.seconds import net.corda.nodeapi.internal.SignedNodeInfo +import net.corda.nodeapi.internal.createDevNetworkMapCa import net.corda.nodeapi.internal.crypto.CertificateAndKeyPair import net.corda.nodeapi.internal.network.NetworkMap -import net.corda.nodeapi.internal.network.NetworkParameters -import net.corda.nodeapi.internal.network.SignedNetworkMap +import net.corda.nodeapi.internal.network.verifiedNetworkMapCert import net.corda.testing.SerializationEnvironmentRule import net.corda.testing.common.internal.testNetworkParameters import net.corda.testing.internal.createDevIntermediateCaCertPath import net.corda.testing.internal.createNodeInfoAndSigned -import org.apache.commons.io.IOUtils import org.assertj.core.api.Assertions.* import org.junit.Before import org.junit.Rule import org.junit.Test -import java.io.FileNotFoundException import java.io.IOException import java.net.URL -import java.nio.charset.Charset import java.security.cert.X509Certificate import javax.ws.rs.core.MediaType import kotlin.test.assertEquals @@ -44,15 +42,15 @@ class NodeInfoWebServiceTest { val testSerialization = SerializationEnvironmentRule(true) private lateinit var rootCaCert: X509Certificate - private lateinit var intermediateCa: CertificateAndKeyPair + private lateinit var networkMapCa: CertificateAndKeyPair private val testNetworkMapConfig = NetworkMapConfig(10.seconds.toMillis(), 10.seconds.toMillis()) @Before fun init() { - val (rootCa, intermediateCa) = createDevIntermediateCaCertPath() + val (rootCa) = createDevIntermediateCaCertPath() rootCaCert = rootCa.certificate - this.intermediateCa = intermediateCa + networkMapCa = createDevNetworkMapCa(rootCa) } @Test @@ -106,8 +104,7 @@ class NodeInfoWebServiceTest { @Test fun `get network map`() { val networkMap = NetworkMap(listOf(randomSHA256(), randomSHA256()), randomSHA256()) - val serializedNetworkMap = networkMap.serialize() - val signedNetworkMap = SignedNetworkMap(serializedNetworkMap, intermediateCa.keyPair.sign(serializedNetworkMap).withCert(intermediateCa.certificate)) + val signedNetworkMap = networkMap.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) val networkMapStorage: NetworkMapStorage = mock { on { getCurrentNetworkMap() }.thenReturn(signedNetworkMap) @@ -117,7 +114,7 @@ class NodeInfoWebServiceTest { it.start() val signedNetworkMapResponse = it.doGet("") verify(networkMapStorage, times(1)).getCurrentNetworkMap() - assertEquals(signedNetworkMapResponse.verified(rootCaCert), networkMap) + assertEquals(signedNetworkMapResponse.verifiedNetworkMapCert(rootCaCert), networkMap) } } @@ -136,33 +133,32 @@ class NodeInfoWebServiceTest { verify(nodeInfoStorage, times(1)).getNodeInfo(nodeInfoHash) assertEquals(nodeInfo, nodeInfoResponse.verified()) - assertThatExceptionOfType(FileNotFoundException::class.java).isThrownBy { - it.doGet("node-info/${randomSHA256()}") - } + assertThatExceptionOfType(IOException::class.java) + .isThrownBy { it.doGet("node-info/${randomSHA256()}") } + .withMessageContaining("404") } } @Test fun `get network parameters`() { - val netParams = testNetworkParameters(emptyList()) - val serializedNetParams = netParams.serialize() - val signedNetParams = SignedData(serializedNetParams, intermediateCa.keyPair.sign(serializedNetParams)) - val netParamsHash = serializedNetParams.hash + val networkParameters = testNetworkParameters(emptyList()) + val signedNetworkParameters = networkParameters.signWithCert(networkMapCa.keyPair.private, networkMapCa.certificate) + val networkParametersHash = signedNetworkParameters.raw.hash val networkMapStorage: NetworkMapStorage = mock { - on { getSignedNetworkParameters(netParamsHash) }.thenReturn(signedNetParams) + on { getSignedNetworkParameters(networkParametersHash) }.thenReturn(signedNetworkParameters) } NetworkManagementWebServer(NetworkHostAndPort("localhost", 0), NodeInfoWebService(mock(), networkMapStorage, testNetworkMapConfig)).use { it.start() - val netParamsResponse = it.doGet>("network-parameter/$netParamsHash") - verify(networkMapStorage, times(1)).getSignedNetworkParameters(netParamsHash) - assertThat(netParamsResponse.verified()).isEqualTo(netParams) - assertThat(netParamsResponse.sig.by).isEqualTo(intermediateCa.keyPair.public) + val netParamsResponse = it.doGet("network-parameters/$networkParametersHash") + verify(networkMapStorage, times(1)).getSignedNetworkParameters(networkParametersHash) + assertThat(netParamsResponse.verified()).isEqualTo(networkParameters) + assertThat(netParamsResponse.sig.by).isEqualTo(networkMapCa.certificate) - assertThatExceptionOfType(FileNotFoundException::class.java).isThrownBy { - it.doGet>("network-parameter/${randomSHA256()}") - } + assertThatExceptionOfType(IOException::class.java) + .isThrownBy { it.doGet("network-parameters/${randomSHA256()}") } + .withMessageContaining("404") } } @@ -173,15 +169,11 @@ class NodeInfoWebServiceTest { requestMethod = "POST" setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM) outputStream.write(payload) - if (responseCode != 200) { - throw IOException("Response Code $responseCode: ${IOUtils.toString(errorStream, Charset.defaultCharset())}") - } - inputStream.close() + checkOkResponse() } } private inline fun NetworkManagementWebServer.doGet(path: String): T { - val url = URL("http://$hostAndPort/network-map/$path") - return url.openHttpConnection().inputStream.use { it.readBytes().deserialize() } + return URL("http://$hostAndPort/network-map/$path").openHttpConnection().responseAs() } } \ No newline at end of file