From 69bfc9b437eec955069c0ba85d754a4fa9c4981b Mon Sep 17 00:00:00 2001 From: Andras Slemmer Date: Mon, 18 Dec 2017 10:11:35 +0000 Subject: [PATCH 01/12] Comment out lines breaking IRS demo --- .../src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt b/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt index 9da48655a9..229daa9c5c 100644 --- a/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt +++ b/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt @@ -71,8 +71,8 @@ object AutoOfferFlow { // and because in a real life app you'd probably have more complex logic here e.g. describing why the report // was filed, checking that the reportee is a regulated entity and not some random node from the wrong // country and so on. - val regulator = serviceHub.identityService.partiesFromName("Regulator", true).single() - subFlow(ReportToRegulatorFlow(regulator, finalTx)) + // val regulator = serviceHub.identityService.partiesFromName("Regulator", true).single() + // subFlow(ReportToRegulatorFlow(regulator, finalTx)) return finalTx } From 58f76ee2da22d27ca11ee4124a8dfbc6182a612e Mon Sep 17 00:00:00 2001 From: Andras Slemmer Date: Fri, 15 Dec 2017 15:29:54 +0000 Subject: [PATCH 02/12] Put IRS spring whatever back on classpath --- samples/irs-demo/build.gradle | 2 + samples/irs-demo/web/build.gradle | 107 +++++++++++++++++------------- 2 files changed, 62 insertions(+), 47 deletions(-) diff --git a/samples/irs-demo/build.gradle b/samples/irs-demo/build.gradle index 5b5b51ecf9..8bfc3785db 100644 --- a/samples/irs-demo/build.gradle +++ b/samples/irs-demo/build.gradle @@ -53,6 +53,8 @@ dependencies { testCompile project(':node-driver') testCompile "junit:junit:$junit_version" testCompile "org.assertj:assertj-core:${assertj_version}" + + integrationTestCompile project(path: ":samples:irs-demo:web", configuration: "demoArtifacts") } bootRepackage { diff --git a/samples/irs-demo/web/build.gradle b/samples/irs-demo/web/build.gradle index 78450b877d..bad8f8da07 100644 --- a/samples/irs-demo/web/build.gradle +++ b/samples/irs-demo/web/build.gradle @@ -1,34 +1,34 @@ buildscript { - repositories { - mavenCentral() - } - dependencies { - classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") - classpath("org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}") - } + repositories { + mavenCentral() + } + dependencies { + classpath("org.springframework.boot:spring-boot-gradle-plugin:${springBootVersion}") + classpath("org.jetbrains.kotlin:kotlin-allopen:${kotlin_version}") + } } plugins { - id 'com.craigburke.client-dependencies' version '1.4.0' + id 'com.craigburke.client-dependencies' version '1.4.0' } clientDependencies { - registry 'realBower', type:'bower', url:'https://registry.bower.io' - realBower { - "angular"("1.5.8") - "jquery"("^3.0.0") - "angular-route"("1.5.8") - "lodash"("^4.13.1") - "angular-fcsa-number"("^1.5.3") - "jquery.maskedinput"("^1.4.1") - "requirejs"("^2.2.0") - "semantic-ui"("^2.2.2", into: "semantic") - } + registry 'realBower', type:'bower', url:'https://registry.bower.io' + realBower { + "angular"("1.5.8") + "jquery"("^3.0.0") + "angular-route"("1.5.8") + "lodash"("^4.13.1") + "angular-fcsa-number"("^1.5.3") + "jquery.maskedinput"("^1.4.1") + "requirejs"("^2.2.0") + "semantic-ui"("^2.2.2", into: "semantic") + } - // put the JS dependencies into src directory so it can easily be referenced - // from HTML files in webapp frontend, useful for testing/development - // Note that this dir is added to .gitignore - installDir = 'src/main/resources/static/js/bower_components' + // put the JS dependencies into src directory so it can easily be referenced + // from HTML files in webapp frontend, useful for testing/development + // Note that this dir is added to .gitignore + installDir = 'src/main/resources/static/js/bower_components' } // Spring Boot plugin adds a numerous hardcoded dependencies in the version much lower then Corda expects @@ -44,36 +44,49 @@ apply plugin: 'org.springframework.boot' apply plugin: 'project-report' apply plugin: 'application' +configurations { + demoArtifacts.extendsFrom testRuntime +} + dependencies { - compile('org.springframework.boot:spring-boot-starter-web') { - exclude module: "spring-boot-starter-logging" - exclude module: "logback-classic" - } - compile("com.fasterxml.jackson.module:jackson-module-kotlin:2.8.9") - compile project(":client:rpc") - compile project(":client:jackson") - compile project(":test-utils") - compile project(path: ":samples:irs-demo:cordapp", configuration: "demoArtifacts") - testCompile('org.springframework.boot:spring-boot-starter-test') { - exclude module: "spring-boot-starter-logging" - exclude module: "logback-classic" - } + compile('org.springframework.boot:spring-boot-starter-web') { + exclude module: "spring-boot-starter-logging" + exclude module: "logback-classic" + } + compile("com.fasterxml.jackson.module:jackson-module-kotlin:2.8.9") + compile project(":client:rpc") + compile project(":client:jackson") + compile project(":test-utils") + compile project(path: ":samples:irs-demo:cordapp", configuration: "demoArtifacts") + testCompile('org.springframework.boot:spring-boot-starter-test') { + exclude module: "spring-boot-starter-logging" + exclude module: "logback-classic" + } } jar { - from sourceSets.test.output - dependsOn clientInstall + from sourceSets.test.output + dependsOn clientInstall } task deployWebapps(type: Copy, dependsOn: ['jar', 'bootRepackage']) { - ext.webappDir = file("build/webapps") + ext.webappDir = file("build/webapps") - from(jar.outputs) - from("src/test/resources/scripts/") { - filter { it - .replace('#JAR_PATH#', jar.archiveName) - .replace('#DIR#', ext.webappDir.getAbsolutePath()) - } - } - into ext.webappDir + from(jar.outputs) + from("src/test/resources/scripts/") { + filter { it + .replace('#JAR_PATH#', jar.archiveName) + .replace('#DIR#', ext.webappDir.getAbsolutePath()) + } + } + into ext.webappDir +} + +task demoJar(type: Jar) { + classifier "test" + from sourceSets.test.output +} + +artifacts { + demoArtifacts demoJar } \ No newline at end of file From fca3d565acbdd0e4c036bc33cbd0c99eb491abda Mon Sep 17 00:00:00 2001 From: Andras Slemmer Date: Fri, 26 Jan 2018 14:28:24 +0000 Subject: [PATCH 03/12] Add 'Regulator' node for AutoOfferFlow, fixing IRS --- samples/irs-demo/cordapp/build.gradle | 8 ++++++++ .../src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt | 4 ++-- .../cordapp/src/test/kotlin/net/corda/irs/Main.kt | 4 +++- .../integration-test/kotlin/net/corda/irs/IRSDemoTest.kt | 4 +++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/samples/irs-demo/cordapp/build.gradle b/samples/irs-demo/cordapp/build.gradle index 015f87342a..3a2dab78d5 100644 --- a/samples/irs-demo/cordapp/build.gradle +++ b/samples/irs-demo/cordapp/build.gradle @@ -94,6 +94,14 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar']) { rpcUsers = ext.rpcUsers useTestClock true } + node { + name "O=Regulator,L=Moscow,C=RU" + p2pPort 10010 + rpcPort 10011 + cordapps = ["${project.group}:finance:$corda_release_version"] + rpcUsers = ext.rpcUsers + useTestClock true + } } task integrationTest(type: Test, dependsOn: []) { diff --git a/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt b/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt index 229daa9c5c..9da48655a9 100644 --- a/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt +++ b/samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt @@ -71,8 +71,8 @@ object AutoOfferFlow { // and because in a real life app you'd probably have more complex logic here e.g. describing why the report // was filed, checking that the reportee is a regulated entity and not some random node from the wrong // country and so on. - // val regulator = serviceHub.identityService.partiesFromName("Regulator", true).single() - // subFlow(ReportToRegulatorFlow(regulator, finalTx)) + val regulator = serviceHub.identityService.partiesFromName("Regulator", true).single() + subFlow(ReportToRegulatorFlow(regulator, finalTx)) return finalTx } diff --git a/samples/irs-demo/cordapp/src/test/kotlin/net/corda/irs/Main.kt b/samples/irs-demo/cordapp/src/test/kotlin/net/corda/irs/Main.kt index 8614e26249..c6de65501f 100644 --- a/samples/irs-demo/cordapp/src/test/kotlin/net/corda/irs/Main.kt +++ b/samples/irs-demo/cordapp/src/test/kotlin/net/corda/irs/Main.kt @@ -1,5 +1,6 @@ package net.corda.irs +import net.corda.core.identity.CordaX500Name import net.corda.core.utilities.getOrThrow import net.corda.testing.core.DUMMY_BANK_A_NAME import net.corda.testing.core.DUMMY_BANK_B_NAME @@ -13,7 +14,8 @@ fun main(args: Array) { driver(useTestClock = true, isDebug = true, waitForAllNodesToFinish = true) { val (nodeA, nodeB) = listOf( startNode(providedName = DUMMY_BANK_A_NAME), - startNode(providedName = DUMMY_BANK_B_NAME) + startNode(providedName = DUMMY_BANK_B_NAME), + startNode(providedName = CordaX500Name("Regulator", "Moscow", "RU")) ).map { it.getOrThrow() } val controller = defaultNotaryNode.getOrThrow() diff --git a/samples/irs-demo/src/integration-test/kotlin/net/corda/irs/IRSDemoTest.kt b/samples/irs-demo/src/integration-test/kotlin/net/corda/irs/IRSDemoTest.kt index 26877ebf93..9a4c73a772 100644 --- a/samples/irs-demo/src/integration-test/kotlin/net/corda/irs/IRSDemoTest.kt +++ b/samples/irs-demo/src/integration-test/kotlin/net/corda/irs/IRSDemoTest.kt @@ -11,6 +11,7 @@ import com.fasterxml.jackson.module.kotlin.readValue import net.corda.client.jackson.JacksonSupport import net.corda.client.rpc.CordaRPCClient import net.corda.core.contracts.UniqueIdentifier +import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party import net.corda.core.messaging.vaultTrackBy import net.corda.core.toFuture @@ -56,7 +57,8 @@ class IRSDemoTest { ) { val (nodeA, nodeB) = listOf( startNode(providedName = DUMMY_BANK_A_NAME, rpcUsers = rpcUsers), - startNode(providedName = DUMMY_BANK_B_NAME, rpcUsers = rpcUsers) + startNode(providedName = DUMMY_BANK_B_NAME, rpcUsers = rpcUsers), + startNode(providedName = CordaX500Name("Regulator", "Moscow", "RU")) ).map { it.getOrThrow() } val controller = defaultNotaryNode.getOrThrow() From 4257891c98a9a0bf115cbdd391534fead47ca438 Mon Sep 17 00:00:00 2001 From: Thomas Schroeter Date: Fri, 26 Jan 2018 16:23:59 +0000 Subject: [PATCH 04/12] Revert "Raft Notary: remove snapshotting" (#2423) This reverts commit cf33be66fffa5d547b6d2370d8dbf0fe4121d714. --- .../transactions/DistributedImmutableMap.kt | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/transactions/DistributedImmutableMap.kt b/node/src/main/kotlin/net/corda/node/services/transactions/DistributedImmutableMap.kt index 4e6bc0ac89..4cb2082fb4 100644 --- a/node/src/main/kotlin/net/corda/node/services/transactions/DistributedImmutableMap.kt +++ b/node/src/main/kotlin/net/corda/node/services/transactions/DistributedImmutableMap.kt @@ -3,7 +3,10 @@ package net.corda.node.services.transactions import io.atomix.copycat.Command import io.atomix.copycat.Query import io.atomix.copycat.server.Commit +import io.atomix.copycat.server.Snapshottable import io.atomix.copycat.server.StateMachine +import io.atomix.copycat.server.storage.snapshot.SnapshotReader +import io.atomix.copycat.server.storage.snapshot.SnapshotWriter import net.corda.core.utilities.contextLogger import net.corda.node.utilities.AppendOnlyPersistentMap import net.corda.nodeapi.internal.persistence.CordaPersistence @@ -16,7 +19,7 @@ import java.util.* * The map contents are backed by a JDBC table. State re-synchronisation is achieved by replaying the command log to the * new (or re-joining) cluster member. */ -class DistributedImmutableMap(val db: CordaPersistence, createMap: () -> AppendOnlyPersistentMap, E, EK>) : StateMachine() { +class DistributedImmutableMap(val db: CordaPersistence, createMap: () -> AppendOnlyPersistentMap, E, EK>) : StateMachine(), Snapshottable { companion object { private val log = contextLogger() } @@ -75,4 +78,29 @@ class DistributedImmutableMap(val db: CordaPersistence, return db.transaction { map.size } } } + + /** + * Writes out all [map] entries to disk. Note that this operation does not load all entries into memory, as the + * [SnapshotWriter] is using a disk-backed buffer internally, and iterating map entries results in only a + * fixed number of recently accessed entries to ever be kept in memory. + */ + override fun snapshot(writer: SnapshotWriter) { + db.transaction { + writer.writeInt(map.size) + map.allPersisted().forEach { writer.writeObject(it.first to it.second) } + } + } + + /** Reads entries from disk and adds them to [map]. */ + override fun install(reader: SnapshotReader) { + val size = reader.readInt() + db.transaction { + map.clear() + // TODO: read & put entries in batches + for (i in 1..size) { + val (key, value) = reader.readObject>>() + map[key] = value + } + } + } } From e19f51d9ac211301e0b31e3ab70484d986f7fc66 Mon Sep 17 00:00:00 2001 From: Rick Parker Date: Fri, 26 Jan 2018 17:44:42 +0000 Subject: [PATCH 05/12] CORDA-959 Filter unschedules and remove database activity from inside mutex. (#2426) * Filter unschedules and remove database activity from inside mutex. * Race condition fix * Bug fix --- .../services/events/NodeSchedulerService.kt | 21 +++++++++++-------- .../events/ScheduledActivityObserver.kt | 2 +- .../events/NodeSchedulerServiceTest.kt | 6 +++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt b/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt index 372ef392b8..72ef0fe6a6 100644 --- a/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt +++ b/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt @@ -61,7 +61,7 @@ class NodeSchedulerService(private val clock: CordaClock, private val serverThread: Executor, private val flowLogicRefFactory: FlowLogicRefFactory, private val log: Logger = staticLog, - scheduledStates: MutableMap = createMap()) + private val scheduledStates: MutableMap = createMap()) : SchedulerService, SingletonSerializeAsToken() { companion object { @@ -153,13 +153,13 @@ class NodeSchedulerService(private val clock: CordaClock, var scheduledAt: Instant = Instant.now() ) - private class InnerState(var scheduledStates: MutableMap) { + private class InnerState { var scheduledStatesQueue: PriorityQueue = PriorityQueue({ a, b -> a.scheduledAt.compareTo(b.scheduledAt) }) var rescheduled: GuavaSettableFuture? = null } - private val mutex = ThreadBox(InnerState(scheduledStates)) + private val mutex = ThreadBox(InnerState()) // We need the [StateMachineManager] to be constructed before this is called in case it schedules a flow. fun start() { mutex.locked { @@ -170,9 +170,9 @@ class NodeSchedulerService(private val clock: CordaClock, override fun scheduleStateActivity(action: ScheduledStateRef) { log.trace { "Schedule $action" } + val previousState = scheduledStates[action.ref] + scheduledStates[action.ref] = action mutex.locked { - val previousState = scheduledStates[action.ref] - scheduledStates[action.ref] = action val previousEarliest = scheduledStatesQueue.peek() scheduledStatesQueue.remove(previousState) scheduledStatesQueue.add(action) @@ -192,12 +192,15 @@ class NodeSchedulerService(private val clock: CordaClock, override fun unscheduleStateActivity(ref: StateRef) { log.trace { "Unschedule $ref" } + val removedAction = scheduledStates.remove(ref) mutex.locked { - val removedAction = scheduledStates.remove(ref) if (removedAction != null) { - scheduledStatesQueue.remove(removedAction) - unfinishedSchedules.countDown() - if (removedAction == scheduledStatesQueue.peek()) { + val wasNext = (removedAction == scheduledStatesQueue.peek()) + val wasRemoved = scheduledStatesQueue.remove(removedAction) + if (wasRemoved) { + unfinishedSchedules.countDown() + } + if (wasNext) { rescheduleWakeUp() } } diff --git a/node/src/main/kotlin/net/corda/node/services/events/ScheduledActivityObserver.kt b/node/src/main/kotlin/net/corda/node/services/events/ScheduledActivityObserver.kt index 2c3e939c98..7b19bd40e7 100644 --- a/node/src/main/kotlin/net/corda/node/services/events/ScheduledActivityObserver.kt +++ b/node/src/main/kotlin/net/corda/node/services/events/ScheduledActivityObserver.kt @@ -18,7 +18,7 @@ class ScheduledActivityObserver private constructor(private val schedulerService fun install(vaultService: VaultService, schedulerService: SchedulerService, flowLogicRefFactory: FlowLogicRefFactory) { val observer = ScheduledActivityObserver(schedulerService, flowLogicRefFactory) vaultService.rawUpdates.subscribe { (consumed, produced) -> - consumed.forEach { schedulerService.unscheduleStateActivity(it.ref) } + consumed.forEach { if (it.state.data is SchedulableState) schedulerService.unscheduleStateActivity(it.ref) } produced.forEach { observer.scheduleStateActivity(it) } } } diff --git a/node/src/test/kotlin/net/corda/node/services/events/NodeSchedulerServiceTest.kt b/node/src/test/kotlin/net/corda/node/services/events/NodeSchedulerServiceTest.kt index de82ec5bbf..6c7e766d37 100644 --- a/node/src/test/kotlin/net/corda/node/services/events/NodeSchedulerServiceTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/events/NodeSchedulerServiceTest.kt @@ -6,16 +6,16 @@ import net.corda.core.contracts.* import net.corda.core.flows.FlowLogic import net.corda.core.flows.FlowLogicRef import net.corda.core.flows.FlowLogicRefFactory -import net.corda.core.utilities.days -import net.corda.testing.internal.rigorousMock import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.concurrent.openFuture import net.corda.core.internal.uncheckedCast import net.corda.core.node.StateLoader +import net.corda.core.utilities.days import net.corda.node.services.api.FlowStarter import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.internal.doLookup +import net.corda.testing.internal.rigorousMock import net.corda.testing.node.TestClock import org.junit.Rule import org.junit.Test @@ -163,7 +163,7 @@ class NodeSchedulerServiceTest { val eventA = schedule(mark + 1.days) val eventB = schedule(mark + 1.days) scheduler.unscheduleStateActivity(eventA.stateRef) - assertWaitingFor(eventA) // XXX: Shouldn't it be waiting for eventB now? + assertWaitingFor(eventB) testClock.advanceBy(1.days) assertStarted(eventB) } From 4851d9ca6a5e5df331e9895feb45aece93ac83b2 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Mon, 29 Jan 2018 12:42:31 +0000 Subject: [PATCH 06/12] Documents rationale for using cordapp, cordaRuntime and cordaCompile Gradle configs. --- docs/source/cordapp-build-systems.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/source/cordapp-build-systems.rst b/docs/source/cordapp-build-systems.rst index 5cba71b9af..3900680d81 100644 --- a/docs/source/cordapp-build-systems.rst +++ b/docs/source/cordapp-build-systems.rst @@ -53,6 +53,10 @@ The ``cordformation`` plugin adds two new gradle configurations: * ``cordaCompile``, which extends ``compile`` * ``cordaRuntime``, which extends ``runtime`` +``cordaCompile`` and ``cordaRuntime`` indicate dependencies that should not be included in the CorDapp JAR. These +configurations should be used for any Corda dependency (e.g. ``corda-core``, ``corda-node``) in order to prevent a +dependency from being included twice (once in the CorDapp JAR and once in the Corda JARs). + To build against Corda, you must add the following to your ``build.gradle`` file: * ``net.corda:corda:$corda_release_version`` as a ``cordaRuntime`` dependency @@ -75,6 +79,15 @@ ways to add another CorDapp as a dependency in your CorDapp's ``build.gradle`` f * ``cordapp project(":another-cordapp")`` (use this if the other CorDapp is defined in a module in the same project) * ``cordapp "net.corda:another-cordapp:1.0"`` (use this otherwise) +The ``cordapp`` gradle configuration serves two purposes: + +* When using the ``cordformation`` Gradle plugin, the ``cordapp`` configuration indicates that this JAR should be + included on your node as a CorDapp +* When using the ``cordapp`` Gradle plugin, the ``cordapp`` configuration prevents the dependency from being included + in the CorDapp JAR + +Note that the ``cordformation`` and ``cordapp`` Gradle plugins can be used together. + Other dependencies ^^^^^^^^^^^^^^^^^^ If your CorDapps have any additional external dependencies, they can be specified like normal Kotlin/Java dependencies @@ -145,4 +158,4 @@ Installing the CorDapp JAR At runtime, nodes will load any CorDapps present in their ``cordapps`` folder. Therefore in order to install a CorDapp on a node, the CorDapp JAR must be added to the ``/cordapps/`` folder, where ``node_dir`` is the folder in which -the node's JAR and configuration files are stored. \ No newline at end of file +the node's JAR and configuration files are stored. From 93054a95905c4bf7383245f16b587a75d88feef5 Mon Sep 17 00:00:00 2001 From: Patrick Kuo Date: Mon, 29 Jan 2018 13:43:16 +0000 Subject: [PATCH 07/12] Add cmdline option for network root truststore and password (#2407) * add cmdline option for network root truststore and password, instead of using node's truststore configuration to avoid confusion. * revert line auto format * fix failing integration test * address PR issue --- .../registration/NodeRegistrationTest.kt | 26 --------------- .../main/kotlin/net/corda/node/ArgsParser.kt | 26 +++++++++++++-- .../net/corda/node/internal/NodeStartup.kt | 12 ++++--- .../registration/NetworkRegistrationHelper.kt | 32 ++++++++++++------- .../kotlin/net/corda/node/ArgsParserTest.kt | 10 ++++-- .../NetworkRegistrationHelperTest.kt | 20 ++++++++---- .../testing/node/internal/DriverDSLImpl.kt | 30 ++++++++++------- 7 files changed, 90 insertions(+), 66 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt b/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt index ea71007414..560987cdf7 100644 --- a/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/utilities/registration/NodeRegistrationTest.kt @@ -1,6 +1,5 @@ package net.corda.node.utilities.registration -import net.corda.core.crypto.Crypto import net.corda.core.identity.CordaX500Name import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow @@ -26,7 +25,6 @@ import net.corda.testing.node.internal.CompatibilityZoneParams import net.corda.testing.node.internal.internalDriver import net.corda.testing.node.internal.network.NetworkMapServer import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy import org.bouncycastle.pkcs.PKCS10CertificationRequest import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest import org.junit.After @@ -38,12 +36,10 @@ import java.io.InputStream import java.net.URL import java.security.KeyPair import java.security.cert.CertPath -import java.security.cert.CertPathValidatorException import java.security.cert.Certificate import java.security.cert.X509Certificate import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream -import javax.security.auth.x500.X500Principal import javax.ws.rs.* import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response @@ -116,28 +112,6 @@ class NodeRegistrationTest { ).returnValue.getOrThrow() } } - - @Test - fun `node registration wrong root cert`() { - val someRootCert = X509Utilities.createSelfSignedCACertificate( - X500Principal("CN=Integration Test Corda Node Root CA,O=R3 Ltd,L=London,C=GB"), - Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME)) - val compatibilityZone = CompatibilityZoneParams( - URL("http://$serverHostAndPort"), - publishNotaries = { server.networkParameters = testNetworkParameters(it) }, - rootCert = someRootCert) - internalDriver( - portAllocation = portAllocation, - compatibilityZone = compatibilityZone, - initialiseSerialization = false, - notarySpecs = listOf(NotarySpec(notaryName)), - startNodesInProcess = true // We need to run the nodes in the same process so that we can capture the correct exception - ) { - assertThatThrownBy { - defaultNotaryNode.getOrThrow() - }.isInstanceOf(CertPathValidatorException::class.java) - } - } } @Path("certificate") diff --git a/node/src/main/kotlin/net/corda/node/ArgsParser.kt b/node/src/main/kotlin/net/corda/node/ArgsParser.kt index c6667d6a11..8353dd7f1a 100644 --- a/node/src/main/kotlin/net/corda/node/ArgsParser.kt +++ b/node/src/main/kotlin/net/corda/node/ArgsParser.kt @@ -1,6 +1,5 @@ package net.corda.node -import com.typesafe.config.ConfigException import joptsimple.OptionParser import joptsimple.util.EnumConverter import net.corda.core.internal.div @@ -34,6 +33,10 @@ class ArgsParser { private val sshdServerArg = optionParser.accepts("sshd", "Enables SSHD server for node administration.") private val noLocalShellArg = optionParser.accepts("no-local-shell", "Do not start the embedded shell locally.") private val isRegistrationArg = optionParser.accepts("initial-registration", "Start initial node registration with Corda network to obtain certificate from the permissioning server.") + private val networkRootTruststorePathArg = optionParser.accepts("network-root-truststore", "Network root trust store obtained from network operator.") + .withRequiredArg() + private val networkRootTruststorePasswordArg = optionParser.accepts("network-root-truststore-password", "Network root trust store password obtained from network operator.") + .withRequiredArg() private val isVersionArg = optionParser.accepts("version", "Print the version and exit") private val justGenerateNodeInfoArg = optionParser.accepts("just-generate-node-info", "Perform the node start-up task necessary to generate its nodeInfo, save it to disk, then quit") @@ -56,8 +59,21 @@ class ArgsParser { val sshdServer = optionSet.has(sshdServerArg) val justGenerateNodeInfo = optionSet.has(justGenerateNodeInfoArg) val bootstrapRaftCluster = optionSet.has(bootstrapRaftClusterArg) - return CmdLineOptions(baseDirectory, configFile, help, loggingLevel, logToConsole, isRegistration, isVersion, - noLocalShell, sshdServer, justGenerateNodeInfo, bootstrapRaftCluster) + val networkRootTruststorePath = optionSet.valueOf(networkRootTruststorePathArg)?.let { Paths.get(it).normalize().toAbsolutePath() } + val networkRootTruststorePassword = optionSet.valueOf(networkRootTruststorePasswordArg) + return CmdLineOptions(baseDirectory, + configFile, + help, + loggingLevel, + logToConsole, + isRegistration, + networkRootTruststorePath, + networkRootTruststorePassword, + isVersion, + noLocalShell, + sshdServer, + justGenerateNodeInfo, + bootstrapRaftCluster) } fun printHelp(sink: PrintStream) = optionParser.printHelpOn(sink) @@ -69,6 +85,8 @@ data class CmdLineOptions(val baseDirectory: Path, val loggingLevel: Level, val logToConsole: Boolean, val isRegistration: Boolean, + val networkRootTruststorePath: Path?, + val networkRootTruststorePassword: String?, val isVersion: Boolean, val noLocalShell: Boolean, val sshdServer: Boolean, @@ -78,6 +96,8 @@ data class CmdLineOptions(val baseDirectory: Path, val config = ConfigHelper.loadConfig(baseDirectory, configFile).parseAsNodeConfiguration() if (isRegistration) { requireNotNull(config.compatibilityZoneURL) { "Compatibility Zone Url must be provided in registration mode." } + requireNotNull(networkRootTruststorePath) { "Network root trust store path must be provided in registration mode." } + requireNotNull(networkRootTruststorePassword) { "Network root trust store password must be provided in registration mode." } } return config } diff --git a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt index 7f3d45ffea..fa3ea52d3a 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt @@ -2,8 +2,11 @@ package net.corda.node.internal import com.jcabi.manifests.Manifests import joptsimple.OptionException -import net.corda.core.internal.* +import net.corda.core.internal.Emoji import net.corda.core.internal.concurrent.thenMatch +import net.corda.core.internal.createDirectories +import net.corda.core.internal.div +import net.corda.core.internal.randomOrNull import net.corda.core.utilities.loggerFor import net.corda.node.* import net.corda.node.services.config.NodeConfiguration @@ -91,7 +94,8 @@ open class NodeStartup(val args: Array) { banJavaSerialisation(conf) preNetworkRegistration(conf) if (shouldRegisterWithNetwork(cmdlineOptions, conf)) { - registerWithNetwork(cmdlineOptions, conf) + // Null checks for [compatibilityZoneURL], [rootTruststorePath] and [rootTruststorePassword] has been done in [CmdLineOptions.loadConfig] + registerWithNetwork(conf, cmdlineOptions.networkRootTruststorePath!!, cmdlineOptions.networkRootTruststorePassword!!) return true } logStartupInfo(versionInfo, cmdlineOptions, conf) @@ -179,7 +183,7 @@ open class NodeStartup(val args: Array) { return !(!cmdlineOptions.isRegistration || compatibilityZoneURL == null) } - open protected fun registerWithNetwork(cmdlineOptions: CmdLineOptions, conf: NodeConfiguration) { + open protected fun registerWithNetwork(conf: NodeConfiguration, networkRootTruststorePath: Path, networkRootTruststorePassword: String) { val compatibilityZoneURL = conf.compatibilityZoneURL!! println() println("******************************************************************") @@ -187,7 +191,7 @@ open class NodeStartup(val args: Array) { println("* Registering as a new participant with Corda network *") println("* *") println("******************************************************************") - NetworkRegistrationHelper(conf, HTTPNetworkRegistrationService(compatibilityZoneURL)).buildKeystore() + NetworkRegistrationHelper(conf, HTTPNetworkRegistrationService(compatibilityZoneURL), networkRootTruststorePath, networkRootTruststorePassword).buildKeystore() } open protected fun loadConfigFile(cmdlineOptions: CmdLineOptions): NodeConfiguration = cmdlineOptions.loadConfig() diff --git a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt index 509ade8a32..a940a28a05 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt @@ -6,14 +6,15 @@ import net.corda.core.internal.* import net.corda.core.utilities.seconds import net.corda.node.services.config.NodeConfiguration import net.corda.nodeapi.internal.crypto.CertificateType +import net.corda.nodeapi.internal.crypto.X509KeyStore import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_CLIENT_CA import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_CLIENT_TLS import net.corda.nodeapi.internal.crypto.X509Utilities.CORDA_ROOT_CA -import net.corda.nodeapi.internal.crypto.x509 import org.bouncycastle.openssl.jcajce.JcaPEMWriter import org.bouncycastle.util.io.pem.PemObject import java.io.StringWriter +import java.nio.file.Path import java.security.KeyPair import java.security.KeyStore import java.security.cert.X509Certificate @@ -22,7 +23,10 @@ import java.security.cert.X509Certificate * Helper for managing the node registration process, which checks for any existing certificates and requests them if * needed. */ -class NetworkRegistrationHelper(private val config: NodeConfiguration, private val certService: NetworkRegistrationService) { +class NetworkRegistrationHelper(private val config: NodeConfiguration, + private val certService: NetworkRegistrationService, + networkRootTrustStorePath: Path, + networkRootTruststorePassword: String) { private companion object { val pollInterval = 10.seconds const val SELF_SIGNED_PRIVATE_KEY = "Self Signed Private Key" @@ -31,20 +35,16 @@ class NetworkRegistrationHelper(private val config: NodeConfiguration, private v private val requestIdStore = config.certificatesDirectory / "certificate-request-id.txt" // TODO: Use different password for private key. private val privateKeyPassword = config.keyStorePassword + private val rootTrustStore: X509KeyStore private val rootCert: X509Certificate init { - require(config.trustStoreFile.exists()) { - "${config.trustStoreFile} does not exist. This file must contain the root CA cert of your compatibility zone. " + + require(networkRootTrustStorePath.exists()) { + "$networkRootTrustStorePath does not exist. This file must contain the root CA cert of your compatibility zone. " + "Please contact your CZ operator." } - val rootCert = config.loadTrustStore().internal.getCertificate(CORDA_ROOT_CA) - require(rootCert != null) { - "${config.trustStoreFile} does not contain a certificate with the key $CORDA_ROOT_CA." + - "This file must contain the root CA cert of your compatibility zone. " + - "Please contact your CZ operator." - } - this.rootCert = rootCert.x509 + rootTrustStore = X509KeyStore.fromFile(networkRootTrustStorePath, networkRootTruststorePassword) + rootCert = rootTrustStore.getCertificate(CORDA_ROOT_CA) } /** @@ -109,7 +109,7 @@ class NetworkRegistrationHelper(private val config: NodeConfiguration, private v throw CertificateRequestException("Received node CA cert has invalid role: $nodeCaCertRole") } - println("Checking root of the certificate path is what we expect.") + // Validate certificate chain returned from the doorman with the root cert obtained via out-of-band process, to prevent MITM attack on doorman server. X509Utilities.validateCertificateChain(rootCert, certificates) println("Certificate signing request approved, storing private key with the certificate chain.") @@ -119,6 +119,14 @@ class NetworkRegistrationHelper(private val config: NodeConfiguration, private v nodeKeyStore.save() println("Node private key and certificate stored in ${config.nodeKeystore}.") + // Save root certificates to trust store. + config.loadTrustStore(createNew = true).update { + println("Generating trust store for corda node.") + // Assumes certificate chain always starts with client certificate and end with root certificate. + setCertificate(CORDA_ROOT_CA, certificates.last()) + } + println("Node trust store stored in ${config.trustStoreFile}.") + config.loadSslKeyStore(createNew = true).update { println("Generating SSL certificate for node messaging service.") val sslKeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) diff --git a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt index 8ba6f00b27..b5f56c4454 100644 --- a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt +++ b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt @@ -7,6 +7,7 @@ import org.assertj.core.api.Assertions.assertThatExceptionOfType import org.junit.Test import org.slf4j.event.Level import java.nio.file.Paths +import kotlin.test.assertEquals class ArgsParserTest { private val parser = ArgsParser() @@ -25,7 +26,9 @@ class ArgsParserTest { noLocalShell = false, sshdServer = false, justGenerateNodeInfo = false, - bootstrapRaftCluster = false)) + bootstrapRaftCluster = false, + networkRootTruststorePassword = null, + networkRootTruststorePath = null)) } @Test @@ -110,8 +113,11 @@ class ArgsParserTest { @Test fun `initial-registration`() { - val cmdLineOptions = parser.parse("--initial-registration") + val cmdLineOptions = parser.parse("--initial-registration", "--network-root-truststore", "/truststore/file.jks", "--network-root-truststore-password", "password-test") assertThat(cmdLineOptions.isRegistration).isTrue() + assertEquals(Paths.get("/truststore/file.jks"), cmdLineOptions.networkRootTruststorePath) + assertEquals("password-test", cmdLineOptions.networkRootTruststorePassword) + } @Test diff --git a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt index b2788fcd25..e7e6775a5e 100644 --- a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt +++ b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt @@ -10,9 +10,11 @@ import net.corda.core.crypto.Crypto import net.corda.core.crypto.SecureHash import net.corda.core.identity.CordaX500Name import net.corda.core.internal.createDirectories +import net.corda.core.internal.div import net.corda.core.internal.x500Name import net.corda.node.services.config.NodeConfiguration import net.corda.nodeapi.internal.crypto.CertificateType +import net.corda.nodeapi.internal.crypto.X509KeyStore import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.testing.core.ALICE_NAME import net.corda.testing.internal.createDevIntermediateCaCertPath @@ -35,10 +37,13 @@ class NetworkRegistrationHelperTest { private val nodeLegalName = ALICE_NAME private lateinit var config: NodeConfiguration + private val networkRootTrustStoreFileName = "network-root-truststore.jks" + private val networkRootTrustStorePassword = "network-root-truststore-password" @Before fun init() { val baseDirectory = fs.getPath("/baseDir").createDirectories() + abstract class AbstractNodeConfiguration : NodeConfiguration config = rigorousMock().also { doReturn(baseDirectory).whenever(it).baseDirectory @@ -62,7 +67,7 @@ class NetworkRegistrationHelperTest { val nodeCaCertPath = createNodeCaCertPath() - saveTrustStoreWithRootCa(nodeCaCertPath.last()) + saveNetworkTrustStore(nodeCaCertPath.last()) createRegistrationHelper(nodeCaCertPath).buildKeystore() val nodeKeystore = config.loadNodeKeyStore() @@ -105,7 +110,7 @@ class NetworkRegistrationHelperTest { @Test fun `node CA with incorrect cert role`() { val nodeCaCertPath = createNodeCaCertPath(type = CertificateType.TLS) - saveTrustStoreWithRootCa(nodeCaCertPath.last()) + saveNetworkTrustStore(nodeCaCertPath.last()) val registrationHelper = createRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) .isThrownBy { registrationHelper.buildKeystore() } @@ -116,7 +121,7 @@ class NetworkRegistrationHelperTest { fun `node CA with incorrect subject`() { val invalidName = CordaX500Name("Foo", "MU", "GB") val nodeCaCertPath = createNodeCaCertPath(legalName = invalidName) - saveTrustStoreWithRootCa(nodeCaCertPath.last()) + saveNetworkTrustStore(nodeCaCertPath.last()) val registrationHelper = createRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) .isThrownBy { registrationHelper.buildKeystore() } @@ -128,7 +133,7 @@ class NetworkRegistrationHelperTest { val wrongRootCert = X509Utilities.createSelfSignedCACertificate( X500Principal("O=Foo,L=MU,C=GB"), Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME)) - saveTrustStoreWithRootCa(wrongRootCert) + saveNetworkTrustStore(wrongRootCert) val registrationHelper = createRegistrationHelper(createNodeCaCertPath()) assertThatThrownBy { registrationHelper.buildKeystore() @@ -155,12 +160,13 @@ class NetworkRegistrationHelperTest { doReturn(requestId).whenever(it).submitRequest(any()) doReturn(response).whenever(it).retrieveCertificates(eq(requestId)) } - return NetworkRegistrationHelper(config, certService) + return NetworkRegistrationHelper(config, certService, config.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword) } - private fun saveTrustStoreWithRootCa(rootCert: X509Certificate) { + private fun saveNetworkTrustStore(rootCert: X509Certificate) { config.certificatesDirectory.createDirectories() - config.loadTrustStore(createNew = true).update { + val rootTruststorePath = config.certificatesDirectory / networkRootTrustStoreFileName + X509KeyStore.fromFile(rootTruststorePath, networkRootTrustStorePassword, createNew = true).update { setCertificate(X509Utilities.CORDA_ROOT_CA, rootCert) } } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index dab5cd7a68..3a41a4627a 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -33,6 +33,7 @@ import net.corda.nodeapi.internal.SignedNodeInfo import net.corda.nodeapi.internal.addShutdownHook import net.corda.nodeapi.internal.config.parseAs import net.corda.nodeapi.internal.config.toConfig +import net.corda.nodeapi.internal.crypto.X509KeyStore import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.network.NetworkParametersCopier import net.corda.nodeapi.internal.network.NodeInfoFilesCopier @@ -237,17 +238,24 @@ class DriverDSLImpl( )) config.corda.certificatesDirectory.createDirectories() - config.corda.loadTrustStore(createNew = true).update { + // Create network root truststore. + val rootTruststorePath = config.corda.certificatesDirectory / "network-root-truststore.jks" + // The network truststore will be provided by the network operator via out-of-band communication. + val rootTruststorePassword = "corda-root-password" + X509KeyStore.fromFile(rootTruststorePath, rootTruststorePassword, createNew = true).update { setCertificate(X509Utilities.CORDA_ROOT_CA, rootCert) } return if (startNodesInProcess) { executorService.fork { - NetworkRegistrationHelper(config.corda, HTTPNetworkRegistrationService(compatibilityZoneURL)).buildKeystore() + NetworkRegistrationHelper(config.corda, HTTPNetworkRegistrationService(compatibilityZoneURL), rootTruststorePath, rootTruststorePassword).buildKeystore() config } } else { - startOutOfProcessMiniNode(config, "--initial-registration").map { config } + startOutOfProcessMiniNode(config, + "--initial-registration", + "--network-root-truststore=${rootTruststorePath.toAbsolutePath()}", + "--network-root-truststore-password=$rootTruststorePassword").map { config } } } @@ -479,8 +487,8 @@ class DriverDSLImpl( when (it.cluster) { null -> startSingleNotary(it, localNetworkMap) is ClusterSpec.Raft, - // DummyCluster is used for testing the notary communication path, and it does not matter - // which underlying consensus algorithm is used, so we just stick to Raft + // DummyCluster is used for testing the notary communication path, and it does not matter + // which underlying consensus algorithm is used, so we just stick to Raft is DummyClusterSpec -> startRaftNotaryCluster(it, localNetworkMap) else -> throw IllegalArgumentException("BFT-SMaRt not supported") } @@ -596,7 +604,7 @@ class DriverDSLImpl( * Start the node with the given flag which is expected to start the node for some function, which once complete will * terminate the node. */ - private fun startOutOfProcessMiniNode(config: NodeConfig, extraCmdLineFlag: String): CordaFuture { + private fun startOutOfProcessMiniNode(config: NodeConfig, vararg extraCmdLineFlag: String): CordaFuture { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null val monitorPort = if (jmxPolicy.startJmxHttpServer) jmxPolicy.jmxHttpServerPortAllocation?.nextPort() else null val process = startOutOfProcessNode( @@ -608,7 +616,7 @@ class DriverDSLImpl( systemProperties, cordappPackages, "200m", - extraCmdLineFlag + *extraCmdLineFlag ) return poll(executorService, "$extraCmdLineFlag (${config.corda.myLegalName})") { @@ -652,7 +660,7 @@ class DriverDSLImpl( } else { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null val monitorPort = if (jmxPolicy.startJmxHttpServer) jmxPolicy.jmxHttpServerPortAllocation?.nextPort() else null - val process = startOutOfProcessNode(config, quasarJarPath, debugPort, jolokiaJarPath, monitorPort, systemProperties, cordappPackages, maximumHeapSize, null) + val process = startOutOfProcessNode(config, quasarJarPath, debugPort, jolokiaJarPath, monitorPort, systemProperties, cordappPackages, maximumHeapSize) if (waitForNodesToFinish) { state.locked { processes += process @@ -763,7 +771,7 @@ class DriverDSLImpl( overriddenSystemProperties: Map, cordappPackages: List, maximumHeapSize: String, - extraCmdLineFlag: String? + vararg extraCmdLineFlag: String ): Process { log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, " + "debug port is " + (debugPort ?: "not enabled") + ", " + @@ -801,9 +809,7 @@ class DriverDSLImpl( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { - if (extraCmdLineFlag != null) { - it += extraCmdLineFlag - } + it += extraCmdLineFlag }.toList() return ProcessUtilities.startCordaProcess( From 9ca63d173d27dc1b910aa3150e0d7bd49ea017a4 Mon Sep 17 00:00:00 2001 From: James Brown Date: Fri, 26 Jan 2018 16:23:55 +0000 Subject: [PATCH 08/12] Update OWASP dependency checker to latest working version --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 66e94fd571..74f4b681de 100644 --- a/build.gradle +++ b/build.gradle @@ -58,7 +58,7 @@ buildscript { ext.rxjava_version = '1.2.4' ext.dokka_version = '0.9.16-eap-2' ext.eddsa_version = '0.2.0' - ext.dependency_checker_version = '3.0.1' + ext.dependency_checker_version = '3.1.0' ext.commons_collections_version = '4.1' ext.beanutils_version = '1.9.3' ext.crash_version = 'cce5a00f114343c1145c1d7756e1dd6df3ea984e' From 28e29c0873c19308e685c5b112f7f652bdbaef63 Mon Sep 17 00:00:00 2001 From: Patrick Kuo Date: Mon, 29 Jan 2018 17:43:48 +0000 Subject: [PATCH 09/12] Fix path issue which causes windows build failure. (#2430) --- node/src/test/kotlin/net/corda/node/ArgsParserTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt index b5f56c4454..96f98d8c33 100644 --- a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt +++ b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt @@ -113,11 +113,11 @@ class ArgsParserTest { @Test fun `initial-registration`() { - val cmdLineOptions = parser.parse("--initial-registration", "--network-root-truststore", "/truststore/file.jks", "--network-root-truststore-password", "password-test") + val truststorePath = Paths.get("truststore") / "file.jks" + val cmdLineOptions = parser.parse("--initial-registration", "--network-root-truststore", "$truststorePath", "--network-root-truststore-password", "password-test") assertThat(cmdLineOptions.isRegistration).isTrue() - assertEquals(Paths.get("/truststore/file.jks"), cmdLineOptions.networkRootTruststorePath) + assertEquals(truststorePath.toAbsolutePath(), cmdLineOptions.networkRootTruststorePath) assertEquals("password-test", cmdLineOptions.networkRootTruststorePassword) - } @Test From 685ab4c9b0e8437355b28466e41f4caf7d326de1 Mon Sep 17 00:00:00 2001 From: Patrick Kuo Date: Mon, 29 Jan 2018 17:44:28 +0000 Subject: [PATCH 10/12] Add trace logging to network map client (#2424) * Add trace logging to network map client * Add trace logging to network map client * address PR issue * address PR issue --- .../node/services/network/NetworkMapClient.kt | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt index 4e1e2c821c..9c12056315 100644 --- a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt @@ -11,6 +11,7 @@ import net.corda.core.serialization.serialize import net.corda.core.utilities.contextLogger import net.corda.core.utilities.minutes import net.corda.core.utilities.seconds +import net.corda.core.utilities.trace import net.corda.node.services.api.NetworkMapCacheInternal import net.corda.node.utilities.NamedThreadFactory import net.corda.nodeapi.internal.SignedNodeInfo @@ -29,10 +30,15 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit class NetworkMapClient(compatibilityZoneURL: URL, private val trustedRoot: X509Certificate) { + companion object { + private val logger = contextLogger() + } + private val networkMapUrl = URL("$compatibilityZoneURL/network-map") fun publish(signedNodeInfo: SignedNodeInfo) { val publishURL = URL("$networkMapUrl/publish") + logger.trace { "Publishing NodeInfo to $publishURL." } publishURL.openHttpConnection().apply { doOutput = true requestMethod = "POST" @@ -40,27 +46,41 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val trustedRoot: X509C outputStream.use { signedNodeInfo.serialize().open().copyTo(it) } checkOkResponse() } + logger.trace { "Published NodeInfo to $publishURL successfully." } } fun getNetworkMap(): NetworkMapResponse { + logger.trace { "Fetching network map update from $networkMapUrl." } val connection = networkMapUrl.openHttpConnection() val signedNetworkMap = connection.responseAs>() val networkMap = signedNetworkMap.verifiedNetworkMapCert(trustedRoot) val timeout = CacheControl.parse(Headers.of(connection.headerFields.filterKeys { it != null }.mapValues { it.value[0] })).maxAgeSeconds().seconds + logger.trace { "Fetched network map update from $networkMapUrl successfully, retrieved ${networkMap.nodeInfoHashes.size} node info hashes. Node Info hashes: ${networkMap.nodeInfoHashes.joinToString("\n")}" } return NetworkMapResponse(networkMap, timeout) } fun getNodeInfo(nodeInfoHash: SecureHash): NodeInfo { - return URL("$networkMapUrl/node-info/$nodeInfoHash").openHttpConnection().responseAs().verified() + val url = URL("$networkMapUrl/node-info/$nodeInfoHash") + logger.trace { "Fetching node info: '$nodeInfoHash' from $url." } + val verifiedNodeInfo = url.openHttpConnection().responseAs().verified() + logger.trace { "Fetched node info: '$nodeInfoHash' successfully. Node Info: $verifiedNodeInfo" } + return verifiedNodeInfo } fun getNetworkParameters(networkParameterHash: SecureHash): SignedDataWithCert { - return URL("$networkMapUrl/network-parameters/$networkParameterHash").openHttpConnection().responseAs() + val url = URL("$networkMapUrl/network-parameters/$networkParameterHash") + logger.trace { "Fetching network parameters: '$networkParameterHash' from $url." } + val networkParameter = url.openHttpConnection().responseAs>() + logger.trace { "Fetched network parameters: '$networkParameterHash' successfully. Network Parameters: $networkParameter" } + return networkParameter } fun myPublicHostname(): String { - val connection = URL("$networkMapUrl/my-hostname").openHttpConnection() - return connection.inputStream.bufferedReader().use(BufferedReader::readLine) + val url = URL("$networkMapUrl/my-hostname") + logger.trace { "Resolving public hostname from '$url'." } + val hostName = url.openHttpConnection().inputStream.bufferedReader().use(BufferedReader::readLine) + logger.trace { "My public hostname is $hostName." } + return hostName } } From d6f9721cb8b117cc96dc592b85db927b34bbda09 Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Tue, 30 Jan 2018 10:59:03 +0100 Subject: [PATCH 11/12] Docs: move serialisation into the CorDapp section. Fix some markup issues (#2429) * Docs: move serialisation into the CorDapp section. Fix some markup issues. * Address review comments --- docs/source/building-a-cordapp-index.rst | 1 + docs/source/node-internals-index.rst | 1 - docs/source/serialization.rst | 218 ++++++++++++----------- 3 files changed, 112 insertions(+), 108 deletions(-) diff --git a/docs/source/building-a-cordapp-index.rst b/docs/source/building-a-cordapp-index.rst index f061c6d432..fa6976870f 100644 --- a/docs/source/building-a-cordapp-index.rst +++ b/docs/source/building-a-cordapp-index.rst @@ -11,6 +11,7 @@ CorDapps cordapp-build-systems building-against-master corda-api + serialization secure-coding-guidelines flow-cookbook cheat-sheet diff --git a/docs/source/node-internals-index.rst b/docs/source/node-internals-index.rst index e5a1c9157a..8e4241a805 100644 --- a/docs/source/node-internals-index.rst +++ b/docs/source/node-internals-index.rst @@ -6,5 +6,4 @@ Node internals node-services vault - serialization messaging diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst index 96fab2adf0..eed77ecdd6 100644 --- a/docs/source/serialization.rst +++ b/docs/source/serialization.rst @@ -1,15 +1,41 @@ +.. highlight:: kotlin +.. raw:: html + + + + Object serialization ==================== .. contents:: -What is serialization (and deserialization)? --------------------------------------------- +Introduction +------------ Object serialization is the process of converting objects into a stream of bytes and, deserialization, the reverse process of creating objects from a stream of bytes. It takes place every time nodes pass objects to each other as messages, when objects are sent to or from RPC clients from the node, and when we store transactions in the database. +Corda pervasively uses a custom form of type safe binary serialisation. This stands in contrast to some other systems that use +weakly or untyped string-based serialisation schemes like JSON or XML. The primary drivers for this were: + +* A desire to have a schema describing what has been serialized alongside the actual data: + + #. To assist with versioning, both in terms of being able to interpret data archived long ago (e.g. trades from + a decade ago, long after the code has changed) and between differing code versions. + #. To make it easier to write generic code e.g. user interfaces that can navigate the serialized form of data. + #. To support cross platform (non-JVM) interaction, where the format of a class file is not so easily interpreted. + +* A desire to use a documented and static wire format that is platform independent, and is not subject to change with + 3rd party library upgrades, etc. +* A desire to support open-ended polymorphism, where the number of subclasses of a superclass can expand over time + and the subclasses do not need to be defined in the schema *upfront*. This is key to many Corda concepts, such as states. +* Increased security by constructing deserialized objects through supported constructors, rather than having + data inserted directly into their fields without an opportunity to validate consistency or intercept attempts to manipulate + supposed invariants. +* Binary formats work better with digital signatures than text based formats, as there's much less scope for + changes that modify syntax but not semantics. + Whitelisting ------------ @@ -44,46 +70,28 @@ It's reproduced here as an example of both ways you can do this for a couple of ``Callable c = (Callable & Serializable) () -> "Hello World";``. AMQP -==== +---- -Originally Corda used a ``Kryo``-based serialization scheme throughout for all serialization contexts. However, it was realised there -was a compelling use case for the definition and development of a custom format based upon AMQP 1.0. The primary drivers for this were: +Corda uses an extended form of AMQP 1.0 as its binary wire protocol. - #. A desire to have a schema describing what has been serialized alongside the actual data: +Corda serialisation is currently used for: - #. To assist with versioning, both in terms of being able to interpret data archived long ago (e.g. trades from - a decade ago, long after the code has changed) and between differing code versions - #. To make it easier to write user interfaces that can navigate the serialized form of data - #. To support cross platform (non-JVM) interaction, where the format of a class file is not so easily interpreted - #. A desire to use a documented and static wire format that is platform independent, and is not subject to change with - 3rd party library upgrades, etc. - #. A desire to support open-ended polymorphism, where the number of subclasses of a superclass can expand over time - and the subclasses do not need to be defined in the schema *upfront*. This is key to many Corda concepts, such as states. - #. Increased security by constructing deserialized objects through supported constructors, rather than having - data inserted directly into their fields without an opportunity to validate consistency or intercept attempts to manipulate - supposed invariants + #. Peer-to-peer networking. + #. Persisted messages, like signed transactions and states. -Delivering this is an ongoing effort by the Corda development team. At present, the ``Kryo``-based format is still used by the RPC framework on -both the client and server side. However, it is planned that the RPC framework will move to the AMQP framework when ready. +.. note:: At present, the Kryo-based format is still used by the RPC framework on both the client and server side. However, it is planned that the RPC framework will move to the AMQP framework soon. -The AMQP framework is currently used for: - - #. The peer-to-peer context, representing inter-node communication - #. The persistence layer, representing contract states persisted into the vault - -Finally, for the checkpointing of flows, Corda will continue to use the existing ``Kryo`` scheme. +For the checkpointing of flows Corda uses a private scheme that is subject to change. It is currently based on the Kryo +framework, but this may not be true in future. This separation of serialization schemes into different contexts allows us to use the most suitable framework for that context rather than -attempting to force a one-size-fits-all approach. ``Kryo`` is more suited to the serialization of a program's stack frames, as it is more flexible +attempting to force a one-size-fits-all approach. Kryo is more suited to the serialization of a program's stack frames, as it is more flexible than our AMQP framework in what it can construct and serialize. However, that flexibility makes it exceptionally difficult to make secure. Conversely, our AMQP framework allows us to concentrate on a secure framework that can be reasoned about and thus made safer, with far fewer security holes. -.. note:: Selection of serialization context should, for the most part, be opaque to CorDapp developers, the Corda framework selecting - the correct context as configured. - -.. note:: For information on our choice of AMQP 1.0, see :doc:`amqp-choice`. For detail on how we utilise AMQP 1.0 and represent - objects in AMQP types, see :doc:`amqp-format`. +Selection of serialization context should, for the most part, be opaque to CorDapp developers, the Corda framework selecting +the correct context as configured. This document describes what is currently and what will be supported in the Corda AMQP format from the perspective of CorDapp developers, to allow CorDapps to take into consideration the future state. The AMQP serialization format will @@ -97,8 +105,9 @@ This section describes the classes and interfaces that the AMQP serialization fo Collection Types ```````````````` -The following collection types are supported. Any implementation of the following will be mapped to *an* implementation of the interface or class on the other end. -For example, if you use a Guava implementation of a collection, it will deserialize as the primitive collection type. +The following collection types are supported. Any implementation of the following will be mapped to *an* implementation +of the interface or class on the other end. For example, if you use a Guava implementation of a collection, it will +deserialize as the primitive collection type. The declared types of properties should only use these types, and not any concrete implementation types (e.g. Guava implementations). Collections must specify their generic type, the generic type parameters will be included in @@ -233,7 +242,7 @@ General Rules .. note:: In circumstances where classes cannot be recompiled, such as when using a third-party library, a proxy serializer can be used to avoid this problem. Details on creating such an object can be found on the - :doc:`cordapp-custom-serializers` page. + :doc:`cordapp-custom-serializers` page. #. The class must be annotated with ``@CordaSerializable`` #. The declared types of constructor arguments, getters, and setters must be supported, and where generics are used, the @@ -304,21 +313,28 @@ For example: .. container:: codeset - .. sourcecode:: Java + .. sourcecode:: kotlin - class Example { - private int a; - private int b; - private int c; + class Example(var a: Int, var b: Int, var c: Int) - public int getA() { return a; } - public int getB() { return b; } - public int getC() { return c; } + .. sourcecode:: java - public void setA(int a) { this.a = a; } - public void setB(int b) { this.b = b; } - public void setC(int c) { this.c = c; } - } + class Example { + private int a; + private int b; + private int c; + + public int getA() { return a; } + public int getB() { return b; } + public int getC() { return c; } + + public void setA(int a) { this.a = a; } + public void setB(int b) { this.b = b; } + public void setC(int c) { this.c = c; } + } + +.. warning:: We do not recommend this pattern! Corda tries to use immutable data structures throughout, and if you + rely heavily on mutable JavaBean style objects then you may sometimes find the API behaves in unintuitive ways. Inaccessible Private Properties ``````````````````````````````` @@ -328,30 +344,26 @@ accessible getter methods, this development idiom is strongly discouraged. For example. - .. container:: codeset +.. container:: codeset - Kotlin: + .. sourcecode:: kotlin - .. sourcecode:: kotlin + class C(val a: Int, private val b: Int) - data class C(val a: Int, private val b: Int) + .. sourcecode:: java - Java: + class C { + public Integer a; + private Integer b; - .. sourcecode:: Java + public C(Integer a, Integer b) { + this.a = a; + this.b = b; + } + } - class C { - public Integer a; - private Integer b; - - C(Integer a, Integer b) { - this.a = a; - this.b = b; - } - } - -When designing stateful objects, is should be remembered that they are not, despite appearances, traditional -programmatic constructs. They are signed over, transformed, serialised, and relationally mapped. As such, +When designing Corda states, it should be remembered that they are not, despite appearances, traditional +OOP style objects. They are signed over, transformed, serialised, and relationally mapped. As such, all elements should be publicly accessible by design. .. warning:: IDEs will indicate erroneously that properties can be given something other than public visibility. Ignore @@ -359,40 +371,38 @@ all elements should be publicly accessible by design. Providing a public getter, as per the following example, is acceptable: - .. container:: codeset +.. container:: codeset - Kotlin: + .. sourcecode:: kotlin - .. sourcecode:: kotlin + class C(val a: Int, b: Int) { + var b: Int = b + private set + } - data class C(val a: Int, private val b: Int) { - public fun getB() = b - } + .. sourcecode:: java - Java: + class C { + public Integer a; + private Integer b; - .. sourcecode:: Java + C(Integer a, Integer b) { + this.a = a; + this.b = b; + } - class C { - public Integer a; - private Integer b; - - C(Integer a, Integer b) { - this.a = a; - this.b = b; - } - - public Integer getB() { - return b; - } - } + public Integer getB() { + return b; + } + } Enums ````` - #. All enums are supported, provided they are annotated with ``@CordaSerializable`` - +All enums are supported, provided they are annotated with ``@CordaSerializable``. Corda supports interoperability of +enumerated type versions. This allows such types to be changed over time without breaking backward (or forward) +compatibility. The rules and mechanisms for doing this are discussed in :doc:`serialization-enum-evolution`. Exceptions `````````` @@ -407,24 +417,23 @@ The following rules apply to supported ``Throwable`` implementations. Kotlin Objects `````````````` - #. Kotlin's non-anonymous ``object`` s (i.e. constructs like ``object foo : Contract {...}``) are singletons and - treated differently. They are recorded into the stream with no properties, and deserialize back to the - singleton instance. Currently, the same is not true of Java singletons, which will deserialize to new instances - of the class - #. Kotlin's anonymous ``object`` s (i.e. constructs like ``object : Contract {...}``) are not currently supported - and will not serialize correctly. They need to be re-written as an explicit class declaration +Kotlin's non-anonymous ``object`` s (i.e. constructs like ``object foo : Contract {...}``) are singletons and +treated differently. They are recorded into the stream with no properties, and deserialize back to the +singleton instance. Currently, the same is not true of Java singletons, which will deserialize to new instances +of the class. This is hard to fix because there's no perfectly standard idiom for Java singletons. -The Carpenter -````````````` +Kotlin's anonymous ``object`` s (i.e. constructs like ``object : Contract {...}``) are not currently supported +and will not serialize correctly. They need to be re-written as an explicit class declaration. -We support a class carpenter that can dynamically manufacture classes from the supplied schema when deserializing, -without the supporting classes being present on the classpath. This can be useful where other components might expect to -be able to use reflection over the deserialized data, and also for ensuring classes not on the classpath can be -deserialized without loading potentially malicious code dynamically without security review outside of a fully sandboxed -environment. A more detailed discussion of the carpenter will be provided in a future update to the documentation. +Class synthesis +--------------- -Future Enhancements -``````````````````` +Corda serialization supports dynamically synthesising classes from the supplied schema when deserializing, +without the supporting classes being present on the classpath. This can be useful where generic code might expect to +be able to use reflection over the deserialized data, for scripting languages that run on the JVM, and also for +ensuring classes not on the classpath can be deserialized without loading potentially malicious code. + +Possible future enhancements include: #. Java singleton support. We will add support for identifying classes which are singletons and identifying the static method responsible for returning the singleton instance @@ -442,10 +451,5 @@ and a version of the current state of the class instantiated. More detail can be found in :doc:`serialization-default-evolution`. -Enum Evolution -`````````````` -Corda supports interoperability of enumerated type versions. This allows such types to be changed over time without breaking -backward (or forward) compatibility. The rules and mechanisms for doing this are discussed in :doc:`serialization-enum-evolution``. - From 970303dc2d277a6b86e7fcd9a0655a2417cfdd07 Mon Sep 17 00:00:00 2001 From: Patrick Kuo Date: Tue, 30 Jan 2018 14:23:24 +0000 Subject: [PATCH 12/12] fix broken test after merge --- .../networkmanage/hsm/SigningServiceIntegrationTest.kt | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt index d7ff304933..563669ece1 100644 --- a/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt +++ b/network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt @@ -126,10 +126,13 @@ class SigningServiceIntegrationTest { } } config.certificatesDirectory.createDirectories() - val trustStore = X509KeyStore.fromFile(config.trustStoreFile, config.trustStorePassword, createNew = true) - trustStore.update { + val networkTrustStorePath = config.certificatesDirectory / "network-root-truststore.jks" + val networkTrustStorePassword = "network-trust-password" + val networkTrustStore = X509KeyStore.fromFile(networkTrustStorePath, networkTrustStorePassword, createNew = true) + networkTrustStore.update { setCertificate(X509Utilities.CORDA_ROOT_CA, rootCaCert) } + val trustStore = X509KeyStore.fromFile(config.trustStoreFile, config.trustStorePassword, createNew = true) val nodeKeyStore = X509KeyStore.fromFile(config.nodeKeystore, config.keyStorePassword, createNew = true) val sslKeyStore = X509KeyStore.fromFile(config.sslKeystore, config.keyStorePassword, createNew = true) config.also { @@ -137,7 +140,7 @@ class SigningServiceIntegrationTest { doReturn(nodeKeyStore).whenever(it).loadNodeKeyStore(any()) doReturn(sslKeyStore).whenever(it).loadSslKeyStore(any()) } - NetworkRegistrationHelper(config, HTTPNetworkRegistrationService(config.compatibilityZoneURL!!)).buildKeystore() + NetworkRegistrationHelper(config, HTTPNetworkRegistrationService(config.compatibilityZoneURL!!), networkTrustStorePath, networkTrustStorePassword).buildKeystore() verify(hsmSigner).sign(any()) } }