From e684cfd1e60f52817e083254612b92be2bbed680 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 13 Jul 2020 10:33:23 +0100 Subject: [PATCH 01/48] NOTICK: Avoid configuring more tasks in the deterministic modules. (#6452) --- core-deterministic/build.gradle | 28 +++++++++++++--------- deterministic.gradle | 2 +- jdk8u-deterministic/build.gradle | 4 +++- node/capsule/build.gradle | 6 ++--- serialization-deterministic/build.gradle | 30 ++++++++++++++---------- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/core-deterministic/build.gradle b/core-deterministic/build.gradle index 7b90dfdd82..5be42d8084 100644 --- a/core-deterministic/build.gradle +++ b/core-deterministic/build.gradle @@ -54,8 +54,8 @@ tasks.named('jar', Jar) { enabled = false } -def coreJarTask = tasks.getByPath(':core:jar') -def originalJar = coreJarTask.outputs.files.singleFile +def coreJarTask = project(':core').tasks.named('jar', Jar) +def originalJar = coreJarTask.map { it.outputs.files.singleFile } def patchCore = tasks.register('patchCore', Zip) { dependsOn coreJarTask @@ -132,7 +132,7 @@ def jarFilter = tasks.register('jarFilter', JarFilterTask) { } } -task determinise(type: ProGuardTask) { +def determinise = tasks.register('determinise', ProGuardTask) { injars jarFilter outjars file("$buildDir/proguard/$jarBaseName-${project.version}.jar") @@ -166,17 +166,20 @@ task determinise(type: ProGuardTask) { keepclassmembers 'class net.corda.core.** { public synthetic ; }' } -task metafix(type: MetaFixerTask) { +def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) + +def metafix = tasks.register('metafix', MetaFixerTask) { outputDir file("$buildDir/libs") jars determinise suffix "" // Strip timestamps from the JAR to make it reproducible. preserveTimestamps = false + finalizedBy checkDeterminism } // DOCSTART 01 -def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { +checkDeterminism.configure { dependsOn jdkTask injars metafix @@ -197,14 +200,17 @@ def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { // DOCEND 01 defaultTasks "determinise" -determinise.finalizedBy metafix -metafix.finalizedBy checkDeterminism -assemble.dependsOn checkDeterminism +determinise.configure { + finalizedBy metafix +} +tasks.named('assemble') { + dependsOn checkDeterminism +} -def deterministicJar = metafix.outputs.files.singleFile +def deterministicJar = metafix.map { it.outputs.files.singleFile } artifacts { - deterministicArtifacts file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix - publish file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix + deterministicArtifacts deterministicJar + publish deterministicJar } tasks.named('sourceJar', Jar) { diff --git a/deterministic.gradle b/deterministic.gradle index 11bd05770f..751af8bfb2 100644 --- a/deterministic.gradle +++ b/deterministic.gradle @@ -11,7 +11,7 @@ evaluationDependsOn(':jdk8u-deterministic') def jdk8uDeterministic = project(':jdk8u-deterministic') ext { - jdkTask = jdk8uDeterministic.assemble + jdkTask = jdk8uDeterministic.tasks.named('assemble') deterministic_jdk_home = jdk8uDeterministic.jdk_home deterministic_rt_jar = jdk8uDeterministic.rt_jar } diff --git a/jdk8u-deterministic/build.gradle b/jdk8u-deterministic/build.gradle index 92338df169..f9a91c9cc8 100644 --- a/jdk8u-deterministic/build.gradle +++ b/jdk8u-deterministic/build.gradle @@ -37,7 +37,9 @@ def copyJdk = tasks.register('copyJdk', Copy) { } } -assemble.dependsOn copyJdk +tasks.named('assemble') { + dependsOn copyJdk +} tasks.named('jar', Jar) { enabled = false } diff --git a/node/capsule/build.gradle b/node/capsule/build.gradle index 14cddd0332..c1c4e2f4c9 100644 --- a/node/capsule/build.gradle +++ b/node/capsule/build.gradle @@ -39,9 +39,9 @@ capsule { def nodeProject = project(':node') task buildCordaJAR(type: FatCapsule, dependsOn: [ - nodeProject.tasks.jar, - project(':core-deterministic').tasks.assemble, - project(':serialization-deterministic').tasks.assemble + nodeProject.tasks.named('jar'), + project(':core-deterministic').tasks.named('assemble'), + project(':serialization-deterministic').tasks.named('assemble') ]) { applicationClass 'net.corda.node.Corda' archiveBaseName = 'corda' diff --git a/serialization-deterministic/build.gradle b/serialization-deterministic/build.gradle index 7ea284daeb..6ad42b0208 100644 --- a/serialization-deterministic/build.gradle +++ b/serialization-deterministic/build.gradle @@ -50,8 +50,8 @@ tasks.named('jar', Jar) { enabled = false } -def serializationJarTask = tasks.getByPath(':serialization:jar') -def originalJar = serializationJarTask.outputs.files.singleFile +def serializationJarTask = project(':serialization').tasks.named('jar', Jar) +def originalJar = serializationJarTask.map { it.outputs.files.singleFile } def patchSerialization = tasks.register('patchSerialization', Zip) { dependsOn serializationJarTask @@ -77,7 +77,7 @@ def patchSerialization = tasks.register('patchSerialization', Zip) { } def predeterminise = tasks.register('predeterminise', ProGuardTask) { - dependsOn project(':core-deterministic').assemble + dependsOn project(':core-deterministic').tasks.named('assemble') injars patchSerialization outjars file("$buildDir/proguard/pre-deterministic-${project.version}.jar") @@ -125,7 +125,7 @@ def jarFilter = tasks.register('jarFilter', JarFilterTask) { } } -task determinise(type: ProGuardTask) { +def determinise = tasks.register('determinise', ProGuardTask) { injars jarFilter outjars file("$buildDir/proguard/$jarBaseName-${project.version}.jar") @@ -154,16 +154,19 @@ task determinise(type: ProGuardTask) { keepclassmembers 'class net.corda.serialization.** { public synthetic ; }' } -task metafix(type: MetaFixerTask) { +def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) + +def metafix = tasks.register('metafix', MetaFixerTask) { outputDir file("$buildDir/libs") jars determinise suffix "" // Strip timestamps from the JAR to make it reproducible. preserveTimestamps = false + finalizedBy checkDeterminism } -def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { +checkDeterminism.configure { dependsOn jdkTask injars metafix @@ -183,14 +186,17 @@ def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { } defaultTasks "determinise" -determinise.finalizedBy metafix -metafix.finalizedBy checkDeterminism -assemble.dependsOn checkDeterminism +determinise.configure { + finalizedBy metafix +} +tasks.named('assemble') { + dependsOn checkDeterminism +} -def deterministicJar = metafix.outputs.files.singleFile +def deterministicJar = metafix.map { it.outputs.files.singleFile } artifacts { - deterministicArtifacts file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix - publish file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix + deterministicArtifacts deterministicJar + publish deterministicJar } tasks.named('sourceJar', Jar) { From 79b75ff1ec4922a1bb0e531ec818d5dd7cd68fee Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Mon, 13 Jul 2020 11:42:30 +0100 Subject: [PATCH 02/48] INFRA-429: Wait for log file to indicate that the process is running (#6433) Wait for log file to indicate that a driver node process is running, instead of trying to open a socket on the port the node is attempting to binding on. This means the driver is more responsive to the node being ready, as well as removing a race condition where the binding test could block the node from starting. As sometimes nodes do not log this information, after 20 seconds we presume the node is up. --- .../testing/node/internal/DriverDSLImpl.kt | 2 +- .../node/internal/InternalTestUtils.kt | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index ee55c9999c..e3822522cd 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -729,7 +729,7 @@ class DriverDSLImpl( val effectiveP2PAddress = config.corda.messagingServerAddress ?: config.corda.p2pAddress val p2pReadyFuture = nodeMustBeStartedFuture( executorService, - effectiveP2PAddress, + config.corda.baseDirectory / "net.corda.node.Corda.${identifier}.stdout.log", process ) { NodeListenProcessDeathException( diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt index 67bfb8e1b8..3c6de690bf 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt @@ -11,6 +11,7 @@ import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.concurrent.openFuture import net.corda.core.internal.div +import net.corda.core.internal.readText import net.corda.core.internal.times import net.corda.core.messaging.CordaRPCOps import net.corda.core.node.services.AttachmentFixup @@ -41,8 +42,10 @@ import rx.subjects.AsyncSubject import java.io.InputStream import java.net.Socket import java.net.SocketException +import java.nio.file.Path import java.sql.DriverManager import java.time.Duration +import java.time.Instant import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit import java.util.jar.JarOutputStream @@ -79,6 +82,8 @@ val FINANCE_CORDAPPS: Set = setOf(FINANCE_CONTRACTS_CORDAPP, FI @JvmField val DUMMY_CONTRACTS_CORDAPP: CustomCordapp = cordappWithPackages("net.corda.testing.contracts") +private const val SECONDS_TO_WAIT_FOR_P2P: Long = 20 + fun cordappsForPackages(vararg packageNames: String): Set = cordappsForPackages(packageNames.asList()) fun cordappsForPackages(packageNames: Iterable): Set { @@ -172,20 +177,28 @@ fun addressMustBeBoundFuture(executorService: ScheduledExecutorService, hostAndP } fun nodeMustBeStartedFuture( - executorService: ScheduledExecutorService, - hostAndPort: NetworkHostAndPort, - listenProcess: Process? = null, - exception: () -> NodeListenProcessDeathException + executorService: ScheduledExecutorService, + logFile: Path, + listenProcess: Process, + exception: () -> NodeListenProcessDeathException ): CordaFuture { - return poll(executorService, "address $hostAndPort to bind") { - if (listenProcess != null && !listenProcess.isAlive) { + val stopPolling = Instant.now().plusSeconds(SECONDS_TO_WAIT_FOR_P2P) + return poll(executorService, "process $listenProcess is running") { + if (!listenProcess.isAlive) { throw exception() } - try { - Socket(hostAndPort.host, hostAndPort.port).close() - Unit - } catch (_exception: SocketException) { - null + when { + logFile.readText().contains("Running P2PMessaging loop") -> { + Unit + } + Instant.now().isAfter(stopPolling) -> { + // Waited for 20 seconds and the log file did not indicate that the PWP loop is running. + // This could be because the log is disabled, so lets try to create a client anyway. + Unit + } + else -> { + null + } } } } From ac4907a4297f2862143386105d82a8340f4daf5a Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Tue, 14 Jul 2020 08:04:52 +0100 Subject: [PATCH 03/48] CORDA-3721 Finishing + retrying a flow cancels its future (#6454) Cancel the future being run by a flow when finishing or retrying it. The cancellation of the future no longer cares about what type of future it is. `StateMachineState` has the `future` field, which holds the 3 (currently) possible types of futures: - sleep - wait for ledger commit - async operation / external operation Move the starting of all futures triggered by actions into `ActionFutureExecutor`. --- .../node/services/statemachine/Action.kt | 8 +- .../statemachine/ActionExecutorImpl.kt | 43 ++------- .../statemachine/ActionFutureExecutor.kt | 96 +++++++++++++++++++ .../statemachine/FlowSleepScheduler.kt | 78 --------------- .../SingleThreadedStateMachineManager.kt | 32 ++++--- .../statemachine/StateMachineManager.kt | 2 - .../transitions/StartedFlowTransition.kt | 7 +- 7 files changed, 134 insertions(+), 132 deletions(-) create mode 100644 node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt delete mode 100644 node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt index 51aadc69cf..6b17fe0870 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt @@ -17,7 +17,7 @@ sealed class Action { /** * Track a transaction hash and notify the state machine once the corresponding transaction has committed. */ - data class TrackTransaction(val hash: SecureHash) : Action() + data class TrackTransaction(val hash: SecureHash, val currentState: StateMachineState) : Action() /** * Send an initial session message to [destination]. @@ -140,7 +140,11 @@ sealed class Action { /** * Execute the specified [operation]. */ - data class ExecuteAsyncOperation(val deduplicationId: String, val operation: FlowAsyncOperation<*>) : Action() + data class ExecuteAsyncOperation( + val deduplicationId: String, + val operation: FlowAsyncOperation<*>, + val currentState: StateMachineState + ) : Action() /** * Release soft locks associated with given ID (currently the flow ID). diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 260ea86cac..2849dc03a1 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -3,7 +3,6 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable import com.codahale.metrics.Gauge import com.codahale.metrics.Reservoir -import net.corda.core.internal.concurrent.thenMatch import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.checkpointSerialize @@ -15,17 +14,17 @@ import net.corda.nodeapi.internal.persistence.contextDatabase import net.corda.nodeapi.internal.persistence.contextTransaction import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import java.sql.SQLException -import java.time.Duration /** * This is the bottom execution engine of flow side-effects. */ internal class ActionExecutorImpl( - private val services: ServiceHubInternal, - private val checkpointStorage: CheckpointStorage, - private val flowMessaging: FlowMessaging, - private val stateMachineManager: StateMachineManagerInternal, - private val checkpointSerializationContext: CheckpointSerializationContext + private val services: ServiceHubInternal, + private val checkpointStorage: CheckpointStorage, + private val flowMessaging: FlowMessaging, + private val stateMachineManager: StateMachineManagerInternal, + private val actionFutureExecutor: ActionFutureExecutor, + private val checkpointSerializationContext: CheckpointSerializationContext ) : ActionExecutor { private companion object { @@ -74,16 +73,8 @@ internal class ActionExecutorImpl( if (action.uuid != null) services.vaultService.softLockRelease(action.uuid) } - @Suspendable private fun executeTrackTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { - services.validatedTransactions.trackTransactionWithNoWarning(action.hash).thenMatch( - success = { transaction -> - fiber.scheduleEvent(Event.TransactionCommitted(transaction)) - }, - failure = { exception -> - fiber.scheduleEvent(Event.Error(exception)) - } - ) + actionFutureExecutor.awaitTransaction(fiber, action) } @Suspendable @@ -157,13 +148,8 @@ internal class ActionExecutorImpl( fiber.scheduleEvent(action.event) } - @Suspendable private fun executeSleepUntil(fiber: FlowFiber, action: Action.SleepUntil) { - stateMachineManager.scheduleFlowSleep( - fiber, - action.currentState, - Duration.between(services.clock.instant(), action.time) - ) + actionFutureExecutor.sleep(fiber, action) } @Suspendable @@ -236,19 +222,10 @@ internal class ActionExecutorImpl( } } - @Suppress("TooGenericExceptionCaught") // this is fully intentional here, see comment in the catch clause - @Suspendable + @Suppress("TooGenericExceptionCaught") private fun executeAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { try { - val operationFuture = action.operation.execute(action.deduplicationId) - operationFuture.thenMatch( - success = { result -> - fiber.scheduleEvent(Event.AsyncOperationCompletion(result)) - }, - failure = { exception -> - fiber.scheduleEvent(Event.AsyncOperationThrows(exception)) - } - ) + actionFutureExecutor.awaitAsyncOperation(fiber, action) } catch (e: Exception) { // Catch and wrap any unexpected exceptions from the async operation // Wrapping the exception allows it to be better handled by the flow hospital diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt new file mode 100644 index 0000000000..dc5d2fc0b9 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt @@ -0,0 +1,96 @@ +package net.corda.node.services.statemachine + +import net.corda.core.internal.concurrent.thenMatch +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.node.services.api.ServiceHubInternal +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit + +internal class ActionFutureExecutor( + private val innerState: StateMachineInnerState, + private val services: ServiceHubInternal, + private val scheduledExecutor: ScheduledExecutorService +) { + + private companion object { + val log = contextLogger() + } + + /** + * Put a flow to sleep for the duration specified in [action]. + * + * @param fiber The [FlowFiber] that will be woken up after sleeping + * @param action The [Action.SleepUntil] to create a future from + */ + fun sleep(fiber: FlowFiber, action: Action.SleepUntil) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + val duration = Duration.between(services.clock.instant(), action.time) + log.debug { "Putting flow ${instance.runId} to sleep for $duration" } + val future = scheduledExecutor.schedule( + { + log.debug { "Scheduling flow wake up event for flow ${instance.runId}" } + scheduleWakeUpEvent(instance, Event.WakeUpFromSleep) + }, + duration.toMillis(), TimeUnit.MILLISECONDS + ) + action.currentState.future = future + } + + /** + * Suspend a flow until its async operation specified in [action] is completed. + * + * @param fiber The [FlowFiber] to resume after completing the async operation + * @param action The [Action.ExecuteAsyncOperation] to create a future from + */ + fun awaitAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + log.debug { "Suspending flow ${instance.runId} until its async operation has completed" } + val future = action.operation.execute(action.deduplicationId) + future.thenMatch( + success = { result -> scheduleWakeUpEvent(instance, Event.AsyncOperationCompletion(result)) }, + failure = { exception -> scheduleWakeUpEvent(instance, Event.AsyncOperationThrows(exception)) } + ) + action.currentState.future = future + } + + /** + * Suspend a flow until the transaction specified in [action] is committed. + * + * @param fiber The [FlowFiber] to resume after the committing the specified transaction + * @param action [Action.TrackTransaction] contains the transaction hash to wait for + */ + fun awaitTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + log.debug { "Suspending flow ${instance.runId} until transaction ${action.hash} is committed" } + val future = services.validatedTransactions.trackTransactionWithNoWarning(action.hash) + future.thenMatch( + success = { transaction -> scheduleWakeUpEvent(instance, Event.TransactionCommitted(transaction)) }, + failure = { exception -> scheduleWakeUpEvent(instance, Event.Error(exception)) } + ) + action.currentState.future = future + } + + private fun cancelFutureIfRunning(fiber: FlowFiber, currentState: StateMachineState) { + // No other future should be running, cancel it if there is + currentState.future?.run { + log.debug { "Cancelling existing future for flow ${fiber.id}" } + if (!isDone) cancel(true) + } + } + + private fun scheduleWakeUpEvent(instance: StateMachineInstanceId, event: Event) { + innerState.withLock { + flows[instance.runId]?.let { flow -> + // Only schedule a wake up event if the fiber the flow is executing on has not changed + if (flow.fiber.instanceId == instance) { + flow.fiber.scheduleEvent(event) + } + } + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt deleted file mode 100644 index 63fcd5c6e8..0000000000 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt +++ /dev/null @@ -1,78 +0,0 @@ -package net.corda.node.services.statemachine - -import net.corda.core.internal.FlowIORequest -import net.corda.core.utilities.contextLogger -import net.corda.core.utilities.debug -import java.time.Duration -import java.util.concurrent.Future -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit - -internal class FlowSleepScheduler(private val innerState: StateMachineInnerState, private val scheduledExecutor: ScheduledExecutorService) { - - private companion object { - val log = contextLogger() - } - - /** - * Put a flow to sleep for a specified duration. - * - * @param fiber The [FlowFiber] that will be woken up after sleeping - * @param currentState The current [StateMachineState] - * @param duration How long to sleep for - */ - fun sleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) { - // No other future should be running, cancel it if there is - currentState.future?.run { - log.debug { "Cancelling the existing future for flow ${fiber.id}" } - cancelIfRunning() - } - currentState.future = setAlarmClock(fiber, duration) - } - - /** - * Cancel a sleeping flow's future. Note, this does not cause the flow to wake up. - * - * @param currentState The current [StateMachineState] - */ - fun cancel(currentState: StateMachineState) { - (currentState.checkpoint.flowState as? FlowState.Started)?.let { flowState -> - if (currentState.isWaitingForFuture && flowState.flowIORequest is FlowIORequest.Sleep) { - (currentState.future as? ScheduledFuture)?.run { - log.debug { "Cancelling the sleep scheduled future for flow ${currentState.flowLogic.runId}" } - cancelIfRunning() - currentState.future = null - } - } - - } - } - - private fun Future<*>.cancelIfRunning() { - if (!isDone) cancel(true) - } - - private fun setAlarmClock(fiber: FlowFiber, duration: Duration): ScheduledFuture { - val instance = fiber.instanceId - log.debug { "Putting flow ${instance.runId} to sleep for $duration" } - return scheduledExecutor.schedule( - { - log.debug { "Scheduling flow wake up event for flow ${instance.runId}" } - scheduleWakeUp(instance) - }, - duration.toMillis(), TimeUnit.MILLISECONDS - ) - } - - private fun scheduleWakeUp(instance: StateMachineInstanceId) { - innerState.withLock { - flows[instance.runId]?.let { flow -> - // Only schedule a wake up event if the fiber the flow is executing on has not changed - if (flow.fiber.instanceId == instance) { - flow.fiber.scheduleEvent(Event.WakeUpFromSleep) - } - } - } - } -} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 2914aecd5d..1d07a75d02 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -47,9 +47,8 @@ import net.corda.serialization.internal.withTokenContext import org.apache.activemq.artemis.utils.ReusableLatch import rx.Observable import java.security.SecureRandom -import java.time.Duration +import java.util.ArrayList import java.util.HashSet -import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -88,7 +87,7 @@ internal class SingleThreadedStateMachineManager( private val metrics = serviceHub.monitoringService.metrics private val sessionToFlow = ConcurrentHashMap() private val flowMessaging: FlowMessaging = FlowMessagingImpl(serviceHub) - private val flowSleepScheduler = FlowSleepScheduler(innerState, scheduledFutureExecutor) + private val actionFutureExecutor = ActionFutureExecutor(innerState, serviceHub, scheduledFutureExecutor) private val flowTimeoutScheduler = FlowTimeoutScheduler(innerState, scheduledFutureExecutor, serviceHub) private val fiberDeserializationChecker = if (serviceHub.configuration.shouldCheckCheckpoints()) FiberDeserializationChecker() else null private val ourSenderUUID = serviceHub.networkService.ourSenderUUID @@ -316,7 +315,7 @@ internal class SingleThreadedStateMachineManager( override fun removeFlow(flowId: StateMachineRunId, removalReason: FlowRemovalReason, lastState: StateMachineState) { innerState.withLock { flowTimeoutScheduler.cancel(flowId) - flowSleepScheduler.cancel(lastState) + lastState.cancelFutureIfRunning() val flow = flows.remove(flowId) if (flow != null) { decrementLiveFibers() @@ -384,7 +383,7 @@ internal class SingleThreadedStateMachineManager( @Suppress("TooGenericExceptionCaught", "ComplexMethod", "MaxLineLength") // this is fully intentional here, see comment in the catch clause override fun retryFlowFromSafePoint(currentState: StateMachineState) { - flowSleepScheduler.cancel(currentState) + currentState.cancelFutureIfRunning() // Get set of external events val flowId = currentState.flowLogic.runId val oldFlowLeftOver = innerState.withLock { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue @@ -634,10 +633,6 @@ internal class SingleThreadedStateMachineManager( flowTimeoutScheduler.cancel(flowId) } - override fun scheduleFlowSleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) { - flowSleepScheduler.sleep(fiber, currentState, duration) - } - private fun tryDeserializeCheckpoint(serializedCheckpoint: Checkpoint.Serialized, flowId: StateMachineRunId): Checkpoint? { return try { serializedCheckpoint.deserialize(checkpointSerializationContext!!) @@ -695,11 +690,12 @@ internal class SingleThreadedStateMachineManager( private fun makeActionExecutor(checkpointSerializationContext: CheckpointSerializationContext): ActionExecutor { return ActionExecutorImpl( - serviceHub, - checkpointStorage, - flowMessaging, - this, - checkpointSerializationContext + serviceHub, + checkpointStorage, + flowMessaging, + this, + actionFutureExecutor, + checkpointSerializationContext ) } @@ -781,4 +777,12 @@ internal class SingleThreadedStateMachineManager( } } } + + private fun StateMachineState.cancelFutureIfRunning() { + future?.run { + logger.debug { "Cancelling future for flow ${flowLogic.runId}" } + if (!isDone) cancel(true) + future = null + } + } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt index 66a5a60797..6c5050962b 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt @@ -10,7 +10,6 @@ import net.corda.core.utilities.Try import net.corda.node.services.messaging.DeduplicationHandler import net.corda.node.services.messaging.ReceivedMessage import rx.Observable -import java.time.Duration /** * A StateMachineManager is responsible for coordination and persistence of multiple [FlowStateMachine] objects. @@ -110,7 +109,6 @@ internal interface StateMachineManagerInternal { fun retryFlowFromSafePoint(currentState: StateMachineState) fun scheduleFlowTimeout(flowId: StateMachineRunId) fun cancelFlowTimeout(flowId: StateMachineRunId) - fun scheduleFlowSleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) } /** diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt index 904ab3f06a..96b6557829 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt @@ -105,11 +105,12 @@ class StartedFlowTransition( // This ensures that the [WaitForLedgerCommit] request is not executed multiple times if extra // [DoRemainingWork] events are pushed onto the fiber's event queue before the flow has really woken up return if (!startingState.isWaitingForFuture) { + val state = startingState.copy(isWaitingForFuture = true) TransitionResult( - newState = startingState.copy(isWaitingForFuture = true), + newState = state, actions = listOf( Action.CreateTransaction, - Action.TrackTransaction(flowIORequest.hash), + Action.TrackTransaction(flowIORequest.hash, state), Action.CommitTransaction ) ) @@ -432,8 +433,8 @@ class StartedFlowTransition( // The `numberOfSuspends` is added to the deduplication ID in case an async // operation is executed multiple times within the same flow. val deduplicationId = context.id.toString() + ":" + currentState.checkpoint.checkpointState.numberOfSuspends.toString() - actions.add(Action.ExecuteAsyncOperation(deduplicationId, flowIORequest.operation)) currentState = currentState.copy(isWaitingForFuture = true) + actions += Action.ExecuteAsyncOperation(deduplicationId, flowIORequest.operation, currentState) FlowContinuation.ProcessEvents } } else { From 75bade2f9292c5b5cf85466ef2d85a2703fceb1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 15 Jul 2020 07:46:26 +0100 Subject: [PATCH 04/48] Added NexusIQ scan to nightly snapshot build (#6461) * also build on standard agent, no K8S requirements --- .ci/dev/publish-branch/Jenkinsfile.nightly | 37 +++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly index f7a35981f6..14b2452af8 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.nightly +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -1,11 +1,27 @@ #!groovy +/** + * Jenkins pipeline to build Corda OS nightly snapshots + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/* +** calculate the stage for NexusIQ evaluation +** * build for snapshots +*/ +def nexusIqStage = "build" + pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() @@ -27,6 +43,25 @@ pipeline { } stages { + stage('Sonatype Check') { + steps { + sh "./gradlew --no-daemon clean jar" + script { + sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" + } + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: manualApplication(nexusAppId), + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: nexusIqStage + ) + } + } + stage('Publish to Artifactory') { steps { rtServer ( From 57e5f27961686b84d780e030626463f554662861 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Wed, 15 Jul 2020 09:09:28 +0100 Subject: [PATCH 05/48] CORDA-3906: Allow usage of SchedulableState in deterministic CorDapps. (#6457) --- .../src/main/kotlin/net/corda/core/contracts/Structures.kt | 6 ++++-- core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/contracts/Structures.kt b/core/src/main/kotlin/net/corda/core/contracts/Structures.kt index 16d2c8cd85..bfae2f8900 100644 --- a/core/src/main/kotlin/net/corda/core/contracts/Structures.kt +++ b/core/src/main/kotlin/net/corda/core/contracts/Structures.kt @@ -89,6 +89,7 @@ interface OwnableState : ContractState { // DOCEND 3 /** Something which is scheduled to happen at a point in time. */ +@KeepForDJVM interface Scheduled { val scheduledAt: Instant } @@ -101,6 +102,7 @@ interface Scheduled { * lifecycle processing needs to take place. e.g. a fixing or a late payment etc. */ @CordaSerializable +@KeepForDJVM data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instant) : Scheduled /** @@ -115,7 +117,7 @@ data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instan * for a particular [ContractState] have been processed/fired etc. If the activity is not "on ledger" then the * scheduled activity shouldn't be either. */ -@DeleteForDJVM +@KeepForDJVM data class ScheduledActivity(val logicRef: FlowLogicRef, override val scheduledAt: Instant) : Scheduled // DOCSTART 2 @@ -134,7 +136,7 @@ interface LinearState : ContractState { val linearId: UniqueIdentifier } // DOCEND 2 -@DeleteForDJVM +@KeepForDJVM interface SchedulableState : ContractState { /** * Indicate whether there is some activity to be performed at some future point in time with respect to this diff --git a/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt b/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt index 7781c38b95..1b08620e2a 100644 --- a/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt +++ b/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt @@ -1,7 +1,9 @@ package net.corda.core.flows import net.corda.core.CordaInternal +import net.corda.core.DeleteForDJVM import net.corda.core.DoNotImplement +import net.corda.core.KeepForDJVM import net.corda.core.serialization.CordaSerializable /** @@ -11,11 +13,13 @@ import net.corda.core.serialization.CordaSerializable * the flow to run at the scheduled time. */ @DoNotImplement +@KeepForDJVM interface FlowLogicRefFactory { /** * Construct a FlowLogicRef. This is intended for cases where the calling code has the relevant class already * and can provide it directly. */ + @DeleteForDJVM fun create(flowClass: Class>, vararg args: Any?): FlowLogicRef /** @@ -30,12 +34,14 @@ interface FlowLogicRefFactory { * [SchedulableFlow] annotation. */ @CordaInternal + @DeleteForDJVM fun createForRPC(flowClass: Class>, vararg args: Any?): FlowLogicRef /** * Converts a [FlowLogicRef] object that was obtained from the calls above into a [FlowLogic], after doing some * validation to ensure it points to a legitimate flow class. */ + @DeleteForDJVM fun toFlowLogic(ref: FlowLogicRef): FlowLogic<*> } @@ -59,4 +65,5 @@ class IllegalFlowLogicException(val type: String, msg: String) : // TODO: align this with the existing [FlowRef] in the bank-side API (probably replace some of the API classes) @CordaSerializable @DoNotImplement +@KeepForDJVM interface FlowLogicRef \ No newline at end of file From ef9934ed4e18dbdbc30ad57425be8d482e80fff6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 15 Jul 2020 10:17:58 +0100 Subject: [PATCH 06/48] Added build discarder settings back (#6465) builds are expired only after 14 days --- .ci/dev/compatibility/JenkinsfileJDK11Compile | 3 ++- .ci/dev/mswin/Jenkinsfile | 1 + .ci/dev/nightly-regression/Jenkinsfile | 1 + .ci/dev/pr-code-checks/Jenkinsfile | 1 + .ci/dev/publish-branch/Jenkinsfile.nightly | 1 + .ci/dev/publish-branch/Jenkinsfile.preview | 1 + .ci/dev/regression/Jenkinsfile | 1 + Jenkinsfile | 1 + 8 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index d2251fad15..f6e9c43195 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -14,6 +14,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } stages { @@ -35,4 +36,4 @@ pipeline { deleteDir() /* clean up our workspace */ } } -} \ No newline at end of file +} diff --git a/.ci/dev/mswin/Jenkinsfile b/.ci/dev/mswin/Jenkinsfile index 923a83ff85..b0e3766e0c 100644 --- a/.ci/dev/mswin/Jenkinsfile +++ b/.ci/dev/mswin/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { ansiColor('xterm') timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) /* * a bit awkward to read diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index dc3979ae23..5c2af45b99 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { timestamps() overrideIndexTriggers(false) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } triggers { pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight' diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index c74639b4f6..a64813c92f 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly index f7a35981f6..b25549729e 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.nightly +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -12,6 +12,7 @@ pipeline { ansiColor('xterm') overrideIndexTriggers(false) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } triggers { diff --git a/.ci/dev/publish-branch/Jenkinsfile.preview b/.ci/dev/publish-branch/Jenkinsfile.preview index e0fb92aa77..e66deeabab 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.preview +++ b/.ci/dev/publish-branch/Jenkinsfile.preview @@ -12,6 +12,7 @@ pipeline { ansiColor('xterm') overrideIndexTriggers(false) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 51f507dac2..4ad11d28f4 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -38,6 +38,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { diff --git a/Jenkinsfile b/Jenkinsfile index 02365f00a4..5c9a1add5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { From 6bae99ef1009b9a64218396d458af9c5c22a4058 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Wed, 15 Jul 2020 10:46:29 +0100 Subject: [PATCH 07/48] CORDA-3909: Upgrade to Corda Gradle plugins 5.0.11. (#6463) This also upgrades the following plugins: - Artifactory: 4.7.3 -> 4.16.1 - Bintray: 1.4 -> 1.8.5 --- build.gradle | 4 +--- constants.properties | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 5286b90831..039ce0ebc8 100644 --- a/build.gradle +++ b/build.gradle @@ -170,7 +170,6 @@ buildscript { dependencies { classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" classpath "org.jetbrains.kotlin:kotlin-allopen:$kotlin_version" - classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.4' classpath "net.corda.plugins:publish-utils:$gradle_plugins_version" classpath "net.corda.plugins:quasar-utils:$gradle_plugins_version" classpath "net.corda.plugins:cordformation:$gradle_plugins_version" @@ -204,7 +203,6 @@ plugins { apply plugin: 'project-report' apply plugin: 'com.github.ben-manes.versions' apply plugin: 'net.corda.plugins.publish-utils' -apply plugin: 'maven-publish' apply plugin: 'com.jfrog.artifactory' apply plugin: "com.bmuschko.docker-remote-api" apply plugin: "com.r3.dependx.dependxies" @@ -626,7 +624,7 @@ dependxiesModule { skipTasks = "test,integrationTest,smokeTest,slowIntegrationTest" } -tasks.register('generateApi', net.corda.plugins.GenerateApi) { +tasks.register('generateApi', net.corda.plugins.apiscanner.GenerateApi) { baseName = "api-corda" } diff --git a/constants.properties b/constants.properties index 2b2775389d..c9877793fb 100644 --- a/constants.properties +++ b/constants.properties @@ -4,7 +4,7 @@ cordaVersion=4.6 versionSuffix=SNAPSHOT -gradlePluginsVersion=5.0.9 +gradlePluginsVersion=5.0.11 kotlinVersion=1.2.71 java8MinUpdateVersion=171 # ***************************************************************# @@ -25,7 +25,7 @@ classgraphVersion=4.8.78 disruptorVersion=3.4.2 typesafeConfigVersion=1.3.4 jsr305Version=3.0.2 -artifactoryPluginVersion=4.7.3 +artifactoryPluginVersion=4.16.1 snakeYamlVersion=1.19 caffeineVersion=2.7.0 metricsVersion=4.1.0 From b58609daba10b3be44b7f0cce48ecd831a2f9d75 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Wed, 15 Jul 2020 21:33:49 +0100 Subject: [PATCH 08/48] Correct name of Artifactory repository for publishing releases --- .ci/dev/regression/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 4ad11d28f4..0361303e5b 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -134,7 +134,7 @@ pipeline { rtGradleDeployer( id: 'deployer', serverId: 'R3-Artifactory', - repo: 'r3-corda-releases' + repo: 'corda-releases' ) rtGradleRun( usesPlugin: true, From 2fa6b5a208c432ef209c208daadba6b4a1a46286 Mon Sep 17 00:00:00 2001 From: Adel El-Beik <48713346+adelel1@users.noreply.github.com> Date: Thu, 16 Jul 2020 09:58:36 +0100 Subject: [PATCH 09/48] CORDA-3769: Switched attachments class loader cache to use caffeine (#6326) * CORDA-3769: Switched attachments class loader cache to use caffeine with original implementation used by determinstic core. * CORDA-3769: Removed default ctor arguments. * CORDA-3769: Switched mapping function to Function type to avoid synthetic method being generated. * CORDA-3769: Now using a cache created from NamedCacheFactory for the attachments class loader cache. * CORDA-3769: Making detekt happy. * CORDA-3769: The finality tests now check for UntrustedAttachmentsException which will actually happen in reality. * CORDA-3769: Refactored after review comments. * CORDA-3769: Removed the AttachmentsClassLoaderSimpleCacheImpl as DJVM does not need it. Also updated due to review comments. * CORDA-3769: Removed the generic parameters from AttachmentsClassLoader. * CORDA-3769: Removed unused imports. * CORDA-3769: Updates from review comments. * CORDA-3769: Updated following review comments. MigrationServicesForResolution now uses cache factory. Ctor updated for AttachmentsClassLoaderSimpleCacheImpl. * CORDA-3769: Reduced max class loader cache size * CORDA-3769: Fixed the attachments class loader cache size to a fixed default * CORDA-3769: Switched attachments class loader size to be reduced by fixed value. --- .../flows/ReceiveFinalityFlowTest.kt | 6 +-- ...ttachmentsClassLoaderSerializationTests.kt | 2 +- .../AttachmentsClassLoaderTests.kt | 5 +- .../transactions/TransactionTests.kt | 10 +++- .../core/internal/ServiceHubCoreInternal.kt | 3 ++ .../internal/AttachmentsClassLoader.kt | 52 +++++++++++++++---- .../ContractUpgradeTransactions.kt | 3 +- .../core/transactions/LedgerTransaction.kt | 34 ++++++++---- .../core/transactions/WireTransaction.kt | 17 ++++-- .../internal/internalAccessTestHelpers.kt | 7 ++- .../net/corda/node/internal/AbstractNode.kt | 5 ++ .../migration/MigrationNamedCacheFactory.kt | 1 + .../MigrationServicesForResolution.kt | 7 ++- .../corda/node/utilities/NodeNamedCache.kt | 5 +- .../node/services/FinalityHandlerTest.kt | 8 +-- .../kotlin/net/corda/testing/dsl/TestDSL.kt | 4 ++ .../internal/TestingNamedCacheFactory.kt | 1 + 17 files changed, 127 insertions(+), 43 deletions(-) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt index 6c50e4e2a2..2a45e2105e 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt @@ -53,7 +53,7 @@ class ReceiveFinalityFlowTest { val paymentReceiverId = paymentReceiverFuture.getOrThrow() assertThat(bob.services.vaultService.queryBy>().states).isEmpty() - bob.assertFlowSentForObservationDueToConstraintError(paymentReceiverId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(paymentReceiverId) // Restart Bob with the contracts CorDapp so that it can recover from the error bob = mockNet.restartNode(bob, parameters = InternalMockNodeParameters(additionalCordapps = listOf(FINANCE_CONTRACTS_CORDAPP))) @@ -69,7 +69,7 @@ class ReceiveFinalityFlowTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -77,6 +77,6 @@ class ReceiveFinalityFlowTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt index 5cfaf252cb..4ca58d6b46 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt @@ -55,7 +55,7 @@ class AttachmentsClassLoaderSerializationTests { arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! }, testNetworkParameters(), SecureHash.zeroHash, - { attachmentTrustCalculator.calculate(it) }) { classLoader -> + { attachmentTrustCalculator.calculate(it) }, attachmentsClassLoaderCache = null) { classLoader -> val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader) val contract = contractClass.getDeclaredConstructor().newInstance() as Contract assertEquals("helloworld", contract.declaredField("magicString").value) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt index fcc081efb6..986b6052ef 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt @@ -23,6 +23,7 @@ import net.corda.core.internal.inputStream import net.corda.core.node.NetworkParameters import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.internal.AttachmentsClassLoader +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.testing.common.internal.testNetworkParameters import net.corda.node.services.attachments.NodeAttachmentTrustCalculator import net.corda.testing.contracts.DummyContract @@ -521,6 +522,7 @@ class AttachmentsClassLoaderTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) val transaction = createLedgerTransaction( inputs, outputs, @@ -532,7 +534,8 @@ class AttachmentsClassLoaderTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt index ec14b84029..500cfa9816 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt @@ -10,6 +10,7 @@ import net.corda.core.internal.AbstractAttachment import net.corda.core.internal.TESTDSL_UPLOADER import net.corda.core.internal.createLedgerTransaction import net.corda.core.node.NotaryInfo +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.WireTransaction import net.corda.testing.common.internal.testNetworkParameters @@ -18,6 +19,7 @@ import net.corda.testing.core.* import net.corda.testing.internal.createWireTransaction import net.corda.testing.internal.fakeAttachment import net.corda.coretesting.internal.rigorousMock +import net.corda.testing.internal.TestingNamedCacheFactory import org.junit.Rule import org.junit.Test import java.math.BigInteger @@ -131,6 +133,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) val transaction = createLedgerTransaction( inputs, outputs, @@ -142,7 +145,8 @@ class TransactionTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() @@ -183,6 +187,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) fun buildTransaction() = createLedgerTransaction( inputs, @@ -195,7 +200,8 @@ class TransactionTests { privacySalt, testNetworkParameters(notaries = listOf(NotaryInfo(DUMMY_NOTARY, true))), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) assertFailsWith { buildTransaction().verify() } diff --git a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt index 5897984c1d..2e59429fb5 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt @@ -5,6 +5,7 @@ import net.corda.core.DeleteForDJVM import net.corda.core.internal.notary.NotaryService import net.corda.core.node.ServiceHub import net.corda.core.node.StatesToRecord +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import java.util.concurrent.ExecutorService // TODO: This should really be called ServiceHubInternal but that name is already taken by net.corda.node.services.api.ServiceHubInternal. @@ -21,6 +22,8 @@ interface ServiceHubCoreInternal : ServiceHub { val notaryService: NotaryService? fun createTransactionsResolver(flow: ResolveTransactionsFlow): TransactionsResolver + + val attachmentsClassLoaderCache: AttachmentsClassLoaderCache } interface TransactionsResolver { diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt index eed759a08e..e93be2de5d 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt @@ -1,5 +1,8 @@ package net.corda.core.serialization.internal +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine +import net.corda.core.DeleteForDJVM import net.corda.core.contracts.Attachment import net.corda.core.contracts.ContractAttachment import net.corda.core.contracts.TransactionVerificationException @@ -21,6 +24,7 @@ import java.lang.ref.WeakReference import java.net.* import java.security.Permission import java.util.* +import java.util.function.Function /** * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only @@ -289,31 +293,27 @@ class AttachmentsClassLoader(attachments: List, */ @VisibleForTesting object AttachmentsClassLoaderBuilder { - private const val CACHE_SIZE = 1000 + const val CACHE_SIZE = 16 - // We use a set here because the ordering of attachments doesn't affect code execution, due to the no - // overlap rule, and attachments don't have any particular ordering enforced by the builders. So we - // can just do unordered comparisons here. But the same attachments run with different network parameters - // may behave differently, so that has to be a part of the cache key. - private data class Key(val hashes: Set, val params: NetworkParameters) - - // This runs in the DJVM so it can't use caffeine. - private val cache: MutableMap = createSimpleCache(CACHE_SIZE).toSynchronised() + private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE) /** * Runs the given block with serialization execution context set up with a (possibly cached) attachments classloader. * * @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup. */ + @Suppress("LongParameterList") fun withAttachmentsClassloaderContext(attachments: List, params: NetworkParameters, txId: SecureHash, isAttachmentTrusted: (Attachment) -> Boolean, parent: ClassLoader = ClassLoader.getSystemClassLoader(), + attachmentsClassLoaderCache: AttachmentsClassLoaderCache?, block: (ClassLoader) -> T): T { val attachmentIds = attachments.map(Attachment::id).toSet() - val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) { + val cache = attachmentsClassLoaderCache ?: fallBackCache + val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function { // Create classloader and load serializers, whitelisted classes val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, isAttachmentTrusted, parent) val serializers = try { @@ -336,7 +336,7 @@ object AttachmentsClassLoaderBuilder { .withWhitelist(whitelistedClasses) .withCustomSerializers(serializers) .withoutCarpenter() - } + }) // Deserialize all relevant classes in the transaction classloader. return SerializationFactory.defaultFactory.withCurrentContext(serializationContext) { @@ -420,6 +420,36 @@ private class AttachmentsHolderImpl : AttachmentsHolder { } } +interface AttachmentsClassLoaderCache { + fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext +} + +@DeleteForDJVM +class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache { + + private val cache: Cache = cacheFactory.buildNamed(Caffeine.newBuilder(), "AttachmentsClassLoader_cache") + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function") + } +} + +class AttachmentsClassLoaderSimpleCacheImpl(cacheSize: Int) : AttachmentsClassLoaderCache { + + private val cache: MutableMap + = createSimpleCache(cacheSize).toSynchronised() + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.computeIfAbsent(key, mappingFunction) + } +} + +// We use a set here because the ordering of attachments doesn't affect code execution, due to the no +// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we +// can just do unordered comparisons here. But the same attachments run with different network parameters +// may behave differently, so that has to be a part of the cache key. +data class AttachmentsClassLoaderKey(val hashes: Set, val params: NetworkParameters) + private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) { override fun getContentLengthLong(): Long = attachment.size.toLong() override fun getInputStream(): InputStream = attachment.open() diff --git a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt index c590047267..277dccc1d2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt @@ -153,7 +153,8 @@ data class ContractUpgradeWireTransaction( listOf(legacyAttachment, upgradedAttachment), params, id, - { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }) { transactionClassLoader -> + { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }, + attachmentsClassLoaderCache = (services as ServiceHubCoreInternal).attachmentsClassLoaderCache) { transactionClassLoader -> val resolvedInput = binaryInput.deserialize() val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader) val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment) diff --git a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt index 3dffc8182c..6c73c299c2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt @@ -26,6 +26,7 @@ import net.corda.core.internal.deserialiseComponentGroup import net.corda.core.internal.isUploaderTrusted import net.corda.core.internal.uncheckedCast import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder import net.corda.core.utilities.contextLogger import java.util.Collections.unmodifiableList @@ -87,7 +88,8 @@ private constructor( private val serializedInputs: List?, private val serializedReferences: List?, private val isAttachmentTrusted: (Attachment) -> Boolean, - private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier + private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier, + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ) : FullTransaction() { init { @@ -124,7 +126,8 @@ private constructor( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { return LedgerTransaction( inputs = inputs, @@ -141,7 +144,8 @@ private constructor( serializedInputs = protect(serializedInputs), serializedReferences = protect(serializedReferences), isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -176,7 +180,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { true }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) } } @@ -218,7 +223,8 @@ private constructor( txAttachments, getParamsWithGoo(), id, - isAttachmentTrusted = isAttachmentTrusted) { transactionClassLoader -> + isAttachmentTrusted = isAttachmentTrusted, + attachmentsClassLoaderCache = attachmentsClassLoaderCache) { transactionClassLoader -> // Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader]. // Only the copy will be used for verification, and the outer shell will be discarded. // This artifice is required to preserve backwards compatibility. @@ -254,7 +260,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = alternateVerifier + verifierFactory = alternateVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) // Read network parameters with backwards compatibility goo. @@ -320,7 +327,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } else { // This branch is only present for backwards compatibility. @@ -704,7 +712,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransaction should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -733,7 +742,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransactions should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -761,7 +771,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -791,7 +802,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } } diff --git a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt index ac7be9afeb..73b286276d 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt @@ -15,6 +15,7 @@ import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.serialize import net.corda.core.utilities.OpaqueBytes import java.security.PublicKey @@ -109,7 +110,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr services.networkParametersService.lookup(hashToResolve) }, // `as?` is used due to [MockServices] not implementing [ServiceHubCoreInternal] - isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true } + isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true }, + attachmentsClassLoaderCache = (services as? ServiceHubCoreInternal)?.attachmentsClassLoaderCache ) ) } @@ -145,7 +147,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, { null }, - { it.isUploaderTrusted() } + { it.isUploaderTrusted() }, + null ) } @@ -161,16 +164,19 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, resolveParameters, - { true } // Any attachment loaded through the DJVM should be trusted + { true }, // Any attachment loaded through the DJVM should be trusted + null ) } + @Suppress("LongParameterList", "ThrowsCount") private fun toLedgerTransactionInternal( resolveIdentity: (PublicKey) -> Party?, resolveAttachment: (SecureHash) -> Attachment?, resolveStateRefAsSerialized: (StateRef) -> SerializedBytes>?, resolveParameters: (SecureHash?) -> NetworkParameters?, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { // Look up public keys to authenticated identities. val authenticatedCommands = commands.lazyMapped { cmd, _ -> @@ -206,7 +212,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr componentGroups, serializedResolvedInputs, serializedResolvedReferences, - isAttachmentTrusted + isAttachmentTrusted, + attachmentsClassLoaderCache ) checkTransactionSize(ltx, resolvedNetworkParameters.maxTransactionSize, serializedResolvedInputs, serializedResolvedReferences) diff --git a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt index 148aaff7bf..c325c805e3 100644 --- a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt +++ b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt @@ -4,6 +4,7 @@ import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash import net.corda.core.identity.Party import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.transactions.ComponentGroup import net.corda.core.transactions.LedgerTransaction import net.corda.core.transactions.WireTransaction @@ -17,6 +18,7 @@ fun WireTransaction.accessGroupHashes() = this.groupHashes fun WireTransaction.accessGroupMerkleRoots() = this.groupsMerkleRoots fun WireTransaction.accessAvailableComponentHashes() = this.availableComponentHashes +@Suppress("LongParameterList") fun createLedgerTransaction( inputs: List>, outputs: List>, @@ -31,8 +33,9 @@ fun createLedgerTransaction( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean -): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted) + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache +): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted, attachmentsClassLoaderCache) fun createContractCreationError(txId: SecureHash, contractClass: String, cause: Throwable) = TransactionVerificationException.ContractCreationError(txId, contractClass, cause) fun createContractRejection(txId: SecureHash, contract: Contract, cause: Throwable) = TransactionVerificationException.ContractRejection(txId, contract, cause) diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 74dcee8c01..c115fd45a4 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -59,6 +59,8 @@ import net.corda.core.schemas.MappedSchema import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.toFuture import net.corda.core.transactions.LedgerTransaction import net.corda.core.utilities.NetworkHostAndPort @@ -317,6 +319,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } else { BasicVerifierFactoryService() } + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory).tokenize() val contractUpgradeService = ContractUpgradeServiceImpl(cacheFactory).tokenize() val auditService = DummyAuditService().tokenize() @Suppress("LeakingThis") @@ -1171,6 +1174,8 @@ abstract class AbstractNode(val configuration: NodeConfiguration, private lateinit var _myInfo: NodeInfo override val myInfo: NodeInfo get() = _myInfo + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache get() = this@AbstractNode.attachmentsClassLoaderCache + private lateinit var _networkParameters: NetworkParameters override val networkParameters: NetworkParameters get() = _networkParameters diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt index 70bb911106..0e2538e8ac 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt @@ -37,6 +37,7 @@ class MigrationNamedCacheFactory(private val metricRegistry: MetricRegistry?, "NodeAttachmentService_contractAttachmentVersions" -> caffeine.maximumSize(defaultCacheSize) "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name.") } } diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt index ec8b5d315d..0186b9659c 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt @@ -15,6 +15,8 @@ import net.corda.core.node.services.NetworkParametersService import net.corda.core.node.services.TransactionStorage import net.corda.core.serialization.deserialize import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.ContractUpgradeLedgerTransaction import net.corda.core.transactions.NotaryChangeLedgerTransaction import net.corda.core.transactions.WireTransaction @@ -62,6 +64,8 @@ class MigrationServicesForResolution( cacheFactory ) + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) + private fun defaultNetworkParameters(): NetworkParameters { logger.warn("Using a dummy set of network parameters for migration.") val clock = Clock.systemUTC() @@ -124,7 +128,8 @@ class MigrationServicesForResolution( networkParameters, tx.id, attachmentTrustCalculator::calculate, - cordappLoader.appClassLoader) { + cordappLoader.appClassLoader, + attachmentsClassLoaderCache) { deserialiseComponentGroup(tx.componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true) } states.filterIndexed {index, _ -> stateIndices.contains(index)}.toList() diff --git a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt index 4a514e0172..4d6dd05b19 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt @@ -63,6 +63,7 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi name == "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) name == "PublicKeyToOwningIdentityCache_cache" -> caffeine.maximumSize(defaultCacheSize) name == "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + name == "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultAttachmentsClassLoaderCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name. Did you add a new cache?") } } @@ -85,4 +86,6 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi } open protected val defaultCacheSize = 1024L -} \ No newline at end of file + private val defaultAttachmentsClassLoaderCacheSize = defaultCacheSize / CACHE_SIZE_DENOMINATOR +} +private const val CACHE_SIZE_DENOMINATOR = 4L \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt index 594932f5c0..b6bb0817b2 100644 --- a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt @@ -50,13 +50,13 @@ class FinalityHandlerTest { getOrThrow() } - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() bob = mockNet.restartNode(bob) // Since we've not done anything to fix the orignal error, we expect the finality handler to be sent to the hospital // again on restart - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() } @@ -96,7 +96,7 @@ class FinalityHandlerTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -104,7 +104,7 @@ class FinalityHandlerTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } private fun TestStartedNode.getTransaction(id: SecureHash): SignedTransaction? { diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt index fa3ec67c93..9a25595d63 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt @@ -14,6 +14,8 @@ import net.corda.core.node.ServiceHub import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.node.services.TransactionStorage +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction @@ -130,6 +132,8 @@ data class TestTransactionDSLInterpreter private constructor( ledgerInterpreter.services.cordappProvider override val notaryService: NotaryService? = null + + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) } private fun copy(): TestTransactionDSLInterpreter = diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt index 6802fd042e..402b3757c0 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt @@ -26,6 +26,7 @@ class TestingNamedCacheFactory private constructor(private val sizeOverride: Lon val configuredCaffeine = when (name) { "DBTransactionStorage_transactions" -> caffeine.maximumWeight(1.MB) "NodeAttachmentService_attachmentContent" -> caffeine.maximumWeight(1.MB) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(sizeOverride) else -> caffeine.maximumSize(sizeOverride) } return configuredCaffeine.build(loader) From 1660e7674b465f64648ae098a5c4bce0d8ded749 Mon Sep 17 00:00:00 2001 From: Euan Cairncross <35581327+cairncross@users.noreply.github.com> Date: Thu, 16 Jul 2020 10:46:42 +0100 Subject: [PATCH 10/48] INFRA-270 Publish archived API docs to Artifactory when tagged (#6309) * Reintroduce `build.gradle` from 4.4 * Add Jenkins publication logic Co-authored-by: Waldemar Zurowski Co-authored-by: Ross Nicoll --- .ci/dev/publish-api-docs/Jenkinsfile | 35 ++++++++ docs/build.gradle | 122 +++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 .ci/dev/publish-api-docs/Jenkinsfile create mode 100644 docs/build.gradle diff --git a/.ci/dev/publish-api-docs/Jenkinsfile b/.ci/dev/publish-api-docs/Jenkinsfile new file mode 100644 index 0000000000..d99d17ef44 --- /dev/null +++ b/.ci/dev/publish-api-docs/Jenkinsfile @@ -0,0 +1,35 @@ +@Library('corda-shared-build-pipeline-steps') + +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +pipeline { + agent { label 'standard' } + options { + ansiColor('xterm') + timestamps() + timeout(time: 3, unit: 'HOURS') + } + + environment { + ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" + } + + stages { + stage('Publish Archived API Docs to Artifactory') { + when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' } + steps { + sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs" + } + } + } + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} diff --git a/docs/build.gradle b/docs/build.gradle new file mode 100644 index 0000000000..09bdac83bc --- /dev/null +++ b/docs/build.gradle @@ -0,0 +1,122 @@ +import org.apache.tools.ant.taskdefs.condition.Os + +apply plugin: 'org.jetbrains.dokka' +apply plugin: 'net.corda.plugins.publish-utils' +apply plugin: 'maven-publish' +apply plugin: 'com.jfrog.artifactory' + +def internalPackagePrefixes(sourceDirs) { + def prefixes = [] + // Kotlin allows packages to deviate from the directory structure, but let's assume they don't: + sourceDirs.collect { sourceDir -> + sourceDir.traverse(type: groovy.io.FileType.DIRECTORIES) { + if (it.name == 'internal') { + prefixes.add sourceDir.toPath().relativize(it.toPath()).toString().replace(File.separator, '.') + } + } + } + prefixes +} + +ext { + // TODO: Add '../client/jfx/src/main/kotlin' and '../client/mock/src/main/kotlin' if we decide to make them into public API + dokkaSourceDirs = files('../core/src/main/kotlin', '../client/rpc/src/main/kotlin', '../finance/workflows/src/main/kotlin', '../finance/contracts/src/main/kotlin', '../client/jackson/src/main/kotlin', + '../testing/test-utils/src/main/kotlin', '../testing/node-driver/src/main/kotlin') + internalPackagePrefixes = internalPackagePrefixes(dokkaSourceDirs) + archivedApiDocsBaseFilename = 'api-docs' +} + +dokka { + outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/kotlin") +} + +task dokkaJavadoc(type: org.jetbrains.dokka.gradle.DokkaTask) { + outputFormat = "javadoc" + outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/javadoc") +} + +[dokka, dokkaJavadoc].collect { + it.configure { + moduleName = 'corda' + processConfigurations = ['compile'] + sourceDirs = dokkaSourceDirs + includes = ['packages.md'] + jdkVersion = 8 + externalDocumentationLink { + url = new URL("http://fasterxml.github.io/jackson-core/javadoc/2.9/") + } + externalDocumentationLink { + url = new URL("https://docs.oracle.com/javafx/2/api/") + } + externalDocumentationLink { + url = new URL("http://www.bouncycastle.org/docs/docs1.5on/") + } + internalPackagePrefixes.collect { packagePrefix -> + packageOptions { + prefix = packagePrefix + suppress = true + } + } + } +} + +task apidocs(dependsOn: ['dokka', 'dokkaJavadoc']) { + group "Documentation" + description "Build API documentation" +} + +task makeHTMLDocs(type: Exec){ + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-html.sh" + } else { + commandLine "bash", "-c", "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-html.sh" + } +} + +task makePDFDocs(type: Exec){ + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-pdf.sh" + } else { + commandLine "bash", "-c", "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-pdf.sh" + } +} + +task makeDocs(dependsOn: ['makeHTMLDocs', 'makePDFDocs']) +apidocs.shouldRunAfter makeDocs + +task archiveApiDocs(type: Tar) { + dependsOn apidocs + from buildDir + include 'html/**' + extension 'tgz' + compression Compression.GZIP +} + +publishing { + publications { + if (System.getProperty('publishApiDocs') != null) { + archivedApiDocs(MavenPublication) { + artifact archiveApiDocs { + artifactId archivedApiDocsBaseFilename + } + } + } + } +} + +artifactoryPublish { + publications('archivedApiDocs') + version = version.replaceAll('-SNAPSHOT', '') + publishPom = false +} + +artifactory { + publish { + contextUrl = artifactory_contextUrl + repository { + repoKey = 'corda-dependencies-dev' + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } +} From fad8999ef4b3b1c15a2e5f398b6ac5f3e2b7878c Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Thu, 16 Jul 2020 10:51:23 +0100 Subject: [PATCH 11/48] CORDA-3862: Make SharedMemoryIncremental public (#6467) --- .../java/net/corda/testing/driver/SharedMemoryIncremental.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java b/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java index fe0d957499..ae5a927186 100644 --- a/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java +++ b/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java @@ -17,7 +17,7 @@ import java.nio.channels.FileChannel; * import sun.misc.Unsafe; * import sun.nio.ch.DirectBuffer; */ -class SharedMemoryIncremental extends PortAllocation { +public class SharedMemoryIncremental extends PortAllocation { static private final int DEFAULT_START_PORT = 10_000; static private final int FIRST_EPHEMERAL_PORT = 30_000; From 9b77f9a170b09567f71622b73b9de8f3ad35e654 Mon Sep 17 00:00:00 2001 From: Razvan Codreanu <52859362+Schife@users.noreply.github.com> Date: Thu, 16 Jul 2020 16:46:10 +0100 Subject: [PATCH 12/48] INFRA-435 Don't publish internal releases to Docker (#6475) --- .ci/dev/regression/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 0361303e5b..85a699180a 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -18,6 +18,7 @@ killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) * Sense environment */ boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(? Date: Fri, 17 Jul 2020 09:39:45 +0100 Subject: [PATCH 13/48] INFRA-330 Use Artifactory as cache for all dependencies (#6253) * Use a virtual repo (corda-remotes) containing all Corda repositories with dependencies * activated when CORDA_USE_CACHE environment variable is set * Update Jenkins configuration to use new functionality * it does *not* affect local builds as long as environment variable is not set! --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 8 ++- .ci/dev/nightly-regression/Jenkinsfile | 8 ++- .ci/dev/regression/Jenkinsfile | 10 +++- Jenkinsfile | 8 ++- build.gradle | 66 +++++++++++++++++----- settings.gradle | 24 +++++++- 6 files changed, 100 insertions(+), 24 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index 4f04153ded..d24a3f7ac4 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -37,6 +37,9 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -68,6 +71,9 @@ pipeline { "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + "-Ddocker.buildbase.tag=11latest " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.dockerfile=DockerfileJDK11Azul" + " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace" } @@ -147,7 +153,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 5c2af45b99..c8d5b0e73a 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -20,6 +20,9 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -36,6 +39,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean pushBuildImage --stacktrace" } @@ -75,7 +81,6 @@ pipeline { } } - post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false @@ -86,4 +91,3 @@ pipeline { } } } - diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 66af41226a..77aad30eeb 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -48,6 +48,9 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::") + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -83,6 +86,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace" } @@ -170,7 +176,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true script { try { @@ -243,4 +249,4 @@ pipeline { deleteDir() /* clean up our workspace */ } } -} \ No newline at end of file +} diff --git a/Jenkinsfile b/Jenkinsfile index fc2c701c29..c27f461148 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,9 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -27,6 +30,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace" } @@ -73,7 +79,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/build.gradle b/build.gradle index e7d5dd3c84..4ecd0a9f8f 100644 --- a/build.gradle +++ b/build.gradle @@ -155,16 +155,34 @@ buildscript { ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion" repositories { mavenLocal() - mavenCentral() - jcenter() - maven { - url 'https://kotlin.bintray.com/kotlinx' - } - maven { - url "$artifactory_contextUrl/corda-dependencies-dev" - } - maven { - url "$artifactory_contextUrl/corda-releases" + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { + url 'https://kotlin.bintray.com/kotlinx' + } + maven { + url "${artifactory_contextUrl}/corda-dependencies-dev" + } + maven { + url "${artifactory_contextUrl}/corda-releases" + } } } dependencies { @@ -357,11 +375,29 @@ allprojects { repositories { mavenLocal() - mavenCentral() - jcenter() - maven { url "$artifactory_contextUrl/corda-dependencies" } - maven { url 'https://repo.gradle.org/gradle/libs-releases' } - maven { url "$artifactory_contextUrl/corda-dev" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + maven { url 'https://repo.gradle.org/gradle/libs-releases' } + maven { url "${artifactory_contextUrl}/corda-dev" } + } } configurations { diff --git a/settings.gradle b/settings.gradle index ae6dad0838..05bb5040fc 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,9 +2,27 @@ pluginManagement { ext.artifactory_contextUrl = 'https://software.r3.com/artifactory' repositories { - mavenLocal() - gradlePluginPortal() - maven { url "$artifactory_contextUrl/corda-dependencies" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenLocal() + gradlePluginPortal() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + } } } // The project is named 'corda-project' and not 'corda' because if this is named the same as the From 5d7060ec3a0f53285595a96719815c79010e3d2d Mon Sep 17 00:00:00 2001 From: Ross Nicoll Date: Fri, 17 Jul 2020 11:26:51 +0000 Subject: [PATCH 14/48] CORDA-3901 Eliminate random reconnect test (#6446) Remove a legacy test for RPC reconnection, which takes 5 minutes to run a random set of tests. This is expensive and low value. --- .../node/services/rpc/RpcReconnectTests.kt | 355 ------------------ 1 file changed, 355 deletions(-) delete mode 100644 node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt deleted file mode 100644 index 242cacdaad..0000000000 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt +++ /dev/null @@ -1,355 +0,0 @@ -package net.corda.node.services.rpc - -import net.corda.client.rpc.CordaRPCClient -import net.corda.client.rpc.CordaRPCClientConfiguration -import net.corda.client.rpc.GracefulReconnect -import net.corda.client.rpc.internal.ReconnectingCordaRPCOps -import net.corda.client.rpc.notUsed -import net.corda.core.contracts.Amount -import net.corda.core.flows.StateMachineRunId -import net.corda.core.internal.concurrent.transpose -import net.corda.core.messaging.StateMachineUpdate -import net.corda.core.node.services.Vault -import net.corda.core.node.services.vault.PageSpecification -import net.corda.core.node.services.vault.QueryCriteria -import net.corda.core.node.services.vault.builder -import net.corda.core.utilities.NetworkHostAndPort -import net.corda.core.utilities.OpaqueBytes -import net.corda.core.utilities.contextLogger -import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.seconds -import net.corda.finance.contracts.asset.Cash -import net.corda.finance.flows.CashIssueAndPaymentFlow -import net.corda.finance.schemas.CashSchemaV1 -import net.corda.node.services.Permissions -import net.corda.node.services.rpc.RpcReconnectTests.Companion.NUMBER_OF_FLOWS_TO_RUN -import net.corda.testing.core.DUMMY_BANK_A_NAME -import net.corda.testing.core.DUMMY_BANK_B_NAME -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeHandle -import net.corda.testing.driver.OutOfProcess -import net.corda.testing.driver.driver -import net.corda.testing.driver.internal.OutOfProcessImpl -import net.corda.testing.driver.internal.incrementalPortAllocation -import net.corda.testing.node.User -import net.corda.testing.node.internal.FINANCE_CORDAPPS -import org.assertj.core.api.Assertions.assertThat -import org.junit.Test -import java.util.* -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread -import kotlin.math.absoluteValue -import kotlin.math.max -import kotlin.test.assertEquals -import kotlin.test.assertTrue -import kotlin.test.currentStackTrace - -/** - * This is a stress test for the rpc reconnection logic, which triggers failures in a probabilistic way. - * - * You can adjust the variable [NUMBER_OF_FLOWS_TO_RUN] to adjust the number of flows to run and the duration of the test. - */ -class RpcReconnectTests { - - companion object { - // this many flows take ~5 minutes - const val NUMBER_OF_FLOWS_TO_RUN = 100 - - private val log = contextLogger() - } - - private val portAllocator = incrementalPortAllocation() - - private lateinit var proxy: RandomFailingProxy - private lateinit var node: NodeHandle - private lateinit var currentAddressPair: AddressPair - - /** - * This test showcases and stress tests the demo [ReconnectingCordaRPCOps]. - * - * Note that during node failure events can be lost and starting flows can become unreliable. - * The only available way to retry failed flows is to attempt a "logical retry" which is also showcased. - * - * This test runs flows in a loop and in the background kills the node or restarts it. - * Also the RPC connection is made through a proxy that introduces random latencies and is also periodically killed. - */ - @Suppress("ComplexMethod") - @Test(timeout=420_000) - fun `test that the RPC client is able to reconnect and proceed after node failure, restart, or connection reset`() { - val nodeRunningTime = { Random().nextInt(12000) + 8000 } - - val demoUser = User("demo", "demo", setOf(Permissions.all())) - - // When this reaches 0 - the test will end. - val flowsCountdownLatch = CountDownLatch(NUMBER_OF_FLOWS_TO_RUN) - // These are the expected progress steps for the CashIssueAndPayFlow. - val expectedProgress = listOf( - "Starting", - "Issuing cash", - "Generating transaction", - "Signing transaction", - "Finalising transaction", - "Broadcasting transaction to participants", - "Paying recipient", - "Generating anonymous identities", - "Generating transaction", - "Signing transaction", - "Finalising transaction", - "Requesting signature by notary service", - "Requesting signature by Notary service", - "Validating response from Notary service", - "Broadcasting transaction to participants", - "Done" - ) - - driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS, startNodesInProcess = false, inMemoryDB = false)) { - fun startBankA(address: NetworkHostAndPort) = startNode(providedName = DUMMY_BANK_A_NAME, rpcUsers = listOf(demoUser), customOverrides = mapOf("rpcSettings.address" to address.toString())) - fun startProxy(addressPair: AddressPair) = RandomFailingProxy(serverPort = addressPair.proxyAddress.port, remotePort = addressPair.nodeAddress.port).start() - - val addresses = (1..2).map { getRandomAddressPair() } - currentAddressPair = addresses[0] - - proxy = startProxy(currentAddressPair) - val (bankA, bankB) = listOf( - startBankA(currentAddressPair.nodeAddress), - startNode(providedName = DUMMY_BANK_B_NAME, rpcUsers = listOf(demoUser)) - ).transpose().getOrThrow() - node = bankA - - val notary = defaultNotaryIdentity - val baseAmount = Amount.parseCurrency("0 USD") - val issuerRef = OpaqueBytes.of(0x01) - - var numDisconnects = 0 - var numReconnects = 0 - val maxStackOccurrences = AtomicInteger() - - val addressesForRpc = addresses.map { it.proxyAddress } - // DOCSTART rpcReconnectingRPC - val onReconnect = { - numReconnects++ - // We only expect to see a single reconnectOnError in the stack trace. Otherwise we're in danger of stack overflow recursion - maxStackOccurrences.set(max(maxStackOccurrences.get(), currentStackTrace().count { it.methodName == "reconnectOnError" })) - Unit - } - val reconnect = GracefulReconnect(onDisconnect = { numDisconnects++ }, onReconnect = onReconnect) - val config = CordaRPCClientConfiguration.DEFAULT.copy( - connectionRetryInterval = 1.seconds, - connectionRetryIntervalMultiplier = 1.0 - ) - val client = CordaRPCClient(addressesForRpc, configuration = config) - val bankAReconnectingRPCConnection = client.start(demoUser.username, demoUser.password, gracefulReconnect = reconnect) - val bankAReconnectingRpc = bankAReconnectingRPCConnection.proxy as ReconnectingCordaRPCOps - // DOCEND rpcReconnectingRPC - - // Observe the vault and collect the observations. - val vaultEvents = Collections.synchronizedList(mutableListOf>()) - // DOCSTART rpcReconnectingRPCVaultTracking - val vaultFeed = bankAReconnectingRpc.vaultTrackByWithPagingSpec( - Cash.State::class.java, - QueryCriteria.VaultQueryCriteria(), - PageSpecification(1, 1)) - val vaultSubscription = vaultFeed.updates.subscribe { update: Vault.Update -> - log.info("vault update produced ${update.produced.map { it.state.data.amount }} consumed ${update.consumed.map { it.ref }}") - vaultEvents.add(update) - } - // DOCEND rpcReconnectingRPCVaultTracking - - // Observe the stateMachine and collect the observations. - val stateMachineEvents = Collections.synchronizedList(mutableListOf()) - val stateMachineSubscription = bankAReconnectingRpc.stateMachinesFeed().updates.subscribe { update -> - log.info(update.toString()) - stateMachineEvents.add(update) - } - - // While the flows are running, randomly apply a different failure scenario. - val nrRestarts = AtomicInteger() - thread(name = "Node killer") { - while (true) { - if (flowsCountdownLatch.count == 0L) break - - // Let the node run for a random time interval. - nodeRunningTime().also { ms -> - log.info("Running node for ${ms / 1000} s.") - Thread.sleep(ms.toLong()) - } - - if (flowsCountdownLatch.count == 0L) break - when (Random().nextInt().rem(7).absoluteValue) { - 0 -> { - log.info("Forcefully killing node and proxy.") - (node as OutOfProcessImpl).onStopCallback() - (node as OutOfProcess).process.destroyForcibly() - proxy.stop() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy.start() - } - 1 -> { - log.info("Forcefully killing node.") - (node as OutOfProcessImpl).onStopCallback() - (node as OutOfProcess).process.destroyForcibly() - node = startBankA(currentAddressPair.nodeAddress).get() - } - 2 -> { - log.info("Shutting down node.") - node.stop() - proxy.stop() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy.start() - } - 3, 4 -> { - log.info("Killing proxy.") - proxy.stop() - Thread.sleep(Random().nextInt(5000).toLong()) - proxy.start() - } - 5 -> { - log.info("Dropping connection.") - proxy.failConnection() - } - 6 -> { - log.info("Performing failover to a different node") - node.stop() - proxy.stop() - currentAddressPair = (addresses - currentAddressPair).first() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy = startProxy(currentAddressPair) - } - } - nrRestarts.incrementAndGet() - } - } - - // Start nrOfFlowsToRun and provide a logical retry function that checks the vault. - val flowProgressEvents = mutableMapOf>() - for (amount in (1..NUMBER_OF_FLOWS_TO_RUN)) { - // DOCSTART rpcReconnectingRPCFlowStarting - bankAReconnectingRpc.runFlowWithLogicalRetry( - runFlow = { rpc -> - log.info("Starting CashIssueAndPaymentFlow for $amount") - val flowHandle = rpc.startTrackedFlowDynamic( - CashIssueAndPaymentFlow::class.java, - baseAmount.plus(Amount.parseCurrency("$amount USD")), - issuerRef, - bankB.nodeInfo.legalIdentities.first(), - false, - notary - ) - val flowId = flowHandle.id - log.info("Started flow $amount with flowId: $flowId") - flowProgressEvents.addEvent(flowId, null) - - flowHandle.stepsTreeFeed?.updates?.notUsed() - flowHandle.stepsTreeIndexFeed?.updates?.notUsed() - // No reconnecting possible. - flowHandle.progress.subscribe( - { prog -> - flowProgressEvents.addEvent(flowId, prog) - log.info("Progress $flowId : $prog") - }, - { error -> - log.error("Error thrown in the flow progress observer", error) - }) - flowHandle.id - }, - hasFlowStarted = { rpc -> - // Query for a state that is the result of this flow. - val criteria = QueryCriteria.VaultCustomQueryCriteria(builder { CashSchemaV1.PersistentCashState::pennies.equal(amount.toLong() * 100) }, status = Vault.StateStatus.ALL) - val results = rpc.vaultQueryByCriteria(criteria, Cash.State::class.java) - log.info("$amount - Found states ${results.states}") - // The flow has completed if a state is found - results.states.isNotEmpty() - }, - onFlowConfirmed = { - flowsCountdownLatch.countDown() - log.info("Flow started for $amount. Remaining flows: ${flowsCountdownLatch.count}") - } - ) - // DOCEND rpcReconnectingRPCFlowStarting - - Thread.sleep(Random().nextInt(250).toLong()) - } - - log.info("Started all flows") - - // Wait until all flows have been started. - val flowsConfirmed = flowsCountdownLatch.await(10, TimeUnit.MINUTES) - - if (flowsConfirmed) { - log.info("Confirmed all flows have started.") - } else { - log.info("Timed out waiting for confirmation that all flows have started. Remaining flows: ${flowsCountdownLatch.count}") - } - - - // Wait for all events to come in and flows to finish. - Thread.sleep(4000) - - val nrFailures = nrRestarts.get() - log.info("Checking results after $nrFailures restarts.") - - // We should get one disconnect and one reconnect for each failure - assertThat(numDisconnects).isEqualTo(numReconnects) - assertThat(numReconnects).isLessThanOrEqualTo(nrFailures) - assertThat(maxStackOccurrences.get()).isLessThan(2) - - // Query the vault and check that states were created for all flows. - fun readCashStates() = bankAReconnectingRpc - .vaultQueryByWithPagingSpec(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(status = Vault.StateStatus.CONSUMED), PageSpecification(1, 10000)) - .states - - var allCashStates = readCashStates() - var nrRetries = 0 - - // It might be necessary to wait more for all events to arrive when the node is slow. - while (allCashStates.size < NUMBER_OF_FLOWS_TO_RUN && nrRetries++ < 50) { - Thread.sleep(2000) - allCashStates = readCashStates() - } - - val allCash = allCashStates.map { it.state.data.amount.quantity }.toSet() - val missingCash = (1..NUMBER_OF_FLOWS_TO_RUN).filterNot { allCash.contains(it.toLong() * 100) } - log.info("Missing cash states: $missingCash") - - assertEquals(NUMBER_OF_FLOWS_TO_RUN, allCashStates.size, "Not all flows were executed successfully") - - // The progress status for each flow can only miss the last events, because the node might have been killed. - val missingProgressEvents = flowProgressEvents.filterValues { expectedProgress.subList(0, it.size) != it } - assertTrue(missingProgressEvents.isEmpty(), "The flow progress tracker is missing events: $missingProgressEvents") - - // DOCSTART missingVaultEvents - // Check that enough vault events were received. - // This check is fuzzy because events can go missing during node restarts. - // Ideally there should be nrOfFlowsToRun events receive but some might get lost for each restart. - assertThat(vaultEvents!!.size + nrFailures * 3).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN) - // DOCEND missingVaultEvents - - // Check that no flow was triggered twice. - val duplicates = allCashStates.groupBy { it.state.data.amount }.filterValues { it.size > 1 } - assertTrue(duplicates.isEmpty(), "${duplicates.size} flows were retried illegally.") - - log.info("State machine events seen: ${stateMachineEvents!!.size}") - // State machine events are very likely to get lost more often because they seem to be sent with a delay. - assertThat(stateMachineEvents.count { it is StateMachineUpdate.Added }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3) - assertThat(stateMachineEvents.count { it is StateMachineUpdate.Removed }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3) - - // Stop the observers. - vaultSubscription.unsubscribe() - stateMachineSubscription.unsubscribe() - bankAReconnectingRPCConnection.close() - } - - proxy.close() - } - - @Synchronized - fun MutableMap>.addEvent(id: StateMachineRunId, progress: String?): Boolean { - return getOrPut(id) { mutableListOf() }.let { if (progress != null) it.add(progress) else false } - } - private fun getRandomAddressPair() = AddressPair(getRandomAddress(), getRandomAddress()) - private fun getRandomAddress() = NetworkHostAndPort("localhost", portAllocator.nextPort()) - - data class AddressPair(val proxyAddress: NetworkHostAndPort, val nodeAddress: NetworkHostAndPort) -} From e6d5842a237930b0a400c3b46bbbb66a6077bc48 Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Fri, 17 Jul 2020 13:57:40 +0100 Subject: [PATCH 15/48] INFRA-482: Correct exception caught testing for node death (#6471) --- .../node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt index d6abe718f1..7cc9b2643e 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt @@ -1,6 +1,7 @@ package net.corda.node.flows import co.paralleluniverse.fibers.Suspendable +import net.corda.core.CordaException import net.corda.core.flows.* import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party @@ -16,7 +17,6 @@ import net.corda.testing.driver.DriverDSL import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.NodeParameters import net.corda.testing.driver.driver -import net.corda.testing.node.internal.ListenProcessDeathException import net.corda.testing.node.internal.assertUncompletedCheckpoints import net.corda.testing.node.internal.enclosedCordapp import org.assertj.core.api.Assertions.assertThat @@ -77,7 +77,7 @@ class FlowCheckpointVersionNodeStartupCheckTest { private fun DriverDSL.assertBobFailsToStartWithLogMessage(logMessage: String) { assertUncompletedCheckpoints(BOB_NAME, 1) - assertFailsWith(ListenProcessDeathException::class) { + assertFailsWith(CordaException::class) { startNode(NodeParameters( providedName = BOB_NAME, customOverrides = mapOf("devMode" to false) From a500084d385fd43eb78dd00c4640b48d486fca6e Mon Sep 17 00:00:00 2001 From: Oliver Knowles Date: Mon, 20 Jul 2020 10:59:08 +0100 Subject: [PATCH 16/48] CORDA-3201 - Enforce separate key for notary identity (#6308) --- detekt-baseline.xml | 2 +- .../net/corda/node/internal/AbstractNode.kt | 23 ++- .../node/services/config/NodeConfiguration.kt | 2 +- .../registration/NetworkRegistrationHelper.kt | 153 ++++++++++++------ .../NetworkRegistrationHelperTest.kt | 116 +++++++++++-- samples/attachment-demo/build.gradle | 6 +- samples/bank-of-corda-demo/build.gradle | 6 +- samples/cordapp-configuration/build.gradle | 6 +- samples/irs-demo/cordapp/build.gradle | 10 +- samples/network-verifier/build.gradle | 6 +- samples/notary-demo/build.gradle | 11 +- samples/simm-valuation-demo/build.gradle | 6 +- samples/trader-demo/build.gradle | 6 +- 13 files changed, 263 insertions(+), 90 deletions(-) diff --git a/detekt-baseline.xml b/detekt-baseline.xml index 8e72535cd4..974e679f57 100644 --- a/detekt-baseline.xml +++ b/detekt-baseline.xml @@ -1435,7 +1435,7 @@ ThrowsCount:JarScanningCordappLoader.kt$JarScanningCordappLoader$private fun parseVersion(versionStr: String?, attributeName: String): Int ThrowsCount:LedgerDSLInterpreter.kt$Verifies$ fun failsWith(expectedMessage: String?): EnforceVerifyOrFail ThrowsCount:MockServices.kt$ fun <T : SerializeAsToken> createMockCordaService(serviceHub: MockServices, serviceConstructor: (AppServiceHub) -> T): T - ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates(registeringPublicKey: PublicKey, certificates: List<X509Certificate>) + ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates( registeringPublicKey: PublicKey, registeringLegalName: CordaX500Name, expectedCertRole: CertRole, certificates: List<X509Certificate> ) ThrowsCount:NodeInfoFilesCopier.kt$NodeInfoFilesCopier$private fun atomicCopy(source: Path, destination: Path) ThrowsCount:NodeVaultService.kt$NodeVaultService$@Throws(VaultQueryException::class) private fun <T : ContractState> _queryBy(criteria: QueryCriteria, paging_: PageSpecification, sorting: Sort, contractStateType: Class<out T>, skipPagingChecks: Boolean): Vault.Page<T> ThrowsCount:NodeVaultService.kt$NodeVaultService$private fun makeUpdates(batch: Iterable<CoreTransaction>, statesToRecord: StatesToRecord, previouslySeen: Boolean): List<Vault.Update<ContractState>> diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index c115fd45a4..59b6b4fca7 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -614,11 +614,22 @@ abstract class AbstractNode(val configuration: NodeConfiguration, val myNotaryIdentity = configuration.notary?.let { if (it.serviceLegalName != null) { - val (notaryIdentity, notaryIdentityKeyPair) = loadNotaryClusterIdentity(it.serviceLegalName) + val (notaryIdentity, notaryIdentityKeyPair) = loadNotaryServiceIdentity(it.serviceLegalName) keyPairs += notaryIdentityKeyPair notaryIdentity } else { - // In case of a single notary service myNotaryIdentity will be the node's single identity. + // The only case where the myNotaryIdentity will be the node's legal identity is for existing single notary services running + // an older version. Current single notary services (V4.6+) sign requests using a separate notary service identity so the + // notary identity will be different from the node's legal identity. + + // This check is here to ensure that a user does not accidentally/intentionally remove the serviceLegalName configuration + // parameter after a notary has been registered. If that was possible then notary would start and sign incoming requests + // with the node's legal identity key, corrupting the data. + check (!cryptoService.containsKey(DISTRIBUTED_NOTARY_KEY_ALIAS)) { + "The notary service key exists in the key store but no notary service legal name has been configured. " + + "Either include the relevant 'notary.serviceLegalName' configuration or validate this key is not necessary " + + "and remove from the key store." + } identity } } @@ -1057,8 +1068,12 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } } - /** Loads pre-generated notary service cluster identity. */ - private fun loadNotaryClusterIdentity(serviceLegalName: CordaX500Name): Pair { + /** + * Loads notary service identity. In the case of the experimental RAFT and BFT notary clusters, this loads the pre-generated + * cluster identity that all worker nodes share. In the case of a simple single notary, this loads the notary service identity + * that is generated during initial registration and is used to sign notarisation requests. + * */ + private fun loadNotaryServiceIdentity(serviceLegalName: CordaX500Name): Pair { val privateKeyAlias = "$DISTRIBUTED_NOTARY_KEY_ALIAS" val compositeKeyAlias = "$DISTRIBUTED_NOTARY_COMPOSITE_KEY_ALIAS" diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt index f2dc3f16cb..a12989e169 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt @@ -151,7 +151,7 @@ fun NodeConfiguration.shouldInitCrashShell() = shouldStartLocalShell() || should data class NotaryConfig( /** Specifies whether the notary validates transactions or not. */ val validating: Boolean, - /** The legal name of cluster in case of a distributed notary service. */ + /** The legal name of the notary service identity. */ val serviceLegalName: CordaX500Name? = null, /** The name of the notary service class to load. */ val className: String? = null, diff --git a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt index 45aa089f9e..8d2558ca8e 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt @@ -25,10 +25,10 @@ import org.bouncycastle.operator.ContentSigner import org.bouncycastle.util.io.pem.PemObject import java.io.IOException import java.io.StringWriter +import java.lang.IllegalStateException import java.net.ConnectException import java.net.URL import java.nio.file.Path -import java.security.KeyPair import java.security.PublicKey import java.security.cert.X509Certificate import java.time.Duration @@ -63,6 +63,7 @@ open class NetworkRegistrationHelper( private val requestIdStore = certificatesDirectory / "certificate-request-id.txt" protected val rootTrustStore: X509KeyStore protected val rootCert: X509Certificate + private val notaryServiceConfig: NotaryServiceConfig? = config.notaryServiceConfig init { require(networkRootTrustStorePath.exists()) { @@ -95,34 +96,70 @@ open class NetworkRegistrationHelper( return } + notaryServiceConfig?.let { validateNotaryServiceKeyAndCert(certStore, it.notaryServiceKeyAlias, it.notaryServiceLegalName) } + val tlsCrlIssuerCert = getTlsCrlIssuerCert() // We use SELF_SIGNED_PRIVATE_KEY as progress indicator so we just store a dummy key and cert. // When registration succeeds, this entry should be deleted. certStore.query { setPrivateKey(SELF_SIGNED_PRIVATE_KEY, AliasPrivateKey(SELF_SIGNED_PRIVATE_KEY), listOf(NOT_YET_REGISTERED_MARKER_KEYS_AND_CERTS.ECDSAR1_CERT), certificateStore.entryPassword) } - val nodeCaPublicKey = loadOrGenerateKeyPair() + val (entityPublicKey, receivedCertificates) = generateKeyPairAndCertificate(nodeCaKeyAlias, myLegalName, certRole, certStore) - val requestId = submitOrResumeCertificateSigningRequest(nodeCaPublicKey, cryptoService.getSigner(nodeCaKeyAlias)) - - val nodeCaCertificates = pollServerForCertificates(requestId) - validateCertificates(nodeCaPublicKey, nodeCaCertificates) - - certStore.setCertPathOnly(nodeCaKeyAlias, nodeCaCertificates) - certStore.value.internal.deleteEntry(SELF_SIGNED_PRIVATE_KEY) - certStore.value.save() - logProgress("Private key '$nodeCaKeyAlias' and its certificate-chain stored successfully.") - - onSuccess(nodeCaPublicKey, cryptoService.getSigner(nodeCaKeyAlias), nodeCaCertificates, tlsCrlIssuerCert?.subjectX500Principal?.toX500Name()) + onSuccess(entityPublicKey, cryptoService.getSigner(nodeCaKeyAlias), receivedCertificates, tlsCrlIssuerCert?.subjectX500Principal?.toX500Name()) // All done, clean up temp files. requestIdStore.deleteIfExists() } - private fun loadOrGenerateKeyPair(): PublicKey { - return if (cryptoService.containsKey(nodeCaKeyAlias)) { - cryptoService.getPublicKey(nodeCaKeyAlias)!! + private fun generateKeyPairAndCertificate(keyAlias: String, legalName: CordaX500Name, certificateRole: CertRole, certStore: CertificateStore): Pair> { + val entityPublicKey = loadOrGenerateKeyPair(keyAlias) + + val requestId = submitOrResumeCertificateSigningRequest(entityPublicKey, legalName, certificateRole, cryptoService.getSigner(keyAlias)) + + val receivedCertificates = pollServerForCertificates(requestId) + validateCertificates(entityPublicKey, legalName, certificateRole, receivedCertificates) + + certStore.setCertPathOnly(keyAlias, receivedCertificates) + certStore.value.internal.deleteEntry(SELF_SIGNED_PRIVATE_KEY) + certStore.value.save() + logProgress("Private key '$keyAlias' and its certificate-chain stored successfully.") + return Pair(entityPublicKey, receivedCertificates) + } + + /** + * Used when registering a notary to validate that the shared notary service key and certificate can be accessed. + * + * In the case that the notary service certificate and key is not available, a new key key is generated and a separate CSR is + * submitted to the Identity Manager. + * + * If this method successfully completes then the [cryptoService] will contain the notary service key and the [certStore] will contain + * the notary service certificate chain. + * + * @throws IllegalStateException If the notary service certificate already exists but the private key is not available. + */ + private fun validateNotaryServiceKeyAndCert(certStore: CertificateStore, notaryServiceKeyAlias: String, notaryServiceLegalName: CordaX500Name) { + if (certStore.contains(notaryServiceKeyAlias) && !cryptoService.containsKey(notaryServiceKeyAlias)) { + throw IllegalStateException("Notary service identity certificate exists but key pair missing. " + + "Please check no old certificates exist in the certificate store.") + } + + if (certStore.contains(notaryServiceKeyAlias)) { + logProgress("Notary service certificate already exists. Continuing with node registration...") + return + } + + logProgress("Generating notary service identity for $notaryServiceLegalName...") + generateKeyPairAndCertificate(notaryServiceKeyAlias, notaryServiceLegalName, CertRole.SERVICE_IDENTITY, certStore) + // The request id store is reused for the next step - registering the node identity. + // Therefore we can remove this to enable it to be reused. + requestIdStore.deleteIfExists() + } + + private fun loadOrGenerateKeyPair(keyAlias: String): PublicKey { + return if (cryptoService.containsKey(keyAlias)) { + cryptoService.getPublicKey(keyAlias)!! } else { - cryptoService.generateKeyPair(nodeCaKeyAlias, cryptoService.defaultTLSSignatureScheme()) + cryptoService.generateKeyPair(keyAlias, cryptoService.defaultTLSSignatureScheme()) } } @@ -137,26 +174,31 @@ open class NetworkRegistrationHelper( return tlsCrlIssuerCert } - private fun validateCertificates(registeringPublicKey: PublicKey, certificates: List) { - val nodeCACertificate = certificates.first() + private fun validateCertificates( + registeringPublicKey: PublicKey, + registeringLegalName: CordaX500Name, + expectedCertRole: CertRole, + certificates: List + ) { + val receivedCertificate = certificates.first() - val nodeCaSubject = try { - CordaX500Name.build(nodeCACertificate.subjectX500Principal) + val certificateSubject = try { + CordaX500Name.build(receivedCertificate.subjectX500Principal) } catch (e: IllegalArgumentException) { - throw CertificateRequestException("Received node CA cert has invalid subject name: ${e.message}") + throw CertificateRequestException("Received cert has invalid subject name: ${e.message}") } - if (nodeCaSubject != myLegalName) { - throw CertificateRequestException("Subject of received node CA cert doesn't match with node legal name: $nodeCaSubject") + if (certificateSubject != registeringLegalName) { + throw CertificateRequestException("Subject of received cert doesn't match with legal name: $certificateSubject") } - val nodeCaCertRole = try { - CertRole.extract(nodeCACertificate) + val receivedCertRole = try { + CertRole.extract(receivedCertificate) } catch (e: IllegalArgumentException) { - throw CertificateRequestException("Unable to extract cert role from received node CA cert: ${e.message}") + throw CertificateRequestException("Unable to extract cert role from received cert: ${e.message}") } - if (certRole != nodeCaCertRole) { - throw CertificateRequestException("Received certificate contains invalid cert role, expected '$certRole', got '$nodeCaCertRole'.") + if (expectedCertRole != receivedCertRole) { + throw CertificateRequestException("Received certificate contains invalid cert role, expected '$expectedCertRole', got '$receivedCertRole'.") } // Validate returned certificate is for the correct public key. @@ -169,22 +211,6 @@ open class NetworkRegistrationHelper( logProgress("Certificate signing request approved, storing private key with the certificate chain.") } - private fun CertificateStore.loadOrCreateKeyPair(alias: String, entryPassword: String = password): KeyPair { - // Create or load self signed keypair from the key store. - // We use the self sign certificate to store the key temporarily in the keystore while waiting for the request approval. - if (alias !in this) { - // NODE_CA should be TLS compatible due to the cert hierarchy structure. - val keyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) - val selfSignCert = X509Utilities.createSelfSignedCACertificate(myLegalName.x500Principal, keyPair) - // Save to the key store. - with(value) { - setPrivateKey(alias, keyPair.private, listOf(selfSignCert), keyPassword = entryPassword) - save() - } - } - return query { getCertificateAndKeyPair(alias, entryPassword) }.keyPair - } - /** * Poll Certificate Signing Server for approved certificate, * enter a slow polling loop if server return null. @@ -226,20 +252,27 @@ open class NetworkRegistrationHelper( * Submit Certificate Signing Request to Certificate signing service if request ID not found in file system. * New request ID will be stored in requestId.txt * @param publicKey public key for which we need a certificate. + * @param legalName legal name of the entity for which we need a certificate. + * @param certRole desired role of the entities certificate. * @param contentSigner the [ContentSigner] that will sign the CSR. * @return Request ID return from the server. */ - private fun submitOrResumeCertificateSigningRequest(publicKey: PublicKey, contentSigner: ContentSigner): String { + private fun submitOrResumeCertificateSigningRequest( + publicKey: PublicKey, + legalName: CordaX500Name, + certRole: CertRole, + contentSigner: ContentSigner + ): String { try { // Retrieve request id from file if exists, else post a request to server. return if (!requestIdStore.exists()) { - val request = X509Utilities.createCertificateSigningRequest(myLegalName.x500Principal, emailAddress, publicKey, contentSigner, certRole) + val request = X509Utilities.createCertificateSigningRequest(legalName.x500Principal, emailAddress, publicKey, contentSigner, certRole) val writer = StringWriter() JcaPEMWriter(writer).use { it.writeObject(PemObject("CERTIFICATE REQUEST", request.encoded)) } logProgress("Certificate signing request with the following information will be submitted to the Corda certificate signing server.") - logProgress("Legal Name: $myLegalName") + logProgress("Legal Name: $legalName") logProgress("Email: $emailAddress") logProgress("Public Key: $publicKey") logProgress("$writer") @@ -277,7 +310,8 @@ class NodeRegistrationConfiguration( val certificatesDirectory: Path, val emailAddress: String, val cryptoService: CryptoService, - val certificateStore: CertificateStore) { + val certificateStore: CertificateStore, + val notaryServiceConfig: NotaryServiceConfig? = null) { constructor(config: NodeConfiguration) : this( p2pSslOptions = config.p2pSslOptions, @@ -287,10 +321,29 @@ class NodeRegistrationConfiguration( certificatesDirectory = config.certificatesDirectory, emailAddress = config.emailAddress, cryptoService = BCCryptoService(config.myLegalName.x500Principal, config.signingCertificateStore), - certificateStore = config.signingCertificateStore.get(true) + certificateStore = config.signingCertificateStore.get(true), + notaryServiceConfig = config.notary?.let { + // Validation of the presence of the notary service legal name is only done here and not in the top level configuration + // file. This is to maintain backwards compatibility with older notaries using the legacy identity structure. Older + // notaries will be signing requests using the nodes legal identity key and therefore no separate notary service entity + // exists. Just having the validation here prevents any new notaries from being created with the legacy identity scheme + // but still allows drop in JAR replacements for old notaries. + requireNotNull(it.serviceLegalName) { + "The notary service legal name must be provided via the 'notary.serviceLegalName' configuration parameter" + } + require(it.serviceLegalName != config.myLegalName) { + "The notary service legal name must be different from the node legal name" + } + NotaryServiceConfig(X509Utilities.DISTRIBUTED_NOTARY_KEY_ALIAS, it.serviceLegalName!!) + } ) } +data class NotaryServiceConfig( + val notaryServiceKeyAlias: String, + val notaryServiceLegalName: CordaX500Name +) + class NodeRegistrationException( message: String?, cause: Throwable? diff --git a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt index dde5082f6b..b17b437fad 100644 --- a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt +++ b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt @@ -28,6 +28,8 @@ import net.corda.testing.core.ALICE_NAME import net.corda.testing.internal.createDevIntermediateCaCertPath import net.corda.coretesting.internal.rigorousMock import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.node.services.config.NotaryConfig +import net.corda.testing.core.DUMMY_NOTARY_NAME import org.assertj.core.api.Assertions.* import org.bouncycastle.asn1.x509.GeneralName import org.bouncycastle.asn1.x509.GeneralSubtree @@ -37,6 +39,7 @@ import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest import org.junit.After import org.junit.Before import org.junit.Test +import java.lang.IllegalStateException import java.nio.file.Files import java.security.PublicKey import java.security.cert.CertPathValidatorException @@ -71,6 +74,7 @@ class NetworkRegistrationHelperTest { doReturn(null).whenever(it).tlsCertCrlDistPoint doReturn(null).whenever(it).tlsCertCrlIssuer doReturn(true).whenever(it).crlCheckSoftFail + doReturn(null).whenever(it).notary } } @@ -120,7 +124,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `missing truststore`() { - val nodeCaCertPath = createNodeCaCertPath() + val nodeCaCertPath = createCertPath() assertThatThrownBy { createFixedResponseRegistrationHelper(nodeCaCertPath) }.hasMessageContaining("This file must contain the root CA cert of your compatibility zone. Please contact your CZ operator.") @@ -128,7 +132,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `node CA with incorrect cert role`() { - val nodeCaCertPath = createNodeCaCertPath(type = CertificateType.TLS) + val nodeCaCertPath = createCertPath(type = CertificateType.TLS) saveNetworkTrustStore(CORDA_ROOT_CA to nodeCaCertPath.last()) val registrationHelper = createFixedResponseRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) @@ -139,7 +143,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `node CA with incorrect subject`() { val invalidName = CordaX500Name("Foo", "MU", "GB") - val nodeCaCertPath = createNodeCaCertPath(legalName = invalidName) + val nodeCaCertPath = createCertPath(legalName = invalidName) saveNetworkTrustStore(CORDA_ROOT_CA to nodeCaCertPath.last()) val registrationHelper = createFixedResponseRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) @@ -220,36 +224,118 @@ class NetworkRegistrationHelperTest { createRegistrationHelper(rootAndIntermediateCA = rootAndIntermediateCA).generateKeysAndRegister() } - private fun createNodeCaCertPath(type: CertificateType = CertificateType.NODE_CA, - legalName: CordaX500Name = nodeLegalName, - publicKey: PublicKey = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME).public, - rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()): List { + @Test(timeout=300_000) + fun `successful registration for notary node`() { + val notaryServiceLegalName = DUMMY_NOTARY_NAME + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = notaryServiceLegalName) + assertThat(notaryNodeConfig.notary).isNotNull + + val rootAndIntermediateCA = createDevIntermediateCaCertPath().also { + saveNetworkTrustStore(CORDA_ROOT_CA to it.first.certificate) + } + + // Mock out the registration service to ensure notary service registration is handled correctly + createRegistrationHelper(CertRole.NODE_CA, notaryNodeConfig) { + when { + it.subject == nodeLegalName.toX500Name() -> { + val certType = CertificateType.values().first { it.role == CertRole.NODE_CA } + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) + } + it.subject == notaryServiceLegalName.toX500Name() -> { + val certType = CertificateType.values().first { it.role == CertRole.SERVICE_IDENTITY } + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType, legalName = notaryServiceLegalName) + } + else -> throw IllegalStateException("Unknown CSR") + } + }.generateKeysAndRegister() + + val nodeKeystore = config.signingCertificateStore.get() + + nodeKeystore.run { + assertFalse(contains(X509Utilities.CORDA_INTERMEDIATE_CA)) + assertFalse(contains(CORDA_ROOT_CA)) + assertFalse(contains(X509Utilities.CORDA_CLIENT_TLS)) + assertThat(CertRole.extract(this[X509Utilities.CORDA_CLIENT_CA])).isEqualTo(CertRole.NODE_CA) + assertThat(CertRole.extract(this[DISTRIBUTED_NOTARY_KEY_ALIAS])).isEqualTo(CertRole.SERVICE_IDENTITY) + } + } + + @Test(timeout=300_000) + fun `notary registration fails when no separate notary service identity configured`() { + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = null) + assertThat(notaryNodeConfig.notary).isNotNull + + assertThatThrownBy { + createRegistrationHelper(nodeConfig = notaryNodeConfig) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("notary service legal name must be provided") + } + + @Test(timeout=300_000) + fun `notary registration fails when notary service identity configured with same legal name as node`() { + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = config.myLegalName) + assertThat(notaryNodeConfig.notary).isNotNull + + assertThatThrownBy { + createRegistrationHelper(nodeConfig = notaryNodeConfig) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("notary service legal name must be different from the node") + } + + private fun createNotaryNodeConfiguration(notaryServiceLegalName: CordaX500Name?): NodeConfiguration { + return rigorousMock().also { + doReturn(config.baseDirectory).whenever(it).baseDirectory + doReturn(config.certificatesDirectory).whenever(it).certificatesDirectory + doReturn(CertificateStoreStubs.P2P.withCertificatesDirectory(config.certificatesDirectory)).whenever(it).p2pSslOptions + doReturn(CertificateStoreStubs.Signing.withCertificatesDirectory(config.certificatesDirectory)).whenever(it) + .signingCertificateStore + doReturn(nodeLegalName).whenever(it).myLegalName + doReturn("").whenever(it).emailAddress + doReturn(null).whenever(it).tlsCertCrlDistPoint + doReturn(null).whenever(it).tlsCertCrlIssuer + doReturn(true).whenever(it).crlCheckSoftFail + doReturn(NotaryConfig(validating = false, serviceLegalName = notaryServiceLegalName)).whenever(it).notary + } + } + + private fun createCertPath(type: CertificateType = CertificateType.NODE_CA, + legalName: CordaX500Name = nodeLegalName, + publicKey: PublicKey = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME).public, + rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()): List { val (rootCa, intermediateCa) = rootAndIntermediateCA val nameConstraints = if (type == CertificateType.NODE_CA) { NameConstraints(arrayOf(GeneralSubtree(GeneralName(GeneralName.directoryName, legalName.toX500Name()))), arrayOf()) } else { null } - val nodeCaCert = X509Utilities.createCertificate( + val cert = X509Utilities.createCertificate( type, intermediateCa.certificate, intermediateCa.keyPair, legalName.x500Principal, publicKey, nameConstraints = nameConstraints) - return listOf(nodeCaCert, intermediateCa.certificate, rootCa.certificate) + return listOf(cert, intermediateCa.certificate, rootCa.certificate) } private fun createFixedResponseRegistrationHelper(response: List, certRole: CertRole = CertRole.NODE_CA): NetworkRegistrationHelper { return createRegistrationHelper(certRole) { response } } - private fun createRegistrationHelper(certRole: CertRole = CertRole.NODE_CA, rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()) = createRegistrationHelper(certRole) { + private fun createRegistrationHelper( + certRole: CertRole = CertRole.NODE_CA, + rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath(), + nodeConfig: NodeConfiguration = config + ) = createRegistrationHelper(certRole, nodeConfig) { val certType = CertificateType.values().first { it.role == certRole } - createNodeCaCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) } - private fun createRegistrationHelper(certRole: CertRole = CertRole.NODE_CA, dynamicResponse: (JcaPKCS10CertificationRequest) -> List): NetworkRegistrationHelper { + private fun createRegistrationHelper( + certRole: CertRole = CertRole.NODE_CA, + nodeConfig: NodeConfiguration = config, + dynamicResponse: (JcaPKCS10CertificationRequest) -> List + ): NetworkRegistrationHelper { val certService = rigorousMock().also { val requests = mutableMapOf() doAnswer { @@ -265,11 +351,11 @@ class NetworkRegistrationHelperTest { } return when (certRole) { - CertRole.NODE_CA -> NodeRegistrationHelper(NodeRegistrationConfiguration(config), certService, NodeRegistrationOption(config.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword)) + CertRole.NODE_CA -> NodeRegistrationHelper(NodeRegistrationConfiguration(nodeConfig), certService, NodeRegistrationOption(nodeConfig.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword)) CertRole.SERVICE_IDENTITY -> NetworkRegistrationHelper( - NodeRegistrationConfiguration(config), + NodeRegistrationConfiguration(nodeConfig), certService, - config.certificatesDirectory / networkRootTrustStoreFileName, + nodeConfig.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword, DISTRIBUTED_NOTARY_KEY_ALIAS, CertRole.SERVICE_IDENTITY) diff --git a/samples/attachment-demo/build.gradle b/samples/attachment-demo/build.gradle index e88c8fc431..43bb2c8cbd 100644 --- a/samples/attachment-demo/build.gradle +++ b/samples/attachment-demo/build.gradle @@ -92,8 +92,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, cordapp project(':samples:attachment-demo:workflows') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating: true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 cordapps = [] rpcUsers = ext.rpcUsers diff --git a/samples/bank-of-corda-demo/build.gradle b/samples/bank-of-corda-demo/build.gradle index e3ff1ad5c3..7749ba3cdd 100644 --- a/samples/bank-of-corda-demo/build.gradle +++ b/samples/bank-of-corda-demo/build.gradle @@ -50,8 +50,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, cordapp project(':finance:contracts') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating: true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10003" diff --git a/samples/cordapp-configuration/build.gradle b/samples/cordapp-configuration/build.gradle index 9d466ed986..723de77b5b 100644 --- a/samples/cordapp-configuration/build.gradle +++ b/samples/cordapp-configuration/build.gradle @@ -27,8 +27,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, cordapp project(':samples:cordapp-configuration:workflows') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { port 10003 diff --git a/samples/irs-demo/cordapp/build.gradle b/samples/irs-demo/cordapp/build.gradle index 50474dd3e5..2fa1ca9f49 100644 --- a/samples/irs-demo/cordapp/build.gradle +++ b/samples/irs-demo/cordapp/build.gradle @@ -62,8 +62,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) cordapp project(':samples:irs-demo:cordapp:workflows-irs') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address("localhost:10003") @@ -121,7 +123,9 @@ task prepareDockerNodes(type: net.corda.plugins.Dockerform, dependsOn: ['jar', n } node { name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] cordapps = ["${project(":finance").group}:contracts:$corda_release_version", "${project(":finance").group}:workflows:$corda_release_version"] rpcUsers = rpcUsersList useTestClock true diff --git a/samples/network-verifier/build.gradle b/samples/network-verifier/build.gradle index f7582c0069..92a2006e81 100644 --- a/samples/network-verifier/build.gradle +++ b/samples/network-verifier/build.gradle @@ -38,8 +38,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) cordapp project(':samples:network-verifier:workflows') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : false] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : false, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { port 10003 diff --git a/samples/notary-demo/build.gradle b/samples/notary-demo/build.gradle index a5a7a40117..ff2683b2a2 100644 --- a/samples/notary-demo/build.gradle +++ b/samples/notary-demo/build.gradle @@ -55,13 +55,15 @@ task deployNodesSingle(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { rpcUsers = [[user: "demou", password: "demop", permissions: ["ALL"]]] } node { - name "O=Notary Service,L=Zurich,C=CH" + name "O=Notary Node,L=Zurich,C=CH" p2pPort 10009 rpcSettings { address "localhost:10010" adminAddress "localhost:10110" } - notary = [validating: true] + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] } } @@ -85,7 +87,7 @@ task deployNodesCustom(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { rpcUsers = [[user: "demou", password: "demop", permissions: ["ALL"]]] } node { - name "O=Notary Service,L=Zurich,C=CH" + name "O=Notary Node,L=Zurich,C=CH" p2pPort 10009 rpcSettings { address "localhost:10010" @@ -93,7 +95,8 @@ task deployNodesCustom(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { } notary = [ validating: true, - className: "net.corda.notarydemo.MyCustomValidatingNotaryService" + className: "net.corda.notarydemo.MyCustomValidatingNotaryService", + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" ] } } diff --git a/samples/simm-valuation-demo/build.gradle b/samples/simm-valuation-demo/build.gradle index f95a10716b..b0af7c3568 100644 --- a/samples/simm-valuation-demo/build.gradle +++ b/samples/simm-valuation-demo/build.gradle @@ -93,8 +93,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, rpcUsers = [['username': "default", 'password': "default", 'permissions': [ 'ALL' ]]] } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10014" diff --git a/samples/trader-demo/build.gradle b/samples/trader-demo/build.gradle index 5eeea06740..5ac022fb30 100644 --- a/samples/trader-demo/build.gradle +++ b/samples/trader-demo/build.gradle @@ -83,8 +83,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) cordapp project(':samples:trader-demo:workflows-trader') } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10003" From e5a8351dc3564710ebf65da12ef730ac9a7ff4a5 Mon Sep 17 00:00:00 2001 From: Dimos Raptis Date: Mon, 20 Jul 2020 11:13:49 +0100 Subject: [PATCH 17/48] ENT-5437 - Add test for sendAll with multiple sessions from the same party (#6476) --- .../messaging/MessagingSendAllTest.kt | 72 +++++++++++++++++++ .../services/messaging/MessagingExecutor.kt | 4 +- 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt diff --git a/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt new file mode 100644 index 0000000000..ad417530d5 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt @@ -0,0 +1,72 @@ +package net.corda.node.services.messaging + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.Destination +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.internal.concurrent.transpose +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.unwrap +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import org.junit.Test +import kotlin.test.assertEquals + +class MessagingSendAllTest { + + @Test(timeout=300_000) + fun `flow can exchange messages with multiple sessions to the same party in parallel`() { + driver(DriverParameters(startNodesInProcess = true)) { + val (alice, bob) = listOf( + startNode(providedName = ALICE_NAME), + startNode(providedName = BOB_NAME) + ).transpose().getOrThrow() + + val bobIdentity = bob.nodeInfo.singleIdentity() + val messages = listOf( + bobIdentity to "hey bob 1", + bobIdentity to "hey bob 2" + ) + + alice.rpc.startFlow(::SenderFlow, messages).returnValue.getOrThrow() + } + } + + @StartableByRPC + @InitiatingFlow + class SenderFlow(private val parties: List>): FlowLogic() { + @Suspendable + override fun call(): String { + val messagesPerSession = parties.toList().map { (party, messageType) -> + val session = initiateFlow(party) + Pair(session, messageType) + }.toMap() + + sendAllMap(messagesPerSession) + val messages = receiveAll(String::class.java, messagesPerSession.keys.toList()) + + messages.map { it.unwrap { payload -> assertEquals("pong", payload) } } + + return "ok" + } + } + + @InitiatedBy(SenderFlow::class) + class RecipientFlow(private val otherPartySession: FlowSession): FlowLogic() { + @Suspendable + override fun call(): String { + otherPartySession.receive().unwrap { it } + otherPartySession.send("pong") + + return "ok" + } + } + +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt b/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt index eead9f5698..0734c958e1 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt @@ -54,8 +54,8 @@ class MessagingExecutor( } @Synchronized - fun sendAll(messages: Map) { - messages.forEach { recipients, message -> send(message, recipients) } + fun sendAll(messages: List>) { + messages.forEach { (recipients, message) -> send(message, recipients) } } @Synchronized From 3721b7c701b633d5884c559a240bc1748e5cf20b Mon Sep 17 00:00:00 2001 From: jakubbielawa Date: Mon, 20 Jul 2020 11:14:23 +0100 Subject: [PATCH 18/48] Decrease the ring buffer size (#6470) --- node/src/main/resources/log4j2.component.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/main/resources/log4j2.component.properties b/node/src/main/resources/log4j2.component.properties index 1b55982139..405c40b154 100644 --- a/node/src/main/resources/log4j2.component.properties +++ b/node/src/main/resources/log4j2.component.properties @@ -1,2 +1,2 @@ Log4jContextSelector=net.corda.node.utilities.logging.AsyncLoggerContextSelectorNoThreadLocal -AsyncLogger.RingBufferSize=262144 \ No newline at end of file +AsyncLogger.RingBufferSize=16384 \ No newline at end of file From d4444e520a0bca20c6c815c90dced7ad700f8608 Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Mon, 20 Jul 2020 11:26:27 +0100 Subject: [PATCH 19/48] ENT-5140: Tighten network parameters checks (#6390) * ENT-5140: Tighten network parameters checks --- .../kotlin/net/corda/node/internal/NetworkParametersReader.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt b/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt index 0bab5cb88e..ce964ab97c 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt @@ -86,6 +86,7 @@ class NetworkParametersReader(private val trustRoot: X509Certificate, logger.info("No network-parameters file found. Expecting network parameters to be available from the network map.") networkMapClient ?: throw Error.NetworkMapNotConfigured() val signedParams = networkMapClient.getNetworkParameters(parametersHash) + signedParams.verifiedNetworkParametersCert(trustRoot) signedParams.serialize().open().copyTo(baseDirectory / NETWORK_PARAMS_FILE_NAME) return signedParams } From 5fef0726a2c54107a0cd6ea08753f571ae65039a Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Mon, 20 Jul 2020 15:09:26 +0100 Subject: [PATCH 20/48] Add missing 'Suspendable' annotations to suspending methods (#6480) --- .../net/corda/node/services/statemachine/ActionExecutorImpl.kt | 2 ++ .../corda/node/services/statemachine/ActionFutureExecutor.kt | 3 +++ 2 files changed, 5 insertions(+) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 2849dc03a1..435ae5d6f3 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -73,6 +73,7 @@ internal class ActionExecutorImpl( if (action.uuid != null) services.vaultService.softLockRelease(action.uuid) } + @Suspendable private fun executeTrackTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { actionFutureExecutor.awaitTransaction(fiber, action) } @@ -223,6 +224,7 @@ internal class ActionExecutorImpl( } @Suppress("TooGenericExceptionCaught") + @Suspendable private fun executeAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { try { actionFutureExecutor.awaitAsyncOperation(fiber, action) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt index dc5d2fc0b9..40ee343707 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt @@ -1,5 +1,6 @@ package net.corda.node.services.statemachine +import co.paralleluniverse.fibers.Suspendable import net.corda.core.internal.concurrent.thenMatch import net.corda.core.utilities.contextLogger import net.corda.core.utilities.debug @@ -45,6 +46,7 @@ internal class ActionFutureExecutor( * @param fiber The [FlowFiber] to resume after completing the async operation * @param action The [Action.ExecuteAsyncOperation] to create a future from */ + @Suspendable fun awaitAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { cancelFutureIfRunning(fiber, action.currentState) val instance = fiber.instanceId @@ -63,6 +65,7 @@ internal class ActionFutureExecutor( * @param fiber The [FlowFiber] to resume after the committing the specified transaction * @param action [Action.TrackTransaction] contains the transaction hash to wait for */ + @Suspendable fun awaitTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { cancelFutureIfRunning(fiber, action.currentState) val instance = fiber.instanceId From 1c48418904d20cd31d20766ef468022c01c387fd Mon Sep 17 00:00:00 2001 From: Viktor Kolomeyko Date: Tue, 21 Jul 2020 09:58:29 +0100 Subject: [PATCH 21/48] NOTICK: Remove `relaxedThoroughness` variable (#6483) It is: a. No longer works in Jenkins b. May result spawning 100s of process that kill Windows build. Evidence: https://ci02.dev.r3.com/blue/organizations/jenkins/Corda-Enterprise%2FCorda-ENT-MS-Win-Compatibility%2Fenterprise/detail/PR-3553/25/pipeline/ --- .../test/kotlin/net/corda/node/internal/AbstractNodeTests.kt | 4 +--- .../kotlin/net/corda/testing/common/internal/Thoroughness.kt | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt diff --git a/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt b/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt index e6655efd90..0fb0382234 100644 --- a/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt +++ b/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt @@ -6,7 +6,6 @@ import net.corda.core.internal.concurrent.transpose import net.corda.core.utilities.contextLogger import net.corda.core.utilities.getOrThrow import net.corda.nodeapi.internal.persistence.DatabaseConfig -import net.corda.testing.common.internal.relaxedThoroughness import net.corda.testing.internal.configureDatabase import net.corda.testing.node.internal.ProcessUtilities.startJavaProcess import org.junit.Rule @@ -43,8 +42,7 @@ class AbstractNodeTests { @Test(timeout=300_000) fun `H2 fix is applied`() { val pool = Executors.newFixedThreadPool(5) - val runs = if (relaxedThoroughness) 1 else 100 - (0 until runs).map { + (0 until 5).map { // Four "nodes" seems to be the magic number to reproduce the problem on CI: val urls = (0 until 4).map { freshURL() } // Haven't been able to reproduce in a warm JVM: diff --git a/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt b/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt deleted file mode 100644 index 298eaa5660..0000000000 --- a/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt +++ /dev/null @@ -1,3 +0,0 @@ -package net.corda.testing.common.internal - -val relaxedThoroughness = System.getenv("TEAMCITY_PROJECT_NAME") == "Pull Requests" From 7261fa690f703923e7bddfd569dd51478ceed4ab Mon Sep 17 00:00:00 2001 From: Dimos Raptis Date: Tue, 21 Jul 2020 13:26:11 +0100 Subject: [PATCH 22/48] CORDA-3506 - Implement session close operations (#6357) --- constants.properties | 2 +- .../flows/FlowExternalAsyncOperationTest.kt | 2 +- .../kotlin/net/corda/core/flows/FlowLogic.kt | 17 ++ .../net/corda/core/flows/FlowSession.kt | 13 + .../net/corda/core/internal/CordaUtils.kt | 2 +- .../net/corda/core/internal/FlowIORequest.kt | 7 + .../corda/node/flows/FlowSessionCloseTest.kt | 273 ++++++++++++++++++ .../node/services/rpc/CheckpointDumperImpl.kt | 12 +- .../statemachine/ActionExecutorImpl.kt | 6 +- .../node/services/statemachine/FlowMonitor.kt | 1 + .../services/statemachine/FlowSessionImpl.kt | 6 + .../statemachine/StateMachineState.kt | 52 ++-- .../DeliverSessionMessageTransition.kt | 72 +++-- .../transitions/ErrorFlowTransition.kt | 3 +- .../transitions/KilledFlowTransition.kt | 3 +- .../transitions/StartedFlowTransition.kt | 272 +++++++++++------ .../transitions/TopLevelTransition.kt | 5 +- .../transitions/TransitionBuilder.kt | 2 + .../transitions/UnstartedFlowTransition.kt | 7 +- .../statemachine/FlowFrameworkTests.kt | 3 +- .../statemachine/FlowMetadataRecordingTest.kt | 2 +- .../statemachine/RetryFlowMockTest.kt | 5 + .../node/internal/InternalMockNetwork.kt | 3 +- .../node/internal/MockNodeMessagingService.kt | 1 + 24 files changed, 598 insertions(+), 173 deletions(-) create mode 100644 node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt diff --git a/constants.properties b/constants.properties index c9877793fb..6cc3b7e4a2 100644 --- a/constants.properties +++ b/constants.properties @@ -11,7 +11,7 @@ java8MinUpdateVersion=171 # When incrementing platformVersion make sure to update # # net.corda.core.internal.CordaUtilsKt.PLATFORM_VERSION as well. # # ***************************************************************# -platformVersion=7 +platformVersion=8 guavaVersion=28.0-jre # Quasar version to use with Java 8: quasarVersion=0.7.12_r3 diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt index cbf1892e51..6b6cfb3891 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt @@ -167,7 +167,7 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Suspendable override fun testCode(): Any = - await(ExternalAsyncOperation(serviceHub) { _, _ -> + await(ExternalAsyncOperation(serviceHub) { serviceHub, _ -> serviceHub.cordaService(FutureService::class.java).createFuture() }) } diff --git a/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt b/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt index f6f502cf98..fd7b16fc5f 100644 --- a/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt +++ b/core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt @@ -25,6 +25,7 @@ import net.corda.core.node.NodeInfo import net.corda.core.node.ServiceHub import net.corda.core.serialization.CordaSerializable import net.corda.core.transactions.SignedTransaction +import net.corda.core.utilities.NonEmptySet import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.UntrustworthyData import net.corda.core.utilities.debug @@ -378,6 +379,22 @@ abstract class FlowLogic { stateMachine.suspend(request, maySkipCheckpoint) } + /** + * Closes the provided sessions and performs cleanup of any resources tied to these sessions. + * + * Note that sessions are closed automatically when the corresponding top-level flow terminates. + * So, it's beneficial to eagerly close them in long-lived flows that might have many open sessions that are not needed anymore and consume resources (e.g. memory, disk etc.). + * A closed session cannot be used anymore, e.g. to send or receive messages. So, you have to ensure you are calling this method only when the provided sessions are not going to be used anymore. + * As a result, any operations on a closed session will fail with an [UnexpectedFlowEndException]. + * When a session is closed, the other side is informed and the session is closed there too eventually. + * To prevent misuse of the API, if there is an attempt to close an uninitialised session the invocation will fail with an [IllegalStateException]. + */ + @Suspendable + fun close(sessions: NonEmptySet) { + val request = FlowIORequest.CloseSessions(sessions) + stateMachine.suspend(request, false) + } + /** * Invokes the given subflow. This function returns once the subflow completes successfully with the result * returned by that subflow's [call] method. If the subflow has a progress tracker, it is attached to the diff --git a/core/src/main/kotlin/net/corda/core/flows/FlowSession.kt b/core/src/main/kotlin/net/corda/core/flows/FlowSession.kt index ac16d6897d..dd09a9d481 100644 --- a/core/src/main/kotlin/net/corda/core/flows/FlowSession.kt +++ b/core/src/main/kotlin/net/corda/core/flows/FlowSession.kt @@ -191,6 +191,19 @@ abstract class FlowSession { */ @Suspendable abstract fun send(payload: Any) + + /** + * Closes this session and performs cleanup of any resources tied to this session. + * + * Note that sessions are closed automatically when the corresponding top-level flow terminates. + * So, it's beneficial to eagerly close them in long-lived flows that might have many open sessions that are not needed anymore and consume resources (e.g. memory, disk etc.). + * A closed session cannot be used anymore, e.g. to send or receive messages. So, you have to ensure you are calling this method only when the session is not going to be used anymore. + * As a result, any operations on a closed session will fail with an [UnexpectedFlowEndException]. + * When a session is closed, the other side is informed and the session is closed there too eventually. + * To prevent misuse of the API, if there is an attempt to close an uninitialised session the invocation will fail with an [IllegalStateException]. + */ + @Suspendable + abstract fun close() } /** diff --git a/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt b/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt index cc99f3a6a2..af2a20b40c 100644 --- a/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt @@ -28,7 +28,7 @@ import java.util.jar.JarInputStream // *Internal* Corda-specific utilities. -const val PLATFORM_VERSION = 7 +const val PLATFORM_VERSION = 8 fun ServicesForResolution.ensureMinimumPlatformVersion(requiredMinPlatformVersion: Int, feature: String) { checkMinimumPlatformVersion(networkParameters.minimumPlatformVersion, requiredMinPlatformVersion, feature) diff --git a/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt b/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt index 0d54a4715a..7ced0d46a0 100644 --- a/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt +++ b/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt @@ -55,6 +55,13 @@ sealed class FlowIORequest { }}, shouldRetrySend=$shouldRetrySend)" } + /** + * Closes the specified sessions. + * + * @property sessions the sessions to be closed. + */ + data class CloseSessions(val sessions: NonEmptySet): FlowIORequest() + /** * Wait for a transaction to be committed to the database. * diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt new file mode 100644 index 0000000000..a7e0cf877e --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt @@ -0,0 +1,273 @@ +package net.corda.node.flows + +import co.paralleluniverse.fibers.Suspendable +import net.corda.client.rpc.CordaRPCClient +import net.corda.core.CordaRuntimeException +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.flows.UnexpectedFlowEndException +import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose +import net.corda.core.messaging.startFlow +import net.corda.core.serialization.CordaSerializable +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.core.utilities.toNonEmptySet +import net.corda.core.utilities.unwrap +import net.corda.node.services.Permissions +import net.corda.node.services.statemachine.transitions.PrematureSessionCloseException +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.User +import net.corda.testing.node.internal.enclosedCordapp +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.Test +import java.sql.SQLTransientConnectionException +import kotlin.test.assertEquals + +class FlowSessionCloseTest { + + private val user = User("user", "pwd", setOf(Permissions.all())) + + @Test(timeout=300_000) + fun `flow cannot close uninitialised session`() { + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), true, null, false).returnValue.getOrThrow() } + .isInstanceOf(CordaRuntimeException::class.java) + .hasMessageContaining(PrematureSessionCloseException::class.java.name) + .hasMessageContaining("The following session was closed before it was initialised") + } + } + } + + @Test(timeout=300_000) + fun `flow cannot access closed session`() { + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) + ).transpose().getOrThrow() + + InitiatorFlow.SessionAPI.values().forEach { sessionAPI -> + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, sessionAPI, false).returnValue.getOrThrow() } + .isInstanceOf(UnexpectedFlowEndException::class.java) + .hasMessageContaining("Tried to access ended session") + } + } + + } + } + + @Test(timeout=300_000) + fun `flow can close initialised session successfully`() { + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, false).returnValue.getOrThrow() + } + } + } + + @Test(timeout=300_000) + fun `flow can close initialised session successfully even in case of failures and replays`() { + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, true).returnValue.getOrThrow() + } + } + } + + @Test(timeout=300_000) + fun `flow can close multiple sessions successfully`() { + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::InitiatorMultipleSessionsFlow, nodeBHandle.nodeInfo.legalIdentities.first()).returnValue.getOrThrow() + } + } + } + + /** + * This test ensures that when sessions are closed, the associated resources are eagerly cleaned up. + * If sessions are not closed, then the node will crash with an out-of-memory error. + * This can be confirmed by commenting out [FlowSession.close] operation in the invoked flow and re-run the test. + */ + @Test(timeout=300_000) + fun `flow looping over sessions can close them to release resources and avoid out-of-memory failures, when the other side does not finish early`() { + driver(DriverParameters(startNodesInProcess = false, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), maximumHeapSize = "256m"), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user), maximumHeapSize = "256m") + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::InitiatorLoopingFlow, nodeBHandle.nodeInfo.legalIdentities.first(), true).returnValue.getOrThrow() + } + } + } + + @Test(timeout=300_000) + fun `flow looping over sessions will close sessions automatically, when the other side finishes early`() { + driver(DriverParameters(startNodesInProcess = false, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { + val (nodeAHandle, nodeBHandle) = listOf( + startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), maximumHeapSize = "256m"), + startNode(providedName = BOB_NAME, rpcUsers = listOf(user), maximumHeapSize = "256m") + ).transpose().getOrThrow() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::InitiatorLoopingFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false).returnValue.getOrThrow() + } + } + } + + + + @InitiatingFlow + @StartableByRPC + class InitiatorFlow(val party: Party, private val prematureClose: Boolean = false, + private val accessClosedSessionWithApi: SessionAPI? = null, + private val retryClose: Boolean = false): FlowLogic() { + + @CordaSerializable + enum class SessionAPI { + SEND, + SEND_AND_RECEIVE, + RECEIVE, + GET_FLOW_INFO + } + + @Suspendable + override fun call() { + val session = initiateFlow(party) + + if (prematureClose) { + session.close() + } + + session.send(retryClose) + sleep(1.seconds) + + if (accessClosedSessionWithApi != null) { + when(accessClosedSessionWithApi) { + SessionAPI.SEND -> session.send("dummy payload ") + SessionAPI.RECEIVE -> session.receive() + SessionAPI.SEND_AND_RECEIVE -> session.sendAndReceive("dummy payload") + SessionAPI.GET_FLOW_INFO -> session.getCounterpartyFlowInfo() + } + } + } + } + + @InitiatedBy(InitiatorFlow::class) + class InitiatedFlow(private val otherSideSession: FlowSession): FlowLogic() { + + companion object { + var thrown = false + } + + @Suspendable + override fun call() { + val retryClose = otherSideSession.receive() + .unwrap{ it } + + otherSideSession.close() + + // failing with a transient exception to force a replay of the close. + if (retryClose) { + if (!thrown) { + thrown = true + throw SQLTransientConnectionException("Connection is not available") + } + } + } + } + + @InitiatingFlow + @StartableByRPC + class InitiatorLoopingFlow(val party: Party, val blockingCounterparty: Boolean = false): FlowLogic() { + @Suspendable + override fun call() { + for (i in 1..1_000) { + val session = initiateFlow(party) + session.sendAndReceive(blockingCounterparty ).unwrap{ assertEquals("Got it", it) } + + /** + * If the counterparty blocks, we need to eagerly close the session and release resources to avoid running out of memory. + * Otherwise, the session end messages from the other side will do that automatically. + */ + if (blockingCounterparty) { + session.close() + } + + logger.info("Completed iteration $i") + } + } + } + + @InitiatedBy(InitiatorLoopingFlow::class) + class InitiatedLoopingFlow(private val otherSideSession: FlowSession): FlowLogic() { + @Suspendable + override fun call() { + val shouldBlock = otherSideSession.receive() + .unwrap{ it } + otherSideSession.send("Got it") + + if (shouldBlock) { + otherSideSession.receive() + } + } + } + + @InitiatingFlow + @StartableByRPC + class InitiatorMultipleSessionsFlow(val party: Party): FlowLogic() { + @Suspendable + override fun call() { + for (round in 1 .. 2) { + val sessions = mutableListOf() + for (session_number in 1 .. 5) { + val session = initiateFlow(party) + sessions.add(session) + session.sendAndReceive("What's up?").unwrap{ assertEquals("All good!", it) } + } + close(sessions.toNonEmptySet()) + } + } + } + + @InitiatedBy(InitiatorMultipleSessionsFlow::class) + class InitiatedMultipleSessionsFlow(private val otherSideSession: FlowSession): FlowLogic() { + @Suspendable + override fun call() { + otherSideSession.receive() + .unwrap{ assertEquals("What's up?", it) } + otherSideSession.send("All good!") + } + } + +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index 2d57f8947e..412ccd72a6 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -60,13 +60,11 @@ import net.corda.nodeapi.internal.lifecycle.NodeLifecycleObserver.Companion.repo import net.corda.node.internal.NodeStartup import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.Checkpoint -import net.corda.node.services.statemachine.DataSessionMessage import net.corda.node.services.statemachine.ErrorState -import net.corda.node.services.statemachine.FlowError +import net.corda.node.services.statemachine.ExistingSessionMessagePayload import net.corda.node.services.statemachine.FlowSessionImpl import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.FlowStateMachineImpl -import net.corda.node.services.statemachine.InitiatedSessionState import net.corda.node.services.statemachine.SessionId import net.corda.node.services.statemachine.SessionState import net.corda.node.services.statemachine.SubFlow @@ -325,6 +323,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri val send: List? = null, val receive: NonEmptySet? = null, val sendAndReceive: List? = null, + val closeSessions: NonEmptySet? = null, val waitForLedgerCommit: SecureHash? = null, val waitForStateConsumption: Set? = null, val getFlowInfo: NonEmptySet? = null, @@ -352,6 +351,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri is FlowIORequest.Send -> SuspendedOn(send = sessionToMessage.toJson()) is FlowIORequest.Receive -> SuspendedOn(receive = sessions) is FlowIORequest.SendAndReceive -> SuspendedOn(sendAndReceive = sessionToMessage.toJson()) + is FlowIORequest.CloseSessions -> SuspendedOn(closeSessions = sessions) is FlowIORequest.WaitForLedgerCommit -> SuspendedOn(waitForLedgerCommit = hash) is FlowIORequest.GetFlowInfo -> SuspendedOn(getFlowInfo = sessions) is FlowIORequest.Sleep -> SuspendedOn(sleepTill = wakeUpAfter) @@ -379,16 +379,14 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri private class ActiveSession( val peer: Party, val ourSessionId: SessionId, - val receivedMessages: List, - val errors: List, + val receivedMessages: List, val peerFlowInfo: FlowInfo, val peerSessionId: SessionId? ) private fun SessionState.toActiveSession(sessionId: SessionId): ActiveSession? { return if (this is SessionState.Initiated) { - val peerSessionId = (initiatedState as? InitiatedSessionState.Live)?.peerSinkSessionId - ActiveSession(peerParty, sessionId, receivedMessages, errors, peerFlowInfo, peerSessionId) + ActiveSession(peerParty, sessionId, receivedMessages, peerFlowInfo, peerSinkSessionId) } else { null } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 435ae5d6f3..7f31d0e743 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -130,13 +130,9 @@ internal class ActionExecutorImpl( log.warn("Propagating error", exception) } for (sessionState in action.sessions) { - // We cannot propagate if the session isn't live. - if (sessionState.initiatedState !is InitiatedSessionState.Live) { - continue - } // Don't propagate errors to the originating session for (errorMessage in action.errorMessages) { - val sinkSessionId = sessionState.initiatedState.peerSinkSessionId + val sinkSessionId = sessionState.peerSinkSessionId val existingMessage = ExistingSessionMessage(sinkSessionId, errorMessage) val deduplicationId = DeduplicationId.createForError(errorMessage.errorId, sinkSessionId) flowMessaging.sendSessionMessage(sessionState.peerParty, existingMessage, SenderDeduplicationId(deduplicationId, action.senderUUID)) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt index 9f80005880..5fba6ae8b7 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt @@ -78,6 +78,7 @@ internal class FlowMonitor( is FlowIORequest.Send -> "to send a message to parties ${request.sessionToMessage.keys.partiesInvolved()}" is FlowIORequest.Receive -> "to receive messages from parties ${request.sessions.partiesInvolved()}" is FlowIORequest.SendAndReceive -> "to send and receive messages from parties ${request.sessionToMessage.keys.partiesInvolved()}" + is FlowIORequest.CloseSessions -> "to close sessions: ${request.sessions}" is FlowIORequest.WaitForLedgerCommit -> "for the ledger to commit transaction with hash ${request.hash}" is FlowIORequest.GetFlowInfo -> "to get flow information from parties ${request.sessions.partiesInvolved()}" is FlowIORequest.Sleep -> "to wake up from sleep ending at ${LocalDateTime.ofInstant(request.wakeUpAfter, ZoneId.systemDefault())}" diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSessionImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSessionImpl.kt index 0dc2e53b23..7d02a23c99 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSessionImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSessionImpl.kt @@ -81,6 +81,12 @@ class FlowSessionImpl( @Suspendable override fun send(payload: Any) = send(payload, maySkipCheckpoint = false) + @Suspendable + override fun close() { + val request = FlowIORequest.CloseSessions(NonEmptySet.of(this)) + return flowStateMachine.suspend(request, false) + } + private fun enforceNotPrimitive(type: Class<*>) { require(!type.isPrimitive) { "Cannot receive primitive type $type" } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 58a072fc99..e652d26145 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -106,6 +106,7 @@ data class Checkpoint( invocationContext, ourIdentity, emptyMap(), + emptySet(), listOf(topLevelSubFlow), numberOfSuspends = 0 ), @@ -132,6 +133,22 @@ data class Checkpoint( return copy(checkpointState = checkpointState.copy(sessions = checkpointState.sessions + session)) } + fun addSessionsToBeClosed(sessionIds: Set): Checkpoint { + return copy(checkpointState = checkpointState.copy(sessionsToBeClosed = checkpointState.sessionsToBeClosed + sessionIds)) + } + + fun removeSessionsToBeClosed(sessionIds: Set): Checkpoint { + return copy(checkpointState = checkpointState.copy(sessionsToBeClosed = checkpointState.sessionsToBeClosed - sessionIds)) + } + + /** + * Returns a copy of the Checkpoint with the specified session removed from the session map. + * @param sessionIds the sessions to remove. + */ + fun removeSessions(sessionIds: Set): Checkpoint { + return copy(checkpointState = checkpointState.copy(sessions = checkpointState.sessions - sessionIds)) + } + /** * Returns a copy of the Checkpoint with a new subFlow stack. * @param subFlows the new List of subFlows. @@ -193,16 +210,18 @@ data class Checkpoint( * @param invocationContext the initiator of the flow. * @param ourIdentity the identity the flow is run as. * @param sessions map of source session ID to session state. + * @param sessionsToBeClosed the sessions that have pending session end messages and need to be closed. This is available to avoid scanning all the sessions. * @param subFlowStack the stack of currently executing subflows. * @param numberOfSuspends the number of flow suspends due to IO API calls. */ @CordaSerializable data class CheckpointState( - val invocationContext: InvocationContext, - val ourIdentity: Party, - val sessions: SessionMap, // This must preserve the insertion order! - val subFlowStack: List, - val numberOfSuspends: Int + val invocationContext: InvocationContext, + val ourIdentity: Party, + val sessions: SessionMap, // This must preserve the insertion order! + val sessionsToBeClosed: Set, + val subFlowStack: List, + val numberOfSuspends: Int ) /** @@ -236,30 +255,25 @@ sealed class SessionState { /** * We have received a confirmation, the peer party and session id is resolved. - * @property errors if not empty the session is in an errored state. + * @property receivedMessages the messages that have been received and are pending processing. + * this could be any [ExistingSessionMessagePayload] type in theory, but it in practice it can only be one of the following types now: + * * [DataSessionMessage] + * * [ErrorSessionMessage] + * * [EndSessionMessage] + * @property otherSideErrored whether the session has received an error from the other side. */ data class Initiated( val peerParty: Party, val peerFlowInfo: FlowInfo, - val receivedMessages: List, - val initiatedState: InitiatedSessionState, - val errors: List, + val receivedMessages: List, + val otherSideErrored: Boolean, + val peerSinkSessionId: SessionId, override val deduplicationSeed: String ) : SessionState() } typealias SessionMap = Map -/** - * Tracks whether an initiated session state is live or has ended. This is a separate state, as we still need the rest - * of [SessionState.Initiated], even when the session has ended, for un-drained session messages and potential future - * [FlowInfo] requests. - */ -sealed class InitiatedSessionState { - data class Live(val peerSinkSessionId: SessionId) : InitiatedSessionState() - object Ended : InitiatedSessionState() { override fun toString() = "Ended" } -} - /** * Represents the way the flow has started. */ diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt index 0aa58241eb..5719139095 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt @@ -1,9 +1,8 @@ package net.corda.node.services.statemachine.transitions -import net.corda.core.flows.FlowException import net.corda.core.flows.UnexpectedFlowEndException -import net.corda.core.identity.Party -import net.corda.core.internal.DeclaredField +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug import net.corda.node.services.statemachine.Action import net.corda.node.services.statemachine.ConfirmSessionMessage import net.corda.node.services.statemachine.DataSessionMessage @@ -12,7 +11,7 @@ import net.corda.node.services.statemachine.ErrorSessionMessage import net.corda.node.services.statemachine.Event import net.corda.node.services.statemachine.ExistingSessionMessage import net.corda.node.services.statemachine.FlowError -import net.corda.node.services.statemachine.InitiatedSessionState +import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.RejectSessionMessage import net.corda.node.services.statemachine.SenderDeduplicationId import net.corda.node.services.statemachine.SessionState @@ -37,6 +36,11 @@ class DeliverSessionMessageTransition( override val startingState: StateMachineState, val event: Event.DeliverSessionMessage ) : Transition { + + private companion object { + val log = contextLogger() + } + override fun transition(): TransitionResult { return builder { // Add the DeduplicationHandler to the pending ones ASAP so in case an error happens we still know @@ -49,7 +53,7 @@ class DeliverSessionMessageTransition( // Check whether we have a session corresponding to the message. val existingSession = startingState.checkpoint.checkpointState.sessions[event.sessionMessage.recipientSessionId] if (existingSession == null) { - freshErrorTransition(CannotFindSessionException(event.sessionMessage.recipientSessionId)) + checkIfMissingSessionIsAnIssue(event.sessionMessage) } else { val payload = event.sessionMessage.payload // Dispatch based on what kind of message it is. @@ -58,7 +62,7 @@ class DeliverSessionMessageTransition( is DataSessionMessage -> dataMessageTransition(existingSession, payload) is ErrorSessionMessage -> errorMessageTransition(existingSession, payload) is RejectSessionMessage -> rejectMessageTransition(existingSession, payload) - is EndSessionMessage -> endMessageTransition() + is EndSessionMessage -> endMessageTransition(payload) } } // Schedule a DoRemainingWork to check whether the flow needs to be woken up. @@ -67,6 +71,14 @@ class DeliverSessionMessageTransition( } } + private fun TransitionBuilder.checkIfMissingSessionIsAnIssue(message: ExistingSessionMessage) { + val payload = message.payload + if (payload is EndSessionMessage) + log.debug { "Received session end message for a session that has already ended: ${event.sessionMessage.recipientSessionId}"} + else + freshErrorTransition(CannotFindSessionException(event.sessionMessage.recipientSessionId)) + } + private fun TransitionBuilder.confirmMessageTransition(sessionState: SessionState, message: ConfirmSessionMessage) { // We received a confirmation message. The corresponding session state must be Initiating. when (sessionState) { @@ -76,9 +88,9 @@ class DeliverSessionMessageTransition( peerParty = event.sender, peerFlowInfo = message.initiatedFlowInfo, receivedMessages = emptyList(), - initiatedState = InitiatedSessionState.Live(message.initiatedSessionId), - errors = emptyList(), - deduplicationSeed = sessionState.deduplicationSeed + peerSinkSessionId = message.initiatedSessionId, + deduplicationSeed = sessionState.deduplicationSeed, + otherSideErrored = false ) val newCheckpoint = currentState.checkpoint.addSession( event.sessionMessage.recipientSessionId to initiatedSession @@ -115,28 +127,11 @@ class DeliverSessionMessageTransition( } private fun TransitionBuilder.errorMessageTransition(sessionState: SessionState, payload: ErrorSessionMessage) { - val exception: Throwable = if (payload.flowException == null) { - UnexpectedFlowEndException("Counter-flow errored", cause = null, originalErrorId = payload.errorId) - } else { - payload.flowException.originalErrorId = payload.errorId - payload.flowException - } - return when (sessionState) { is SessionState.Initiated -> { - when (exception) { - // reflection used to access private field - is UnexpectedFlowEndException -> DeclaredField( - UnexpectedFlowEndException::class.java, - "peer", - exception - ).value = sessionState.peerParty - is FlowException -> DeclaredField(FlowException::class.java, "peer", exception).value = sessionState.peerParty - } val checkpoint = currentState.checkpoint val sessionId = event.sessionMessage.recipientSessionId - val flowError = FlowError(payload.errorId, exception) - val newSessionState = sessionState.copy(errors = sessionState.errors + flowError) + val newSessionState = sessionState.copy(receivedMessages = sessionState.receivedMessages + payload) currentState = currentState.copy( checkpoint = checkpoint.addSession(sessionId to newSessionState) ) @@ -165,23 +160,26 @@ class DeliverSessionMessageTransition( } } - private fun TransitionBuilder.endMessageTransition() { + private fun TransitionBuilder.endMessageTransition(payload: EndSessionMessage) { + val sessionId = event.sessionMessage.recipientSessionId val sessions = currentState.checkpoint.checkpointState.sessions - val sessionState = sessions[sessionId] - if (sessionState == null) { - return freshErrorTransition(CannotFindSessionException(sessionId)) - } + // a check has already been performed to confirm the session exists for this message before this method is invoked. + val sessionState = sessions[sessionId]!! when (sessionState) { is SessionState.Initiated -> { - val newSessionState = sessionState.copy(initiatedState = InitiatedSessionState.Ended) - currentState = currentState.copy( - checkpoint = currentState.checkpoint.addSession(sessionId to newSessionState) + val flowState = currentState.checkpoint.flowState + // flow must have already been started when session end messages are being delivered. + if (flowState !is FlowState.Started) + return freshErrorTransition(UnexpectedEventInState()) - ) + val newSessionState = sessionState.copy(receivedMessages = sessionState.receivedMessages + payload) + val newCheckpoint = currentState.checkpoint.addSession(event.sessionMessage.recipientSessionId to newSessionState) + .addSessionsToBeClosed(setOf(event.sessionMessage.recipientSessionId)) + currentState = currentState.copy(checkpoint = newCheckpoint) } else -> { - freshErrorTransition(UnexpectedEventInState()) + freshErrorTransition(PrematureSessionEndException(event.sessionMessage.recipientSessionId)) } } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt index 551807fcdf..ba5ecaa6bd 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt @@ -117,8 +117,9 @@ class ErrorFlowTransition( sessionState } } + // if we have already received error message from the other side, we don't include that session in the list to avoid propagating errors. val initiatedSessions = sessions.values.mapNotNull { session -> - if (session is SessionState.Initiated && session.errors.isEmpty()) { + if (session is SessionState.Initiated && !session.otherSideErrored) { session } else { null diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt index 5c7b095e80..9c44f5988c 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt @@ -105,8 +105,9 @@ class KilledFlowTransition( sessionState } } + // if we have already received error message from the other side, we don't include that session in the list to avoid propagating errors. val initiatedSessions = sessions.values.mapNotNull { session -> - if (session is SessionState.Initiated && session.errors.isEmpty()) { + if (session is SessionState.Initiated && !session.otherSideErrored) { session } else { null diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt index 96b6557829..cea423134f 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt @@ -1,13 +1,18 @@ package net.corda.node.services.statemachine.transitions +import net.corda.core.flows.FlowException import net.corda.core.flows.FlowInfo import net.corda.core.flows.FlowSession import net.corda.core.flows.UnexpectedFlowEndException +import net.corda.core.identity.Party +import net.corda.core.internal.DeclaredField import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes +import net.corda.core.utilities.contextLogger import net.corda.core.utilities.toNonEmptySet import net.corda.node.services.statemachine.* -import java.lang.IllegalStateException +import org.slf4j.Logger +import kotlin.collections.LinkedHashMap /** * This transition describes what should happen with a specific [FlowIORequest]. Note that at this time the request @@ -20,28 +25,62 @@ class StartedFlowTransition( override val startingState: StateMachineState, val started: FlowState.Started ) : Transition { + + companion object { + private val logger: Logger = contextLogger() + } + override fun transition(): TransitionResult { val flowIORequest = started.flowIORequest - val checkpoint = startingState.checkpoint - val errorsToThrow = collectRelevantErrorsToThrow(flowIORequest, checkpoint) + val (newState, errorsToThrow) = collectRelevantErrorsToThrow(startingState, flowIORequest) if (errorsToThrow.isNotEmpty()) { return TransitionResult( - newState = startingState.copy(isFlowResumed = true), + newState = newState.copy(isFlowResumed = true), // throw the first exception. TODO should this aggregate all of them somehow? actions = listOf(Action.CreateTransaction), continuation = FlowContinuation.Throw(errorsToThrow[0]) ) } - return when (flowIORequest) { - is FlowIORequest.Send -> sendTransition(flowIORequest) - is FlowIORequest.Receive -> receiveTransition(flowIORequest) - is FlowIORequest.SendAndReceive -> sendAndReceiveTransition(flowIORequest) - is FlowIORequest.WaitForLedgerCommit -> waitForLedgerCommitTransition(flowIORequest) - is FlowIORequest.Sleep -> sleepTransition(flowIORequest) - is FlowIORequest.GetFlowInfo -> getFlowInfoTransition(flowIORequest) - is FlowIORequest.WaitForSessionConfirmations -> waitForSessionConfirmationsTransition() - is FlowIORequest.ExecuteAsyncOperation<*> -> executeAsyncOperation(flowIORequest) - FlowIORequest.ForceCheckpoint -> executeForceCheckpoint() + val sessionsToBeTerminated = findSessionsToBeTerminated(startingState) + // if there are sessions to be closed, we close them as part of this transition and normal processing will continue on the next transition. + return if (sessionsToBeTerminated.isNotEmpty()) { + terminateSessions(sessionsToBeTerminated) + } else { + when (flowIORequest) { + is FlowIORequest.Send -> sendTransition(flowIORequest) + is FlowIORequest.Receive -> receiveTransition(flowIORequest) + is FlowIORequest.SendAndReceive -> sendAndReceiveTransition(flowIORequest) + is FlowIORequest.CloseSessions -> closeSessionTransition(flowIORequest) + is FlowIORequest.WaitForLedgerCommit -> waitForLedgerCommitTransition(flowIORequest) + is FlowIORequest.Sleep -> sleepTransition(flowIORequest) + is FlowIORequest.GetFlowInfo -> getFlowInfoTransition(flowIORequest) + is FlowIORequest.WaitForSessionConfirmations -> waitForSessionConfirmationsTransition() + is FlowIORequest.ExecuteAsyncOperation<*> -> executeAsyncOperation(flowIORequest) + FlowIORequest.ForceCheckpoint -> executeForceCheckpoint() + } + } + } + + private fun findSessionsToBeTerminated(startingState: StateMachineState): SessionMap { + return startingState.checkpoint.checkpointState.sessionsToBeClosed.mapNotNull { sessionId -> + val sessionState = startingState.checkpoint.checkpointState.sessions[sessionId]!! as SessionState.Initiated + if (sessionState.receivedMessages.isNotEmpty() && sessionState.receivedMessages.first() is EndSessionMessage) { + sessionId to sessionState + } else { + null + } + }.toMap() + } + + private fun terminateSessions(sessionsToBeTerminated: SessionMap): TransitionResult { + return builder { + val sessionsToRemove = sessionsToBeTerminated.keys + val newCheckpoint = currentState.checkpoint.removeSessions(sessionsToRemove) + .removeSessionsToBeClosed(sessionsToRemove) + currentState = currentState.copy(checkpoint = newCheckpoint) + actions.add(Action.RemoveSessionBindings(sessionsToRemove)) + actions.add(Action.ScheduleEvent(Event.DoRemainingWork)) + FlowContinuation.ProcessEvents } } @@ -149,6 +188,34 @@ class StartedFlowTransition( } } + private fun closeSessionTransition(flowIORequest: FlowIORequest.CloseSessions): TransitionResult { + return builder { + val sessionIdsToRemove = flowIORequest.sessions.map { sessionToSessionId(it) }.toSet() + val existingSessionsToRemove = currentState.checkpoint.checkpointState.sessions.filter { (sessionId, _) -> + sessionIdsToRemove.contains(sessionId) + } + val alreadyClosedSessions = sessionIdsToRemove.filter { sessionId -> sessionId !in existingSessionsToRemove } + if (alreadyClosedSessions.isNotEmpty()) { + logger.warn("Attempting to close already closed sessions: $alreadyClosedSessions") + } + + if (existingSessionsToRemove.isNotEmpty()) { + val sendEndMessageActions = existingSessionsToRemove.values.mapIndexed { index, state -> + val sinkSessionId = (state as SessionState.Initiated).peerSinkSessionId + val message = ExistingSessionMessage(sinkSessionId, EndSessionMessage) + val deduplicationId = DeduplicationId.createForNormal(currentState.checkpoint, index, state) + Action.SendExisting(state.peerParty, message, SenderDeduplicationId(deduplicationId, currentState.senderUUID)) + } + + currentState = currentState.copy(checkpoint = currentState.checkpoint.removeSessions(existingSessionsToRemove.keys)) + actions.add(Action.RemoveSessionBindings(sessionIdsToRemove)) + actions.add(Action.SendMultiple(emptyList(), sendEndMessageActions)) + } + + resumeFlowLogic(Unit) + } + } + private fun receiveTransition(flowIORequest: FlowIORequest.Receive): TransitionResult { return builder { val sessionIdToSession = LinkedHashMap() @@ -199,7 +266,8 @@ class StartedFlowTransition( someNotFound = true } else { newSessionMessages[sessionId] = sessionState.copy(receivedMessages = messages.subList(1, messages.size).toList()) - resultMessages[sessionId] = messages[0].payload + // at this point, we've already checked for errors and session ends, so it's guaranteed that the first message will be a data message. + resultMessages[sessionId] = (messages[0] as DataSessionMessage).payload } } else -> { @@ -257,12 +325,6 @@ class StartedFlowTransition( val checkpoint = startingState.checkpoint val newSessions = LinkedHashMap(checkpoint.checkpointState.sessions) var index = 0 - for ((sourceSessionId, _) in sourceSessionIdToMessage) { - val existingSessionState = checkpoint.checkpointState.sessions[sourceSessionId] ?: return freshErrorTransition(CannotFindSessionException(sourceSessionId)) - if (existingSessionState is SessionState.Initiated && existingSessionState.initiatedState is InitiatedSessionState.Ended) { - return freshErrorTransition(IllegalStateException("Tried to send to ended session $sourceSessionId")) - } - } val messagesByType = sourceSessionIdToMessage.toList() .map { (sourceSessionId, message) -> Triple(sourceSessionId, checkpoint.checkpointState.sessions[sourceSessionId]!!, message) } @@ -286,17 +348,13 @@ class StartedFlowTransition( val newBufferedMessages = initiatingSessionState.bufferedMessages + Pair(deduplicationId, sessionMessage) newSessions[sourceSessionId] = initiatingSessionState.copy(bufferedMessages = newBufferedMessages) } - val sendExistingActions = messagesByType[SessionState.Initiated::class]?.mapNotNull {(_, sessionState, message) -> + val sendExistingActions = messagesByType[SessionState.Initiated::class]?.map {(_, sessionState, message) -> val initiatedSessionState = sessionState as SessionState.Initiated - if (initiatedSessionState.initiatedState !is InitiatedSessionState.Live) - null - else { - val sessionMessage = DataSessionMessage(message) - val deduplicationId = DeduplicationId.createForNormal(checkpoint, index++, initiatedSessionState) - val sinkSessionId = initiatedSessionState.initiatedState.peerSinkSessionId - val existingMessage = ExistingSessionMessage(sinkSessionId, sessionMessage) - Action.SendExisting(initiatedSessionState.peerParty, existingMessage, SenderDeduplicationId(deduplicationId, startingState.senderUUID)) - } + val sessionMessage = DataSessionMessage(message) + val deduplicationId = DeduplicationId.createForNormal(checkpoint, index++, initiatedSessionState) + val sinkSessionId = initiatedSessionState.peerSinkSessionId + val existingMessage = ExistingSessionMessage(sinkSessionId, sessionMessage) + Action.SendExisting(initiatedSessionState.peerParty, existingMessage, SenderDeduplicationId(deduplicationId, startingState.senderUUID)) } ?: emptyList() if (sendInitialActions.isNotEmpty() || sendExistingActions.isNotEmpty()) { @@ -309,21 +367,68 @@ class StartedFlowTransition( return (session as FlowSessionImpl).sourceSessionId } - private fun collectErroredSessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { - return sessionIds.flatMap { sessionId -> - val sessionState = checkpoint.checkpointState.sessions[sessionId]!! - when (sessionState) { - is SessionState.Uninitiated -> emptyList() - is SessionState.Initiating -> { - if (sessionState.rejectionError == null) { - emptyList() - } else { - listOf(sessionState.rejectionError.exception) + private fun collectErroredSessionErrors(startingState: StateMachineState, sessionIds: Collection): Pair> { + var newState = startingState + val errors = sessionIds.filter { sessionId -> + startingState.checkpoint.checkpointState.sessions.containsKey(sessionId) + }.flatMap { sessionId -> + val sessionState = startingState.checkpoint.checkpointState.sessions[sessionId]!! + when (sessionState) { + is SessionState.Uninitiated -> emptyList() + is SessionState.Initiating -> { + if (sessionState.rejectionError == null) { + emptyList() + } else { + listOf(sessionState.rejectionError.exception) + } + } + is SessionState.Initiated -> { + if (sessionState.receivedMessages.isNotEmpty() && sessionState.receivedMessages.first() is ErrorSessionMessage) { + val errorMessage = sessionState.receivedMessages.first() as ErrorSessionMessage + val exception = convertErrorMessageToException(errorMessage, sessionState.peerParty) + val newSessionState = sessionState.copy(receivedMessages = sessionState.receivedMessages.subList(1, sessionState.receivedMessages.size), otherSideErrored = true) + val newCheckpoint = startingState.checkpoint.addSession(sessionId to newSessionState) + newState = startingState.copy(checkpoint = newCheckpoint) + listOf(exception) + } else { + emptyList() + } + } } } - is SessionState.Initiated -> sessionState.errors.map(FlowError::exception) - } + return Pair(newState, errors) + } + + private fun convertErrorMessageToException(errorMessage: ErrorSessionMessage, peer: Party): Throwable { + val exception: Throwable = if (errorMessage.flowException == null) { + UnexpectedFlowEndException("Counter-flow errored", cause = null, originalErrorId = errorMessage.errorId) + } else { + errorMessage.flowException.originalErrorId = errorMessage.errorId + errorMessage.flowException } + when (exception) { + // reflection used to access private field + is UnexpectedFlowEndException -> DeclaredField( + UnexpectedFlowEndException::class.java, + "peer", + exception + ).value = peer + is FlowException -> DeclaredField(FlowException::class.java, "peer", exception).value = peer + } + return exception + } + + private fun collectUncloseableSessions(sessionIds: Collection, checkpoint: Checkpoint): List { + val uninitialisedSessions = sessionIds.mapNotNull { sessionId -> + if (!checkpoint.checkpointState.sessions.containsKey(sessionId)) + null + else + sessionId to checkpoint.checkpointState.sessions[sessionId] + } + .filter { (_, sessionState) -> sessionState !is SessionState.Initiated } + .map { it.first } + + return uninitialisedSessions.map { PrematureSessionCloseException(it) } } private fun collectErroredInitiatingSessionErrors(checkpoint: Checkpoint): List { @@ -333,77 +438,64 @@ class StartedFlowTransition( } private fun collectEndedSessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { - return sessionIds.mapNotNull { sessionId -> - val sessionState = checkpoint.checkpointState.sessions[sessionId]!! - when (sessionState) { - is SessionState.Initiated -> { - if (sessionState.initiatedState === InitiatedSessionState.Ended) { - UnexpectedFlowEndException( - "Tried to access ended session $sessionId", - cause = null, - originalErrorId = context.secureRandom.nextLong() - ) - } else { - null - } - } - else -> null - } + return sessionIds.filter { sessionId -> + !checkpoint.checkpointState.sessions.containsKey(sessionId) + }.map {sessionId -> + UnexpectedFlowEndException( + "Tried to access ended session $sessionId", + cause = null, + originalErrorId = context.secureRandom.nextLong() + ) } } - private fun collectEndedEmptySessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { - return sessionIds.mapNotNull { sessionId -> - val sessionState = checkpoint.checkpointState.sessions[sessionId]!! - when (sessionState) { - is SessionState.Initiated -> { - if (sessionState.initiatedState === InitiatedSessionState.Ended && - sessionState.receivedMessages.isEmpty()) { - UnexpectedFlowEndException( - "Tried to access ended session $sessionId with empty buffer", - cause = null, - originalErrorId = context.secureRandom.nextLong() - ) - } else { - null - } - } - else -> null - } - } - } - - private fun collectRelevantErrorsToThrow(flowIORequest: FlowIORequest<*>, checkpoint: Checkpoint): List { + private fun collectRelevantErrorsToThrow(startingState: StateMachineState, flowIORequest: FlowIORequest<*>): Pair> { return when (flowIORequest) { is FlowIORequest.Send -> { val sessionIds = flowIORequest.sessionToMessage.keys.map(this::sessionToSessionId) - collectErroredSessionErrors(sessionIds, checkpoint) + collectEndedSessionErrors(sessionIds, checkpoint) + val (newState, erroredSessionErrors) = collectErroredSessionErrors(startingState, sessionIds) + val endedSessionErrors = collectEndedSessionErrors(sessionIds, startingState.checkpoint) + Pair(newState, erroredSessionErrors + endedSessionErrors) } is FlowIORequest.Receive -> { val sessionIds = flowIORequest.sessions.map(this::sessionToSessionId) - collectErroredSessionErrors(sessionIds, checkpoint) + collectEndedEmptySessionErrors(sessionIds, checkpoint) + val (newState, erroredSessionErrors) = collectErroredSessionErrors(startingState, sessionIds) + val endedSessionErrors = collectEndedSessionErrors(sessionIds, startingState.checkpoint) + Pair(newState, erroredSessionErrors + endedSessionErrors) } is FlowIORequest.SendAndReceive -> { val sessionIds = flowIORequest.sessionToMessage.keys.map(this::sessionToSessionId) - collectErroredSessionErrors(sessionIds, checkpoint) + collectEndedSessionErrors(sessionIds, checkpoint) + val (newState, erroredSessionErrors) = collectErroredSessionErrors(startingState, sessionIds) + val endedSessionErrors = collectEndedSessionErrors(sessionIds, startingState.checkpoint) + Pair(newState, erroredSessionErrors + endedSessionErrors) } is FlowIORequest.WaitForLedgerCommit -> { - collectErroredSessionErrors(checkpoint.checkpointState.sessions.keys, checkpoint) + return collectErroredSessionErrors(startingState, startingState.checkpoint.checkpointState.sessions.keys) } is FlowIORequest.GetFlowInfo -> { - collectErroredSessionErrors(flowIORequest.sessions.map(this::sessionToSessionId), checkpoint) + val sessionIds = flowIORequest.sessions.map(this::sessionToSessionId) + val (newState, erroredSessionErrors) = collectErroredSessionErrors(startingState, sessionIds) + val endedSessionErrors = collectEndedSessionErrors(sessionIds, startingState.checkpoint) + Pair(newState, erroredSessionErrors + endedSessionErrors) + } + is FlowIORequest.CloseSessions -> { + val sessionIds = flowIORequest.sessions.map(this::sessionToSessionId) + val (newState, erroredSessionErrors) = collectErroredSessionErrors(startingState, sessionIds) + val uncloseableSessionErrors = collectUncloseableSessions(sessionIds, startingState.checkpoint) + Pair(newState, erroredSessionErrors + uncloseableSessionErrors) } is FlowIORequest.Sleep -> { - emptyList() + Pair(startingState, emptyList()) } is FlowIORequest.WaitForSessionConfirmations -> { - collectErroredInitiatingSessionErrors(checkpoint) + val errors = collectErroredInitiatingSessionErrors(startingState.checkpoint) + Pair(startingState, errors) } is FlowIORequest.ExecuteAsyncOperation<*> -> { - emptyList() + Pair(startingState, emptyList()) } FlowIORequest.ForceCheckpoint -> { - emptyList() + Pair(startingState, emptyList()) } } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 1b7d79dfec..4846ee101d 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -18,7 +18,6 @@ import net.corda.node.services.statemachine.FlowRemovalReason import net.corda.node.services.statemachine.FlowSessionImpl import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.InitialSessionMessage -import net.corda.node.services.statemachine.InitiatedSessionState import net.corda.node.services.statemachine.SenderDeduplicationId import net.corda.node.services.statemachine.SessionId import net.corda.node.services.statemachine.SessionMessage @@ -267,8 +266,8 @@ class TopLevelTransition( private fun TransitionBuilder.sendEndMessages() { val sendEndMessageActions = currentState.checkpoint.checkpointState.sessions.values.mapIndexed { index, state -> - if (state is SessionState.Initiated && state.initiatedState is InitiatedSessionState.Live) { - val message = ExistingSessionMessage(state.initiatedState.peerSinkSessionId, EndSessionMessage) + if (state is SessionState.Initiated) { + val message = ExistingSessionMessage(state.peerSinkSessionId, EndSessionMessage) val deduplicationId = DeduplicationId.createForNormal(currentState.checkpoint, index, state) Action.SendExisting(state.peerParty, message, SenderDeduplicationId(deduplicationId, currentState.senderUUID)) } else { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt index 5e6ca3adbb..dac380b3c2 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt @@ -81,3 +81,5 @@ class TransitionBuilder(val context: TransitionContext, initialState: StateMachi class CannotFindSessionException(sessionId: SessionId) : IllegalStateException("Couldn't find session with id $sessionId") class UnexpectedEventInState : IllegalStateException("Unexpected event") +class PrematureSessionCloseException(sessionId: SessionId): IllegalStateException("The following session was closed before it was initialised: $sessionId") +class PrematureSessionEndException(sessionId: SessionId): IllegalStateException("A premature session end message was received before the session was initialised: $sessionId") \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt index c85830fb03..7361943cde 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt @@ -8,7 +8,6 @@ import net.corda.node.services.statemachine.DeduplicationId import net.corda.node.services.statemachine.ExistingSessionMessage import net.corda.node.services.statemachine.FlowStart import net.corda.node.services.statemachine.FlowState -import net.corda.node.services.statemachine.InitiatedSessionState import net.corda.node.services.statemachine.SenderDeduplicationId import net.corda.node.services.statemachine.SessionState import net.corda.node.services.statemachine.StateMachineState @@ -45,7 +44,7 @@ class UnstartedFlowTransition( val initiatingMessage = flowStart.initiatingMessage val initiatedState = SessionState.Initiated( peerParty = flowStart.peerSession.counterparty, - initiatedState = InitiatedSessionState.Live(initiatingMessage.initiatorSessionId), + peerSinkSessionId = initiatingMessage.initiatorSessionId, peerFlowInfo = FlowInfo( flowVersion = flowStart.senderCoreFlowVersion ?: initiatingMessage.flowVersion, appName = initiatingMessage.appName @@ -55,8 +54,8 @@ class UnstartedFlowTransition( } else { listOf(DataSessionMessage(initiatingMessage.firstPayload)) }, - errors = emptyList(), - deduplicationSeed = "D-${initiatingMessage.initiatorSessionId.toLong}-${initiatingMessage.initiationEntropy}" + deduplicationSeed = "D-${initiatingMessage.initiatorSessionId.toLong}-${initiatingMessage.initiationEntropy}", + otherSideErrored = false ) val confirmationMessage = ConfirmSessionMessage(flowStart.initiatedSessionId, flowStart.initiatedFlowInfo) val sessionMessage = ExistingSessionMessage(initiatingMessage.initiatorSessionId, confirmationMessage) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index feafb34279..d6eb21aa33 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -201,7 +201,7 @@ class FlowFrameworkTests { } @Test(timeout=300_000) - fun `other side ends before doing expected send`() { + fun `other side ends before doing expected send`() { bobNode.registerCordappFlowFactory(ReceiveFlow::class) { NoOpFlow() } val resultFuture = aliceNode.services.startFlow(ReceiveFlow(bob)).resultFuture mockNet.runNetwork() @@ -868,6 +868,7 @@ class FlowFrameworkTests { session.send(1) // ... then pause this one until it's received the session-end message from the other side receivedOtherFlowEnd.acquire() + session.sendAndReceive(2) } } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index ddac3afba8..0fae5c91bb 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -247,7 +247,7 @@ class FlowMetadataRecordingTest { it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) ) assertThat(it.launchingCordapp).contains("custom-cordapp") - assertEquals(7, it.platformVersion) + assertEquals(8, it.platformVersion) assertEquals(nodeAHandle.nodeInfo.singleIdentity().name.toString(), it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt index 3f5c249424..ee93d937d2 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt @@ -183,6 +183,11 @@ class RetryFlowMockTest { override fun send(payload: Any) { TODO("not implemented") } + + override fun close() { + TODO("Not yet implemented") + } + }), nodeA.services.newContext()).get() records.next() // Killing it should remove it. diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt index aea0e9d5d0..9ae6f1f9d2 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt @@ -529,7 +529,8 @@ open class InternalMockNetwork(cordappPackages: List = emptyList(), } private fun pumpAll(): Boolean { - val transferredMessages = messagingNetwork.endpoints.map { it.pumpReceive(false) } + val transferredMessages = messagingNetwork.endpoints.filter { it.active } + .map { it.pumpReceive(false) } return transferredMessages.any { it != null } } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt index 3b81fbc2ef..a64da72048 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt @@ -173,6 +173,7 @@ class MockNodeMessagingService(private val configuration: NodeConfiguration, it.join() } running = false + stateHelper.active = false network.netNodeHasShutdown(myAddress) } From 50c51d3e6fcb30ae6979cc637ad6a78a8d042842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Tue, 21 Jul 2020 15:42:57 +0100 Subject: [PATCH 23/48] Empty JUnit results are not allowed (#6488) --- .ci/dev/nightly-regression/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 7dd4301440..303bd722a9 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -56,7 +56,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml' } cleanup { deleteDir() /* clean up our workspace */ From 7437630d56b0e846e7ec3b233f294466c5ab589d Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Tue, 21 Jul 2020 15:50:21 +0100 Subject: [PATCH 24/48] Empty JUnit results are not allowed --- .ci/dev/mswin/Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/dev/mswin/Jenkinsfile b/.ci/dev/mswin/Jenkinsfile index b0e3766e0c..714fadf4fb 100644 --- a/.ci/dev/mswin/Jenkinsfile +++ b/.ci/dev/mswin/Jenkinsfile @@ -65,7 +65,7 @@ pipeline { post { always { archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true bat '.ci/kill_corda_procs.cmd' } cleanup { @@ -87,7 +87,7 @@ pipeline { post { always { archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true bat '.ci/kill_corda_procs.cmd' } cleanup { From 5ee262653007551031a60011dc16b8c01c3fb840 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Tue, 21 Jul 2020 15:55:56 +0100 Subject: [PATCH 25/48] Empty JUnit results are not allowed --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 2 +- .ci/dev/regression/Jenkinsfile | 2 +- Jenkinsfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index d24a3f7ac4..1dd3b40043 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -153,7 +153,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml' } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 9af46eba66..8de096bccf 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -177,7 +177,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true script { try { diff --git a/Jenkinsfile b/Jenkinsfile index c27f461148..c96e217de3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -79,7 +79,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ From a03fb8c6fd903ce9db88f4f073f1d012d647af13 Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Tue, 21 Jul 2020 18:10:09 +0100 Subject: [PATCH 26/48] INFRA-438: Close session factory before closing the locator (#6477) --- .../net/corda/node/services/messaging/P2PMessagingClient.kt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt index 0fcb7a3ca7..b9035a6975 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt @@ -121,6 +121,7 @@ class P2PMessagingClient(val config: NodeConfiguration, var bridgeSession: ClientSession? = null var bridgeNotifyConsumer: ClientConsumer? = null var networkChangeSubscription: Subscription? = null + var sessionFactory: ClientSessionFactory? = null fun sendMessage(address: String, message: ClientMessage) = producer!!.send(address, message) } @@ -172,7 +173,7 @@ class P2PMessagingClient(val config: NodeConfiguration, minLargeMessageSize = maxMessageSize + JOURNAL_HEADER_SIZE isUseGlobalPools = nodeSerializationEnv != null } - val sessionFactory = locator!!.createSessionFactory().addFailoverListener(::failoverCallback) + sessionFactory = locator!!.createSessionFactory().addFailoverListener(::failoverCallback) // Login using the node username. The broker will authenticate us as its node (as opposed to another peer) // using our TLS certificate. // Note that the acknowledgement of messages is not flushed to the Artermis journal until the default buffer @@ -490,8 +491,10 @@ class P2PMessagingClient(val config: NodeConfiguration, // Wait for the main loop to notice the consumer has gone and finish up. shutdownLatch.await() } + // Only first caller to gets running true to protect against double stop, which seems to happen in some integration tests. state.locked { + sessionFactory?.close() locator?.close() } } From ca37b9b7372584620cb9cfe6dd4e5dc771369376 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 22 Jul 2020 09:26:39 +0100 Subject: [PATCH 27/48] PR Code Checks use `standard` Jenkins agent (#6496) --- .ci/dev/pr-code-checks/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index a64813c92f..20cf49f912 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -4,7 +4,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() timeout(time: 3, unit: 'HOURS') From 05532c0419dbdfd6b7817028b044bff7184c417f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 22 Jul 2020 12:41:22 +0100 Subject: [PATCH 28/48] NexusIQ updates (#6499) * every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application * NexusIQ application application *has* to exist before a build starts --- .ci/dev/regression/Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 85a699180a..4678bbda6c 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -57,14 +57,15 @@ pipeline { sh "./gradlew --no-daemon clean jar" script { sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" - def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim() def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() def artifactId = 'corda' nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" } nexusPolicyEvaluation ( failBuildOnNetworkError: false, - iqApplication: manualApplication(nexusAppId), + iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], iqStage: nexusIqStage ) From d810067ab686e9472ed799445b09fb4f979a094d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 22 Jul 2020 14:43:14 +0100 Subject: [PATCH 29/48] NOTICK: Fixed for JDK11 builds (#6501) * NexusIQ every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application * NexusIQ application application has to exist before a build starts * Fixed repository name for publishing, use OS instead of Ent one --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 24 ++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index 1dd3b40043..b692a4d736 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -1,3 +1,14 @@ +#!groovy +/** + * Jenkins pipeline to build Corda OS release with JDK11 + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob @@ -22,13 +33,13 @@ if (isReleaseTag) { default: nexusIqStage = "operate" } } + pipeline { - agent { - label 'k8s' - } + agent { label 'k8s' } options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { @@ -48,14 +59,15 @@ pipeline { sh "./gradlew --no-daemon clean jar" script { sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" - def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim() def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() def artifactId = 'corda' nexusAppId = "jenkins-${groupId}-${artifactId}-jdk11-${version}" } nexusPolicyEvaluation ( failBuildOnNetworkError: false, - iqApplication: manualApplication(nexusAppId), + iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], iqStage: nexusIqStage ) @@ -132,7 +144,7 @@ pipeline { rtGradleDeployer( id: 'deployer', serverId: 'R3-Artifactory', - repo: 'r3-corda-releases' + repo: 'corda-releases' ) rtGradleRun( usesPlugin: true, From a41152edf69d80ca25da62f485bb4fd12eb17ac8 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Wed, 22 Jul 2020 16:19:20 +0100 Subject: [PATCH 30/48] CORDA-3899 Refactor flow's transient fields (#6441) Refactor `FlowStateMachineImpl.transientValues` and `FlowStateMachineImpl.transientState` to stop the fields from exposing the fact that they are nullable. This is done by having private backing fields `transientValuesReference` and `transientStateReference` that can be null. The nullability is still needed due to serialisation and deserialisation of flow fibers. The fields are transient and therefore will be null when reloaded from the database. Getters and setters hide the private field, allowing a non-null field to returned. There is no point other than in `FlowCreator` where the transient fields can be null. Therefore the non null checks that are being made are valid. Add custom kryo serialisation and deserialisation to `TransientValues` and `StateMachineState` to ensure that neither of the objects are ever touched by kryo. --- .../node/services/statemachine/FlowCreator.kt | 8 +- .../FlowDefaultUncaughtExceptionHandler.kt | 16 +-- .../node/services/statemachine/FlowMonitor.kt | 4 +- .../statemachine/FlowStateMachineImpl.kt | 128 ++++++++++-------- .../SingleThreadedStateMachineManager.kt | 17 +-- .../statemachine/StateMachineState.kt | 15 +- .../statemachine/FlowFrameworkTests.kt | 22 +-- 7 files changed, 118 insertions(+), 92 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt index be8026b73f..08a006c345 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt @@ -69,11 +69,11 @@ class FlowCreator( val checkpoint = oldCheckpoint.copy(status = Checkpoint.FlowStatus.RUNNABLE) val fiber = checkpoint.getFiberFromCheckpoint(runId) ?: return null val resultFuture = openFuture() - fiber.transientValues = TransientReference(createTransientValues(runId, resultFuture)) fiber.logic.stateMachine = fiber verifyFlowLogicIsSuspendable(fiber.logic) val state = createStateMachineState(checkpoint, fiber, true) - fiber.transientState = TransientReference(state) + fiber.transientValues = createTransientValues(runId, resultFuture) + fiber.transientState = state return Flow(fiber, resultFuture) } @@ -91,7 +91,7 @@ class FlowCreator( // have access to the fiber (and thereby the service hub) val flowStateMachineImpl = FlowStateMachineImpl(flowId, flowLogic, scheduler) val resultFuture = openFuture() - flowStateMachineImpl.transientValues = TransientReference(createTransientValues(flowId, resultFuture)) + flowStateMachineImpl.transientValues = createTransientValues(flowId, resultFuture) flowLogic.stateMachine = flowStateMachineImpl val frozenFlowLogic = (flowLogic as FlowLogic<*>).checkpointSerialize(context = checkpointSerializationContext) val flowCorDappVersion = FlowStateMachineImpl.createSubFlowVersion( @@ -113,7 +113,7 @@ class FlowCreator( existingCheckpoint != null, deduplicationHandler, senderUUID) - flowStateMachineImpl.transientState = TransientReference(state) + flowStateMachineImpl.transientState = state return Flow(flowStateMachineImpl, resultFuture) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt index 0dc5f28791..44a3c8876b 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt @@ -39,18 +39,14 @@ class FlowDefaultUncaughtExceptionHandler( val id = fiber.id if (!fiber.resultFuture.isDone) { fiber.transientState.let { state -> - if (state != null) { - fiber.logger.warn("Forcing flow $id into overnight observation") - flowHospital.forceIntoOvernightObservation(state.value, listOf(throwable)) - val hospitalizedCheckpoint = state.value.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) - val hospitalizedState = state.value.copy(checkpoint = hospitalizedCheckpoint) - fiber.transientState = TransientReference(hospitalizedState) - } else { - fiber.logger.warn("The fiber's transient state is not set, cannot force flow $id into in-memory overnight observation, status will still be updated in database") - } + fiber.logger.warn("Forcing flow $id into overnight observation") + flowHospital.forceIntoOvernightObservation(state, listOf(throwable)) + val hospitalizedCheckpoint = state.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) + val hospitalizedState = state.copy(checkpoint = hospitalizedCheckpoint) + fiber.transientState = hospitalizedState } - scheduledExecutor.schedule({ setFlowToHospitalizedRescheduleOnFailure(id) }, 0, TimeUnit.SECONDS) } + scheduledExecutor.schedule({ setFlowToHospitalizedRescheduleOnFailure(id) }, 0, TimeUnit.SECONDS) } @Suppress("TooGenericExceptionCaught") diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt index 5fba6ae8b7..26ab9d5987 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt @@ -96,12 +96,12 @@ internal class FlowMonitor( private fun FlowStateMachineImpl<*>.ioRequest() = (snapshot().checkpoint.flowState as? FlowState.Started)?.flowIORequest private fun FlowStateMachineImpl<*>.ongoingDuration(now: Instant): Duration { - return transientState?.value?.checkpoint?.timestamp?.let { Duration.between(it, now) } ?: Duration.ZERO + return transientState.checkpoint.timestamp.let { Duration.between(it, now) } ?: Duration.ZERO } private fun FlowStateMachineImpl<*>.isSuspended() = !snapshot().isFlowResumed - private fun FlowStateMachineImpl<*>.isStarted() = transientState?.value?.checkpoint?.flowState is FlowState.Started + private fun FlowStateMachineImpl<*>.isStarted() = transientState.checkpoint.flowState is FlowState.Started private operator fun StaffedFlowHospital.contains(flow: FlowStateMachine<*>) = contains(flow.id) } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 5277d89638..ce4fdea2bd 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -6,6 +6,10 @@ import co.paralleluniverse.fibers.FiberScheduler import co.paralleluniverse.fibers.Suspendable import co.paralleluniverse.strands.Strand import co.paralleluniverse.strands.channels.Channel +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.KryoSerializable +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output import net.corda.core.concurrent.CordaFuture import net.corda.core.context.InvocationContext import net.corda.core.contracts.StateRef @@ -58,7 +62,6 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory import org.slf4j.MDC import java.util.concurrent.TimeUnit -import kotlin.reflect.KProperty1 class FlowPermissionException(message: String) : FlowException(message) @@ -86,52 +89,65 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, private val SERIALIZER_BLOCKER = Fiber::class.java.getDeclaredField("SERIALIZER_BLOCKER").apply { isAccessible = true }.get(null) } - override val serviceHub get() = getTransientField(TransientValues::serviceHub) - data class TransientValues( - val eventQueue: Channel, - val resultFuture: CordaFuture, - val database: CordaPersistence, - val transitionExecutor: TransitionExecutor, - val actionExecutor: ActionExecutor, - val stateMachine: StateMachine, - val serviceHub: ServiceHubInternal, - val checkpointSerializationContext: CheckpointSerializationContext, - val unfinishedFibers: ReusableLatch, - val waitTimeUpdateHook: (id: StateMachineRunId, timeout: Long) -> Unit - ) + val eventQueue: Channel, + val resultFuture: CordaFuture, + val database: CordaPersistence, + val transitionExecutor: TransitionExecutor, + val actionExecutor: ActionExecutor, + val stateMachine: StateMachine, + val serviceHub: ServiceHubInternal, + val checkpointSerializationContext: CheckpointSerializationContext, + val unfinishedFibers: ReusableLatch, + val waitTimeUpdateHook: (id: StateMachineRunId, timeout: Long) -> Unit + ) : KryoSerializable { + override fun write(kryo: Kryo?, output: Output?) { + throw IllegalStateException("${TransientValues::class.qualifiedName} should never be serialized") + } - internal var transientValues: TransientReference? = null - internal var transientState: TransientReference? = null - - /** - * What sender identifier to put on messages sent by this flow. This will either be the identifier for the current - * state machine manager / messaging client, or null to indicate this flow is restored from a checkpoint and - * the de-duplication of messages it sends should not be optimised since this could be unreliable. - */ - override val ourSenderUUID: String? - get() = transientState?.value?.senderUUID - - private fun getTransientField(field: KProperty1): A { - val suppliedValues = transientValues ?: throw IllegalStateException("${field.name} wasn't supplied!") - return field.get(suppliedValues.value) + override fun read(kryo: Kryo?, input: Input?) { + throw IllegalStateException("${TransientValues::class.qualifiedName} should never be deserialized") + } } - private fun extractThreadLocalTransaction(): TransientReference { - val transaction = contextTransaction - contextTransactionOrNull = null - return TransientReference(transaction) - } + private var transientValuesReference: TransientReference? = null + internal var transientValues: TransientValues + // After the flow has been created, the transient values should never be null + get() = transientValuesReference!!.value + set(values) { + check(transientValuesReference?.value == null) { "The transient values should only be set once when initialising a flow" } + transientValuesReference = TransientReference(values) + } + + private var transientStateReference: TransientReference? = null + internal var transientState: StateMachineState + // After the flow has been created, the transient state should never be null + get() = transientStateReference!!.value + set(state) { + transientStateReference = TransientReference(state) + } /** * Return the logger for this state machine. The logger name incorporates [id] and so including it in the log message * is not necessary. */ override val logger = log - override val resultFuture: CordaFuture get() = uncheckedCast(getTransientField(TransientValues::resultFuture)) - override val context: InvocationContext get() = transientState!!.value.checkpoint.checkpointState.invocationContext - override val ourIdentity: Party get() = transientState!!.value.checkpoint.checkpointState.ourIdentity - override val isKilled: Boolean get() = transientState!!.value.isKilled + + override val instanceId: StateMachineInstanceId get() = StateMachineInstanceId(id, super.getId()) + + override val serviceHub: ServiceHubInternal get() = transientValues.serviceHub + override val stateMachine: StateMachine get() = transientValues.stateMachine + override val resultFuture: CordaFuture get() = uncheckedCast(transientValues.resultFuture) + + override val context: InvocationContext get() = transientState.checkpoint.checkpointState.invocationContext + override val ourIdentity: Party get() = transientState.checkpoint.checkpointState.ourIdentity + override val isKilled: Boolean get() = transientState.isKilled + /** + * What sender identifier to put on messages sent by this flow. This will either be the identifier for the current + * state machine manager / messaging client, or null to indicate this flow is restored from a checkpoint and + * the de-duplication of messages it sends should not be optimised since this could be unreliable. + */ + override val ourSenderUUID: String? get() = transientState.senderUUID internal val softLockedStates = mutableSetOf() @@ -143,9 +159,9 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, @Suspendable private fun processEvent(transitionExecutor: TransitionExecutor, event: Event): FlowContinuation { setLoggingContext() - val stateMachine = getTransientField(TransientValues::stateMachine) - val oldState = transientState!!.value - val actionExecutor = getTransientField(TransientValues::actionExecutor) + val stateMachine = transientValues.stateMachine + val oldState = transientState + val actionExecutor = transientValues.actionExecutor val transition = stateMachine.transition(event, oldState) val (continuation, newState) = transitionExecutor.executeTransition(this, oldState, event, transition, actionExecutor) // Ensure that the next state that is being written to the transient state maintains the [isKilled] flag @@ -153,7 +169,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, if (oldState.isKilled && !newState.isKilled) { newState.isKilled = true } - transientState = TransientReference(newState) + transientState = newState setLoggingContext() return continuation } @@ -171,15 +187,15 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, @Suspendable private fun processEventsUntilFlowIsResumed(isDbTransactionOpenOnEntry: Boolean, isDbTransactionOpenOnExit: Boolean): Any? { checkDbTransaction(isDbTransactionOpenOnEntry) - val transitionExecutor = getTransientField(TransientValues::transitionExecutor) - val eventQueue = getTransientField(TransientValues::eventQueue) + val transitionExecutor = transientValues.transitionExecutor + val eventQueue = transientValues.eventQueue try { eventLoop@ while (true) { val nextEvent = try { eventQueue.receive() } catch (interrupted: InterruptedException) { log.error("Flow interrupted while waiting for events, aborting immediately") - (transientValues?.value?.resultFuture as? OpenFuture<*>)?.setException(KilledFlowException(id)) + (transientValues.resultFuture as? OpenFuture<*>)?.setException(KilledFlowException(id)) abortFiber() } val continuation = processEvent(transitionExecutor, nextEvent) @@ -246,7 +262,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, isDbTransactionOpenOnEntry: Boolean, isDbTransactionOpenOnExit: Boolean): FlowContinuation { checkDbTransaction(isDbTransactionOpenOnEntry) - val transitionExecutor = getTransientField(TransientValues::transitionExecutor) + val transitionExecutor = transientValues.transitionExecutor val continuation = processEvent(transitionExecutor, event) checkDbTransaction(isDbTransactionOpenOnExit) return continuation @@ -270,7 +286,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, } private fun openThreadLocalWormhole() { - val threadLocal = getTransientField(TransientValues::database).hikariPoolThreadLocal + val threadLocal = transientValues.database.hikariPoolThreadLocal if (threadLocal != null) { val valueFromThread = swappedOutThreadLocalValue(threadLocal) threadLocal.set(valueFromThread) @@ -332,7 +348,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, } recordDuration(startTime) - getTransientField(TransientValues::unfinishedFibers).countDown() + transientValues.unfinishedFibers.countDown() } @Suspendable @@ -476,7 +492,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, @Suspendable override fun suspend(ioRequest: FlowIORequest, maySkipCheckpoint: Boolean): R { - val serializationContext = TransientReference(getTransientField(TransientValues::checkpointSerializationContext)) + val serializationContext = TransientReference(transientValues.checkpointSerializationContext) val transaction = extractThreadLocalTransaction() parkAndSerialize { _, _ -> setLoggingContext() @@ -524,13 +540,19 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, return subFlowStack.any { IdempotentFlow::class.java.isAssignableFrom(it.flowClass) } } + private fun extractThreadLocalTransaction(): TransientReference { + val transaction = contextTransaction + contextTransactionOrNull = null + return TransientReference(transaction) + } + @Suspendable override fun scheduleEvent(event: Event) { - getTransientField(TransientValues::eventQueue).send(event) + transientValues.eventQueue.send(event) } override fun snapshot(): StateMachineState { - return transientState!!.value + return transientState } /** @@ -538,13 +560,9 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, * retried. */ override fun updateTimedFlowTimeout(timeoutSeconds: Long) { - getTransientField(TransientValues::waitTimeUpdateHook).invoke(id, timeoutSeconds) + transientValues.waitTimeUpdateHook.invoke(id, timeoutSeconds) } - override val stateMachine get() = getTransientField(TransientValues::stateMachine) - - override val instanceId: StateMachineInstanceId get() = StateMachineInstanceId(id, super.getId()) - /** * Records the duration of this flow – from call() to completion or failure. * Note that the duration will include the time the flow spent being parked, and not just the total diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 1d07a75d02..fba5833661 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -261,14 +261,9 @@ internal class SingleThreadedStateMachineManager( unfinishedFibers.countDown() val state = flow.fiber.transientState - return@withLock if (state != null) { - state.value.isKilled = true - flow.fiber.scheduleEvent(Event.DoRemainingWork) - true - } else { - logger.info("Flow $id has not been initialised correctly and cannot be killed") - false - } + state.isKilled = true + flow.fiber.scheduleEvent(Event.DoRemainingWork) + true } else { // It may be that the id refers to a checkpoint that couldn't be deserialised into a flow, so we delete it if it exists. database.transaction { checkpointStorage.removeCheckpoint(id) } @@ -386,7 +381,7 @@ internal class SingleThreadedStateMachineManager( currentState.cancelFutureIfRunning() // Get set of external events val flowId = currentState.flowLogic.runId - val oldFlowLeftOver = innerState.withLock { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue + val oldFlowLeftOver = innerState.withLock { flows[flowId] }?.fiber?.transientValues?.eventQueue if (oldFlowLeftOver == null) { logger.error("Unable to find flow for flow $flowId. Something is very wrong. The flow will not retry.") return @@ -592,7 +587,7 @@ internal class SingleThreadedStateMachineManager( ): CordaFuture> { val existingFlow = innerState.withLock { flows[flowId] } - val existingCheckpoint = if (existingFlow != null && existingFlow.fiber.transientState?.value?.isAnyCheckpointPersisted == true) { + val existingCheckpoint = if (existingFlow != null && existingFlow.fiber.transientState.isAnyCheckpointPersisted) { // Load the flow's checkpoint // The checkpoint will be missing if the flow failed before persisting the original checkpoint // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) @@ -756,7 +751,7 @@ internal class SingleThreadedStateMachineManager( // The flow's event queue may be non-empty in case it shut down abruptly. We handle outstanding events here. private fun drainFlowEventQueue(flow: Flow<*>) { while (true) { - val event = flow.fiber.transientValues!!.value.eventQueue.tryReceive() ?: return + val event = flow.fiber.transientValues.eventQueue.tryReceive() ?: return when (event) { is Event.DoRemainingWork -> {} is Event.DeliverSessionMessage -> { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index e652d26145..6c124d41e6 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -1,5 +1,9 @@ package net.corda.node.services.statemachine +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.KryoSerializable +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output import net.corda.core.context.InvocationContext import net.corda.core.crypto.SecureHash import net.corda.core.flows.Destination @@ -15,6 +19,7 @@ import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.checkpointDeserialize import net.corda.core.utilities.Try import net.corda.node.services.messaging.DeduplicationHandler +import java.lang.IllegalStateException import java.time.Instant import java.util.concurrent.Future @@ -55,7 +60,15 @@ data class StateMachineState( @Volatile var isKilled: Boolean, val senderUUID: String? -) +) : KryoSerializable { + override fun write(kryo: Kryo?, output: Output?) { + throw IllegalStateException("${StateMachineState::class.qualifiedName} should never be serialized") + } + + override fun read(kryo: Kryo?, input: Input?) { + throw IllegalStateException("${StateMachineState::class.qualifiedName} should never be deserialized") + } +} /** * @param checkpointState the state of the checkpoint diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index d6eb21aa33..1967f9ff63 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -26,6 +26,7 @@ import net.corda.core.internal.FlowIORequest import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.concurrent.flatMap import net.corda.core.internal.concurrent.openFuture +import net.corda.core.internal.declaredField import net.corda.core.messaging.MessageRecipients import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy @@ -173,9 +174,12 @@ class FlowFrameworkTests { val flow = ReceiveFlow(bob) val fiber = aliceNode.services.startFlow(flow) as FlowStateMachineImpl // Before the flow runs change the suspend action to throw an exception - val throwingActionExecutor = SuspendThrowingActionExecutor(Exception("Thrown during suspend"), - fiber.transientValues!!.value.actionExecutor) - fiber.transientValues = TransientReference(fiber.transientValues!!.value.copy(actionExecutor = throwingActionExecutor)) + val throwingActionExecutor = SuspendThrowingActionExecutor( + Exception("Thrown during suspend"), + fiber.transientValues.actionExecutor + ) + fiber.declaredField>("transientValuesReference").value = + TransientReference(fiber.transientValues.copy(actionExecutor = throwingActionExecutor)) mockNet.runNetwork() fiber.resultFuture.getOrThrow() assertThat(aliceNode.smm.allStateMachines).isEmpty() @@ -679,14 +683,14 @@ class FlowFrameworkTests { SuspendingFlow.hookBeforeCheckpoint = { val flowFiber = this as? FlowStateMachineImpl<*> - flowState = flowFiber!!.transientState!!.value.checkpoint.flowState + flowState = flowFiber!!.transientState.checkpoint.flowState if (firstExecution) { throw HospitalizeFlowException() } else { dbCheckpointStatusBeforeSuspension = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status currentDBSession().clear() // clear session as Hibernate with fails with 'org.hibernate.NonUniqueObjectException' once it tries to save a DBFlowCheckpoint upon checkpoint - inMemoryCheckpointStatusBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status + inMemoryCheckpointStatusBeforeSuspension = flowFiber.transientState.checkpoint.status futureFiber.complete(flowFiber) } @@ -701,7 +705,7 @@ class FlowFrameworkTests { } // flow is in hospital assertTrue(flowState is FlowState.Unstarted) - val inMemoryHospitalizedCheckpointStatus = aliceNode.internals.smm.snapshot().first().transientState?.value?.checkpoint?.status + val inMemoryHospitalizedCheckpointStatus = aliceNode.internals.smm.snapshot().first().transientState.checkpoint.status assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, inMemoryHospitalizedCheckpointStatus) aliceNode.database.transaction { val checkpoint = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second @@ -727,13 +731,13 @@ class FlowFrameworkTests { SuspendingFlow.hookAfterCheckpoint = { val flowFiber = this as? FlowStateMachineImpl<*> - flowState = flowFiber!!.transientState!!.value.checkpoint.flowState + flowState = flowFiber!!.transientState.checkpoint.flowState if (firstExecution) { throw HospitalizeFlowException() } else { dbCheckpointStatus = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status - inMemoryCheckpointStatus = flowFiber.transientState!!.value.checkpoint.status + inMemoryCheckpointStatus = flowFiber.transientState.checkpoint.status futureFiber.complete(flowFiber) } @@ -820,7 +824,7 @@ class FlowFrameworkTests { } else { val flowFiber = this as? FlowStateMachineImpl<*> dbCheckpointStatus = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status - inMemoryCheckpointStatus = flowFiber!!.transientState!!.value.checkpoint.status + inMemoryCheckpointStatus = flowFiber!!.transientState.checkpoint.status persistedException = aliceNode.internals.checkpointStorage.getDBCheckpoint(flowFiber.id)!!.exceptionDetails } } From c33720c73d0fd40b8d98bcbb63aafb0d7ed8bd5b Mon Sep 17 00:00:00 2001 From: Joseph Zuniga-Daly <59851625+josephzunigadaly@users.noreply.github.com> Date: Wed, 22 Jul 2020 17:31:59 +0100 Subject: [PATCH 31/48] CORDA-3717: Apply custom serializers to checkpoints (#6392) * CORDA-3717: Apply custom serializers to checkpoints * Remove try/catch to fix TooGenericExceptionCaught detekt rule * Rename exception * Extract method * Put calls to the userSerializer on their own lines to improve readability * Remove unused constructors from exception * Remove unused proxyType field * Give field a descriptive name * Explain why we are looking for two type parameters when we only use one * Tidy up the fetching of types * Use 0 seconds when forcing a flow checkpoint inside test * Add test to check references are restored correctly * Add CheckpointCustomSerializer interface * Wire up the new CheckpointCustomSerializer interface * Use kryo default for abstract classes * Remove unused imports * Remove need for external library in tests * Make file match original to remove from diff * Remove maySkipCheckpoint from calls to sleep * Add newline to end of file * Test custom serializers mapped to interfaces * Test serializer configured with abstract class * Move test into its own package * Rename test * Move flows and serializers into their own source file * Move broken map into its own source file * Delete comment now source file is simpler * Rename class to have a shorter name * Add tests that run the checkpoint serializer directly * Check serialization of final classes * Register as default unless the target class is final * Test PublicKey serializer has not been overridden * Add a broken serializer for EdDSAPublicKey to make test more robust * Split serializer registration into default and non-default registrations. Run registrations at the right time to preserve Cordas own custom serializers. * Check for duplicate custom checkpoint serializers * Add doc comments * Add doc comments to CustomSerializerCheckpointAdaptor * Add test to check duplicate serializers are logged * Do not log the duplicate serializer warning when the duplicate is the same class * Update doc comment for CheckpointCustomSerializer * Sort serializers by classname so we are not registering in an unknown or random order * Add test to serialize a class that references itself * Store custom serializer type in the Kryo stream so we can spot when a different serializer is being used to deserialize * Testing has shown that registering custom serializers as default is more robust when adding new cordapps * Remove new line character * Remove unused imports * Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt * Remove comment * Update comment on exception * Make CustomSerializerCheckpointAdaptor internal * Revert "Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt" This reverts commit b835de79bd21f0048be741e7fc5f0c3088516d2b. * Restore "Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt"" This reverts commit 718873a4e963bad4e327bb200e7bb4de44bc47ad. * Pass the class loader instead of the context * Do less work in test setup * Make the serialization context unique for CustomCheckpointSerializerTest so we get a new Kryo pool for the test * Rebuild the Kryo pool for the given context when we change custom serializers * Rebuild all Kryo pools on serializer change to keep serializer list consistent * Move the custom serializer list into CheckpointSerializationContext to reduce scope from global to a serialization context * Remove unused imports * Make the new checkpointCustomSerializers property default to the empty list * Delegate implementation using kotlin language feature --- .ci/api-current.txt | 4 + .../kotlin/net/corda/core/cordapp/Cordapp.kt | 3 + .../core/internal/cordapp/CordappImpl.kt | 3 + .../SerializationCustomSerializer.kt | 23 ++ .../internal/CheckpointSerializationAPI.kt | 9 + .../kryo/CustomSerializerCheckpointAdaptor.kt | 103 +++++++++ .../kryo/KryoCheckpointSerializer.kt | 54 ++++- .../CustomCheckpointSerializerTest.kt | 99 ++++++++ .../DifficultToSerialize.kt | 27 +++ .../DuplicateSerializerLogTest.kt | 59 +++++ ...cateSerializerLogWithSameSerializerTest.kt | 58 +++++ ...ckNetworkCustomCheckpointSerializerTest.kt | 75 ++++++ .../ReferenceLoopTest.kt | 75 ++++++ .../customcheckpointserializer/TestCorDapp.kt | 214 ++++++++++++++++++ .../kotlin/net/corda/node/internal/Node.kt | 4 +- .../cordapp/JarScanningCordappLoader.kt | 6 + .../node/internal/cordapp/VirtualCordapps.kt | 4 + .../internal/CheckpointSerializationScheme.kt | 6 +- .../InternalSerializationTestHelpers.kt | 6 +- 19 files changed, 826 insertions(+), 6 deletions(-) create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CustomSerializerCheckpointAdaptor.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/CustomCheckpointSerializerTest.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DifficultToSerialize.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/MockNetworkCustomCheckpointSerializerTest.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/ReferenceLoopTest.kt create mode 100644 node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/TestCorDapp.kt diff --git a/.ci/api-current.txt b/.ci/api-current.txt index 64e351610e..10374f09e3 100644 --- a/.ci/api-current.txt +++ b/.ci/api-current.txt @@ -5398,6 +5398,10 @@ public interface net.corda.core.schemas.QueryableState extends net.corda.core.co ## public interface net.corda.core.schemas.StatePersistable ## +public interface net.corda.core.serialization.CheckpointCustomSerializer + public abstract OBJ fromProxy(PROXY) + public abstract PROXY toProxy(OBJ) +## public interface net.corda.core.serialization.ClassWhitelist public abstract boolean hasListed(Class) ## diff --git a/core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt b/core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt index 753e842fe6..1dd153e0ae 100644 --- a/core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt +++ b/core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt @@ -7,6 +7,7 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.FlowLogic import net.corda.core.internal.cordapp.CordappImpl.Companion.UNKNOWN_VALUE import net.corda.core.schemas.MappedSchema +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.SerializationCustomSerializer import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken @@ -29,6 +30,7 @@ import java.net.URL * @property services List of RPC services * @property serializationWhitelists List of Corda plugin registries * @property serializationCustomSerializers List of serializers + * @property checkpointCustomSerializers List of serializers for checkpoints * @property customSchemas List of custom schemas * @property allFlows List of all flow classes * @property jarPath The path to the JAR for this CorDapp @@ -49,6 +51,7 @@ interface Cordapp { val services: List> val serializationWhitelists: List val serializationCustomSerializers: List> + val checkpointCustomSerializers: List> val customSchemas: Set val allFlows: List>> val jarPath: URL diff --git a/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt b/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt index d511ba7860..1c5d69e511 100644 --- a/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt +++ b/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt @@ -9,6 +9,7 @@ import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.notary.NotaryService import net.corda.core.internal.toPath import net.corda.core.schemas.MappedSchema +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.SerializationCustomSerializer import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken @@ -25,6 +26,7 @@ data class CordappImpl( override val services: List>, override val serializationWhitelists: List, override val serializationCustomSerializers: List>, + override val checkpointCustomSerializers: List>, override val customSchemas: Set, override val allFlows: List>>, override val jarPath: URL, @@ -79,6 +81,7 @@ data class CordappImpl( services = emptyList(), serializationWhitelists = emptyList(), serializationCustomSerializers = emptyList(), + checkpointCustomSerializers = emptyList(), customSchemas = emptySet(), jarPath = Paths.get("").toUri().toURL(), info = UNKNOWN_INFO, diff --git a/core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt b/core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt index d0c910b638..ed387f8f94 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt @@ -25,3 +25,26 @@ interface SerializationCustomSerializer { */ fun fromProxy(proxy: PROXY): OBJ } + +/** + * Allows CorDapps to provide custom serializers for classes that do not serialize successfully during a checkpoint. + * In this case, a proxy serializer can be written that implements this interface whose purpose is to move between + * unserializable types and an intermediate representation. + * + * NOTE: Only implement this interface if you have a class that triggers an error during normal checkpoint + * serialization/deserialization. + */ +@KeepForDJVM +interface CheckpointCustomSerializer { + /** + * Should facilitate the conversion of the third party object into the serializable + * local class specified by [PROXY] + */ + fun toProxy(obj: OBJ): PROXY + + /** + * Should facilitate the conversion of the proxy object into a new instance of the + * unserializable type + */ + fun fromProxy(proxy: PROXY): OBJ +} diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt index 98fdcd730d..510986141c 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt @@ -56,6 +56,10 @@ interface CheckpointSerializationContext { * otherwise they appear as new copies of the object. */ val objectReferencesEnabled: Boolean + /** + * User defined custom serializers for use in checkpoint serialization. + */ + val checkpointCustomSerializers: Iterable> /** * Helper method to return a new context based on this context with the property added. @@ -86,6 +90,11 @@ interface CheckpointSerializationContext { * A shallow copy of this context but with the given encoding whitelist. */ fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist): CheckpointSerializationContext + + /** + * A shallow copy of this context but with the given custom serializers. + */ + fun withCheckpointCustomSerializers(checkpointCustomSerializers: Iterable>): CheckpointSerializationContext } /* diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CustomSerializerCheckpointAdaptor.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CustomSerializerCheckpointAdaptor.kt new file mode 100644 index 0000000000..4f3475696b --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CustomSerializerCheckpointAdaptor.kt @@ -0,0 +1,103 @@ +package net.corda.nodeapi.internal.serialization.kryo + +import com.esotericsoftware.kryo.Kryo +import com.esotericsoftware.kryo.Serializer +import com.esotericsoftware.kryo.io.Input +import com.esotericsoftware.kryo.io.Output +import net.corda.core.serialization.CheckpointCustomSerializer +import net.corda.serialization.internal.amqp.CORDAPP_TYPE +import java.lang.reflect.Type +import kotlin.reflect.jvm.javaType +import kotlin.reflect.jvm.jvmErasure + +/** + * Adapts CheckpointCustomSerializer for use in Kryo + */ +internal class CustomSerializerCheckpointAdaptor(private val userSerializer : CheckpointCustomSerializer) : Serializer() { + + /** + * The class name of the serializer we are adapting. + */ + val serializerName: String = userSerializer.javaClass.name + + /** + * The input type of this custom serializer. + */ + val cordappType: Type + + /** + * Check we have access to the types specified on the CheckpointCustomSerializer interface. + * + * Throws UnableToDetermineSerializerTypesException if the types are missing. + */ + init { + val types: List = userSerializer::class + .supertypes + .filter { it.jvmErasure == CheckpointCustomSerializer::class } + .flatMap { it.arguments } + .mapNotNull { it.type?.javaType } + + // We are expecting a cordapp type and a proxy type. + // We will only use the cordapp type in this class + // but we want to check both are present. + val typeParameterCount = 2 + if (types.size != typeParameterCount) { + throw UnableToDetermineSerializerTypesException("Unable to determine serializer parent types") + } + cordappType = types[CORDAPP_TYPE] + } + + /** + * Serialize obj to the Kryo stream. + */ + override fun write(kryo: Kryo, output: Output, obj: OBJ) { + + fun writeToKryo(obj: T) = kryo.writeClassAndObject(output, obj) + + // Write serializer type + writeToKryo(serializerName) + + // Write proxy object + writeToKryo(userSerializer.toProxy(obj)) + } + + /** + * Deserialize an object from the Kryo stream. + */ + override fun read(kryo: Kryo, input: Input, type: Class): OBJ { + + @Suppress("UNCHECKED_CAST") + fun readFromKryo() = kryo.readClassAndObject(input) as T + + // Check the serializer type + checkSerializerType(readFromKryo()) + + // Read the proxy object + return userSerializer.fromProxy(readFromKryo()) + } + + /** + * Throws a `CustomCheckpointSerializersHaveChangedException` if the serializer type in the kryo stream does not match the serializer + * type for this custom serializer. + * + * @param checkpointSerializerType Serializer type from the Kryo stream + */ + private fun checkSerializerType(checkpointSerializerType: String) { + if (checkpointSerializerType != serializerName) + throw CustomCheckpointSerializersHaveChangedException("The custom checkpoint serializers have changed while checkpoints exist. " + + "Please restore the CorDapps to when this checkpoint was created.") + } +} + +/** + * Thrown when the input/output types are missing from the custom serializer. + */ +class UnableToDetermineSerializerTypesException(message: String) : RuntimeException(message) + +/** + * Thrown when the custom serializer is found to be reading data from another type of custom serializer. + * + * This was expected to happen if the user adds or removes CorDapps while checkpoints exist but it turned out that registering serializers + * as default made the system reliable. + */ +class CustomCheckpointSerializersHaveChangedException(message: String) : RuntimeException(message) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoCheckpointSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoCheckpointSerializer.kt index 6a73119ce6..06698d99ad 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoCheckpointSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoCheckpointSerializer.kt @@ -10,12 +10,14 @@ import com.esotericsoftware.kryo.io.Output import com.esotericsoftware.kryo.pool.KryoPool import com.esotericsoftware.kryo.serializers.ClosureSerializer import net.corda.core.internal.uncheckedCast +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.ClassWhitelist import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.CheckpointSerializer import net.corda.core.utilities.ByteSequence +import net.corda.core.utilities.loggerFor import net.corda.serialization.internal.AlwaysAcceptEncodingWhitelist import net.corda.serialization.internal.ByteBufferInputStream import net.corda.serialization.internal.CheckpointSerializationContextImpl @@ -40,10 +42,10 @@ private object AutoCloseableSerialisationDetector : Serializer() } object KryoCheckpointSerializer : CheckpointSerializer { - private val kryoPoolsForContexts = ConcurrentHashMap, KryoPool>() + private val kryoPoolsForContexts = ConcurrentHashMap>>, KryoPool>() private fun getPool(context: CheckpointSerializationContext): KryoPool { - return kryoPoolsForContexts.computeIfAbsent(Pair(context.whitelist, context.deserializationClassLoader)) { + return kryoPoolsForContexts.computeIfAbsent(Triple(context.whitelist, context.deserializationClassLoader, context.checkpointCustomSerializers)) { KryoPool.Builder { val serializer = Fiber.getFiberSerializer(false) as KryoSerializer val classResolver = CordaClassResolver(context).apply { setKryo(serializer.kryo) } @@ -56,12 +58,60 @@ object KryoCheckpointSerializer : CheckpointSerializer { addDefaultSerializer(AutoCloseable::class.java, AutoCloseableSerialisationDetector) register(ClosureSerializer.Closure::class.java, CordaClosureSerializer) classLoader = it.second + + // Add custom serializers + val customSerializers = buildCustomSerializerAdaptors(context) + warnAboutDuplicateSerializers(customSerializers) + val classToSerializer = mapInputClassToCustomSerializer(context.deserializationClassLoader, customSerializers) + addDefaultCustomSerializers(this, classToSerializer) } }.build() } } + /** + * Returns a sorted list of CustomSerializerCheckpointAdaptor based on the custom serializers inside context. + * + * The adaptors are sorted by serializerName which maps to javaClass.name for the serializer class + */ + private fun buildCustomSerializerAdaptors(context: CheckpointSerializationContext) = + context.checkpointCustomSerializers.map { CustomSerializerCheckpointAdaptor(it) }.sortedBy { it.serializerName } + + /** + * Returns a list of pairs where the first element is the input class of the custom serializer and the second element is the + * custom serializer. + */ + private fun mapInputClassToCustomSerializer(classLoader: ClassLoader, customSerializers: Iterable>) = + customSerializers.map { getInputClassForCustomSerializer(classLoader, it) to it } + + /** + * Returns the Class object for the serializers input type. + */ + private fun getInputClassForCustomSerializer(classLoader: ClassLoader, customSerializer: CustomSerializerCheckpointAdaptor<*, *>): Class<*> { + val typeNameWithoutGenerics = customSerializer.cordappType.typeName.substringBefore('<') + return classLoader.loadClass(typeNameWithoutGenerics) + } + + /** + * Emit a warning if two or more custom serializers are found for the same input type. + */ + private fun warnAboutDuplicateSerializers(customSerializers: Iterable>) = + customSerializers + .groupBy({ it.cordappType }, { it.serializerName }) + .filter { (_, serializerNames) -> serializerNames.distinct().size > 1 } + .forEach { (inputType, serializerNames) -> loggerFor().warn("Duplicate custom checkpoint serializer for type $inputType. Serializers: ${serializerNames.joinToString(", ")}") } + + /** + * Register all custom serializers as default, this class + subclass, registrations. + * + * Serializers registered before this will take priority. This needs to run after registrations we want to keep otherwise it may + * replace them. + */ + private fun addDefaultCustomSerializers(kryo: Kryo, classToSerializer: Iterable, CustomSerializerCheckpointAdaptor<*, *>>>) = + classToSerializer + .forEach { (clazz, customSerializer) -> kryo.addDefaultSerializer(clazz, customSerializer) } + private fun CheckpointSerializationContext.kryo(task: Kryo.() -> T): T { return getPool(this).run { kryo -> kryo.context.ensureCapacity(properties.size) diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/CustomCheckpointSerializerTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/CustomCheckpointSerializerTest.kt new file mode 100644 index 0000000000..0efb030fff --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/CustomCheckpointSerializerTest.kt @@ -0,0 +1,99 @@ +package net.corda.node.customcheckpointserializer + +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import net.corda.core.crypto.generateKeyPair +import net.corda.core.serialization.EncodingWhitelist +import net.corda.core.serialization.internal.CheckpointSerializationContext +import net.corda.core.serialization.internal.checkpointDeserialize +import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.coretesting.internal.rigorousMock +import net.corda.serialization.internal.AllWhitelist +import net.corda.serialization.internal.CheckpointSerializationContextImpl +import net.corda.serialization.internal.CordaSerializationEncoding +import net.corda.testing.core.internal.CheckpointSerializationEnvironmentRule +import org.junit.Assert +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized + +@RunWith(Parameterized::class) +class CustomCheckpointSerializerTest(private val compression: CordaSerializationEncoding?) { + companion object { + @Parameterized.Parameters(name = "{0}") + @JvmStatic + fun compression() = arrayOf(null) + CordaSerializationEncoding.values() + } + + @get:Rule + val serializationRule = CheckpointSerializationEnvironmentRule(inheritable = true) + private val context: CheckpointSerializationContext = CheckpointSerializationContextImpl( + deserializationClassLoader = javaClass.classLoader, + whitelist = AllWhitelist, + properties = emptyMap(), + objectReferencesEnabled = true, + encoding = compression, + encodingWhitelist = rigorousMock().also { + if (compression != null) doReturn(true).whenever(it).acceptEncoding(compression) + }, + checkpointCustomSerializers = listOf( + TestCorDapp.TestAbstractClassSerializer(), + TestCorDapp.TestClassSerializer(), + TestCorDapp.TestInterfaceSerializer(), + TestCorDapp.TestFinalClassSerializer(), + TestCorDapp.BrokenPublicKeySerializer() + ) + ) + + @Test(timeout=300_000) + fun `test custom checkpoint serialization`() { + testBrokenMapSerialization(DifficultToSerialize.BrokenMapClass()) + } + + @Test(timeout=300_000) + fun `test custom checkpoint serialization using interface`() { + testBrokenMapSerialization(DifficultToSerialize.BrokenMapInterfaceImpl()) + } + + @Test(timeout=300_000) + fun `test custom checkpoint serialization using abstract class`() { + testBrokenMapSerialization(DifficultToSerialize.BrokenMapAbstractImpl()) + } + + @Test(timeout=300_000) + fun `test custom checkpoint serialization using final class`() { + testBrokenMapSerialization(DifficultToSerialize.BrokenMapFinal()) + } + + @Test(timeout=300_000) + fun `test PublicKey serializer has not been overridden`() { + + val publicKey = generateKeyPair().public + + // Serialize/deserialize + val checkpoint = publicKey.checkpointSerialize(context) + val deserializedCheckpoint = checkpoint.checkpointDeserialize(context) + + // Check the elements are as expected + Assert.assertArrayEquals(publicKey.encoded, deserializedCheckpoint.encoded) + } + + + private fun testBrokenMapSerialization(brokenMap : MutableMap): MutableMap { + // Add elements to the map + brokenMap.putAll(mapOf("key" to "value")) + + // Serialize/deserialize + val checkpoint = brokenMap.checkpointSerialize(context) + val deserializedCheckpoint = checkpoint.checkpointDeserialize(context) + + // Check the elements are as expected + Assert.assertEquals(1, deserializedCheckpoint.size) + Assert.assertEquals("value", deserializedCheckpoint.get("key")) + + // Return map for extra checks + return deserializedCheckpoint + } +} + diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DifficultToSerialize.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DifficultToSerialize.kt new file mode 100644 index 0000000000..f272e71ebf --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DifficultToSerialize.kt @@ -0,0 +1,27 @@ +package net.corda.node.customcheckpointserializer + +import net.corda.core.flows.FlowException + +class DifficultToSerialize { + + // Broken Map + // This map breaks the rules for the put method. Making the normal map serializer fail. + + open class BrokenMapBaseImpl(delegate: MutableMap = mutableMapOf()) : MutableMap by delegate { + override fun put(key: K, value: V): V? = throw FlowException("Broken on purpose") + } + + // A class to test custom serializers applied to implementations + class BrokenMapClass : BrokenMapBaseImpl() + + // An interface and implementation to test custom serializers applied to interface types + interface BrokenMapInterface : MutableMap + class BrokenMapInterfaceImpl : BrokenMapBaseImpl(), BrokenMapInterface + + // An abstract class and implementation to test custom serializers applied to interface types + abstract class BrokenMapAbstract : BrokenMapBaseImpl(), MutableMap + class BrokenMapAbstractImpl : BrokenMapAbstract() + + // A final class + final class BrokenMapFinal: BrokenMapBaseImpl() +} diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt new file mode 100644 index 0000000000..2f87e1005f --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt @@ -0,0 +1,59 @@ +package net.corda.node.customcheckpointserializer + +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.messaging.startFlow +import net.corda.core.serialization.CheckpointCustomSerializer +import net.corda.core.utilities.getOrThrow +import net.corda.node.logging.logFile +import net.corda.testing.driver.driver +import org.assertj.core.api.Assertions +import org.junit.Test +import java.time.Duration + +class DuplicateSerializerLogTest{ + @Test(timeout=300_000) + fun `check duplicate serialisers are logged`() { + driver { + val node = startNode(startInSameProcess = false).getOrThrow() + node.rpc.startFlow(::TestFlow).returnValue.get() + + val text = node.logFile().readLines().filter { it.startsWith("[WARN") } + + // Initial message is correct + Assertions.assertThat(text).anyMatch {it.contains("Duplicate custom checkpoint serializer for type net.corda.node.customcheckpointserializer.DifficultToSerialize\$BrokenMapInterface. Serializers: ")} + // Message mentions TestInterfaceSerializer + Assertions.assertThat(text).anyMatch {it.contains("net.corda.node.customcheckpointserializer.TestCorDapp\$TestInterfaceSerializer")} + // Message mentions DuplicateSerializer + Assertions.assertThat(text).anyMatch {it.contains("net.corda.node.customcheckpointserializer.DuplicateSerializerLogTest\$DuplicateSerializer")} + } + } + + @StartableByRPC + @InitiatingFlow + class TestFlow : FlowLogic>() { + override fun call(): DifficultToSerialize.BrokenMapInterface { + val brokenMap: DifficultToSerialize.BrokenMapInterface = DifficultToSerialize.BrokenMapInterfaceImpl() + brokenMap.putAll(mapOf("test" to "input")) + + sleep(Duration.ofSeconds(0)) + + return brokenMap + } + } + + @Suppress("unused") + class DuplicateSerializer : + CheckpointCustomSerializer, HashMap> { + + override fun toProxy(obj: DifficultToSerialize.BrokenMapInterface): HashMap { + val proxy = HashMap() + return obj.toMap(proxy) + } + override fun fromProxy(proxy: HashMap): DifficultToSerialize.BrokenMapInterface { + return DifficultToSerialize.BrokenMapInterfaceImpl() + .also { it.putAll(proxy) } + } + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt new file mode 100644 index 0000000000..598b1ed401 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt @@ -0,0 +1,58 @@ +package net.corda.node.customcheckpointserializer + +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.messaging.startFlow +import net.corda.core.serialization.CheckpointCustomSerializer +import net.corda.core.serialization.CordaSerializable +import net.corda.core.utilities.getOrThrow +import net.corda.node.logging.logFile +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.internal.enclosedCordapp +import org.assertj.core.api.Assertions +import org.junit.Test +import java.time.Duration + +class DuplicateSerializerLogWithSameSerializerTest { + @Test(timeout=300_000) + fun `check duplicate serialisers are logged not logged for the same class`() { + + // Duplicate the cordapp in this node + driver(DriverParameters(cordappsForAllNodes = listOf(this.enclosedCordapp(), this.enclosedCordapp()))) { + val node = startNode(startInSameProcess = false).getOrThrow() + node.rpc.startFlow(::TestFlow).returnValue.get() + + val text = node.logFile().readLines().filter { it.startsWith("[WARN") } + + // Initial message is not logged + Assertions.assertThat(text) + .anyMatch { !it.contains("Duplicate custom checkpoint serializer for type ") } + // Log does not mention DuplicateSerializerThatShouldNotBeLogged + Assertions.assertThat(text) + .anyMatch { !it.contains("DuplicateSerializerThatShouldNotBeLogged") } + } + } + + @CordaSerializable + class UnusedClass + + @Suppress("unused") + class DuplicateSerializerThatShouldNotBeLogged : CheckpointCustomSerializer { + override fun toProxy(obj: UnusedClass): String = "" + override fun fromProxy(proxy: String): UnusedClass = UnusedClass() + } + + @StartableByRPC + @InitiatingFlow + class TestFlow : FlowLogic() { + override fun call(): UnusedClass { + val unusedClass = UnusedClass() + + sleep(Duration.ofSeconds(0)) + + return unusedClass + } + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/MockNetworkCustomCheckpointSerializerTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/MockNetworkCustomCheckpointSerializerTest.kt new file mode 100644 index 0000000000..5bd60293c4 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/MockNetworkCustomCheckpointSerializerTest.kt @@ -0,0 +1,75 @@ +package net.corda.node.customcheckpointserializer + +import co.paralleluniverse.fibers.Suspendable +import net.corda.testing.node.MockNetwork +import net.corda.testing.node.MockNetworkParameters +import org.assertj.core.api.Assertions +import org.junit.After +import org.junit.Before +import org.junit.Test + +class MockNetworkCustomCheckpointSerializerTest { + private lateinit var mockNetwork: MockNetwork + + @Before + fun setup() { + mockNetwork = MockNetwork(MockNetworkParameters(cordappsForAllNodes = listOf(TestCorDapp.getCorDapp()))) + } + + @After + fun shutdown() { + mockNetwork.stopNodes() + } + + @Test(timeout = 300_000) + fun `flow suspend with custom kryo serializer`() { + val node = mockNetwork.createPartyNode() + val expected = 5 + val actual = node.startFlow(TestCorDapp.TestFlowWithDifficultToSerializeLocalVariable(5)).get() + + Assertions.assertThat(actual).isEqualTo(expected) + } + + @Test(timeout = 300_000) + fun `check references are restored correctly`() { + val node = mockNetwork.createPartyNode() + val expectedReference = DifficultToSerialize.BrokenMapClass() + expectedReference.putAll(mapOf("one" to 1)) + val actualReference = node.startFlow(TestCorDapp.TestFlowCheckingReferencesWork(expectedReference)).get() + + Assertions.assertThat(actualReference).isSameAs(expectedReference) + Assertions.assertThat(actualReference["one"]).isEqualTo(1) + } + + @Test(timeout = 300_000) + @Suspendable + fun `check serialization of interfaces`() { + val node = mockNetwork.createPartyNode() + val result = node.startFlow(TestCorDapp.TestFlowWithDifficultToSerializeLocalVariableAsInterface(5)).get() + Assertions.assertThat(result).isEqualTo(5) + } + + @Test(timeout = 300_000) + @Suspendable + fun `check serialization of abstract classes`() { + val node = mockNetwork.createPartyNode() + val result = node.startFlow(TestCorDapp.TestFlowWithDifficultToSerializeLocalVariableAsAbstract(5)).get() + Assertions.assertThat(result).isEqualTo(5) + } + + @Test(timeout = 300_000) + @Suspendable + fun `check serialization of final classes`() { + val node = mockNetwork.createPartyNode() + val result = node.startFlow(TestCorDapp.TestFlowWithDifficultToSerializeLocalVariableAsFinal(5)).get() + Assertions.assertThat(result).isEqualTo(5) + } + + @Test(timeout = 300_000) + @Suspendable + fun `check PublicKey serializer has not been overridden`() { + val node = mockNetwork.createPartyNode() + val result = node.startFlow(TestCorDapp.TestFlowCheckingPublicKeySerializer()).get() + Assertions.assertThat(result.encoded).isEqualTo(node.info.legalIdentities.first().owningKey.encoded) + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/ReferenceLoopTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/ReferenceLoopTest.kt new file mode 100644 index 0000000000..92a8d396c4 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/ReferenceLoopTest.kt @@ -0,0 +1,75 @@ +package net.corda.node.customcheckpointserializer + +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import net.corda.core.serialization.CheckpointCustomSerializer +import net.corda.core.serialization.EncodingWhitelist +import net.corda.core.serialization.internal.CheckpointSerializationContext +import net.corda.core.serialization.internal.checkpointDeserialize +import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.coretesting.internal.rigorousMock +import net.corda.serialization.internal.AllWhitelist +import net.corda.serialization.internal.CheckpointSerializationContextImpl +import net.corda.serialization.internal.CordaSerializationEncoding +import net.corda.testing.core.internal.CheckpointSerializationEnvironmentRule +import org.junit.Assert +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized + +@RunWith(Parameterized::class) +class ReferenceLoopTest(private val compression: CordaSerializationEncoding?) { + companion object { + @Parameterized.Parameters(name = "{0}") + @JvmStatic + fun compression() = arrayOf(null) + CordaSerializationEncoding.values() + } + + @get:Rule + val serializationRule = CheckpointSerializationEnvironmentRule(inheritable = true) + private val context: CheckpointSerializationContext = CheckpointSerializationContextImpl( + deserializationClassLoader = javaClass.classLoader, + whitelist = AllWhitelist, + properties = emptyMap(), + objectReferencesEnabled = true, + encoding = compression, + encodingWhitelist = rigorousMock() + .also { + if (compression != null) doReturn(true).whenever(it) + .acceptEncoding(compression) + }, + checkpointCustomSerializers = listOf(PersonSerializer())) + + @Test(timeout=300_000) + fun `custom checkpoint serialization with reference loop`() { + val person = Person("Test name") + + val result = person.checkpointSerialize(context).checkpointDeserialize(context) + + Assert.assertEquals("Test name", result.name) + Assert.assertEquals("Test name", result.bestFriend.name) + Assert.assertSame(result, result.bestFriend) + } + + /** + * Test class that will hold a reference to itself + */ + class Person(val name: String, bestFriend: Person? = null) { + val bestFriend: Person = bestFriend ?: this + } + + /** + * Custom serializer for the Person class + */ + @Suppress("unused") + class PersonSerializer : CheckpointCustomSerializer> { + override fun toProxy(obj: Person): Map { + return mapOf("name" to obj.name, "bestFriend" to obj.bestFriend) + } + + override fun fromProxy(proxy: Map): Person { + return Person(proxy["name"] as String, proxy["bestFriend"] as Person?) + } + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/TestCorDapp.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/TestCorDapp.kt new file mode 100644 index 0000000000..1d3e929dde --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/TestCorDapp.kt @@ -0,0 +1,214 @@ +package net.corda.node.customcheckpointserializer + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.FlowException +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.StartableByRPC +import net.corda.core.serialization.CheckpointCustomSerializer +import net.corda.testing.node.internal.CustomCordapp +import net.corda.testing.node.internal.enclosedCordapp +import net.i2p.crypto.eddsa.EdDSAPublicKey +import org.assertj.core.api.Assertions +import java.security.PublicKey +import java.time.Duration + +/** + * Contains all the flows and custom serializers for testing custom checkpoint serializers + */ +class TestCorDapp { + + companion object { + fun getCorDapp(): CustomCordapp = enclosedCordapp() + } + + // Flows + @StartableByRPC + class TestFlowWithDifficultToSerializeLocalVariableAsAbstract(private val purchase: Int) : FlowLogic() { + @Suspendable + override fun call(): Int { + + // This object is difficult to serialize with Kryo + val difficultToSerialize: DifficultToSerialize.BrokenMapAbstract = DifficultToSerialize.BrokenMapAbstractImpl() + difficultToSerialize.putAll(mapOf("foo" to purchase)) + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Return value from deserialized object + return difficultToSerialize["foo"] ?: 0 + } + } + + @StartableByRPC + class TestFlowWithDifficultToSerializeLocalVariableAsFinal(private val purchase: Int) : FlowLogic() { + @Suspendable + override fun call(): Int { + + // This object is difficult to serialize with Kryo + val difficultToSerialize: DifficultToSerialize.BrokenMapFinal = DifficultToSerialize.BrokenMapFinal() + difficultToSerialize.putAll(mapOf("foo" to purchase)) + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Return value from deserialized object + return difficultToSerialize["foo"] ?: 0 + } + } + + @StartableByRPC + class TestFlowWithDifficultToSerializeLocalVariableAsInterface(private val purchase: Int) : FlowLogic() { + @Suspendable + override fun call(): Int { + + // This object is difficult to serialize with Kryo + val difficultToSerialize: DifficultToSerialize.BrokenMapInterface = DifficultToSerialize.BrokenMapInterfaceImpl() + difficultToSerialize.putAll(mapOf("foo" to purchase)) + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Return value from deserialized object + return difficultToSerialize["foo"] ?: 0 + } + } + + @StartableByRPC + class TestFlowWithDifficultToSerializeLocalVariable(private val purchase: Int) : FlowLogic() { + @Suspendable + override fun call(): Int { + + // This object is difficult to serialize with Kryo + val difficultToSerialize: DifficultToSerialize.BrokenMapClass = DifficultToSerialize.BrokenMapClass() + difficultToSerialize.putAll(mapOf("foo" to purchase)) + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Return value from deserialized object + return difficultToSerialize["foo"] ?: 0 + } + } + + @StartableByRPC + class TestFlowCheckingReferencesWork(private val reference: DifficultToSerialize.BrokenMapClass) : + FlowLogic>() { + + private val referenceField = reference + @Suspendable + override fun call(): DifficultToSerialize.BrokenMapClass { + + val ref = referenceField + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Check all objects refer to same object + Assertions.assertThat(reference).isSameAs(referenceField) + Assertions.assertThat(referenceField).isSameAs(ref) + + // Return deserialized object + return ref + } + } + + + @StartableByRPC + class TestFlowCheckingPublicKeySerializer : + FlowLogic() { + + @Suspendable + override fun call(): PublicKey { + val ref = ourIdentity.owningKey + + // Force a checkpoint + sleep(Duration.ofSeconds(0)) + + // Return deserialized object + return ref + } + } + + // Custom serializers + + @Suppress("unused") + class TestInterfaceSerializer : + CheckpointCustomSerializer, HashMap> { + + override fun toProxy(obj: DifficultToSerialize.BrokenMapInterface): HashMap { + val proxy = HashMap() + return obj.toMap(proxy) + } + override fun fromProxy(proxy: HashMap): DifficultToSerialize.BrokenMapInterface { + return DifficultToSerialize.BrokenMapInterfaceImpl() + .also { it.putAll(proxy) } + } + } + + @Suppress("unused") + class TestClassSerializer : + CheckpointCustomSerializer, HashMap> { + + override fun toProxy(obj: DifficultToSerialize.BrokenMapClass): HashMap { + val proxy = HashMap() + return obj.toMap(proxy) + } + override fun fromProxy(proxy: HashMap): DifficultToSerialize.BrokenMapClass { + return DifficultToSerialize.BrokenMapClass() + .also { it.putAll(proxy) } + } + } + + @Suppress("unused") + class TestAbstractClassSerializer : + CheckpointCustomSerializer, HashMap> { + + override fun toProxy(obj: DifficultToSerialize.BrokenMapAbstract): HashMap { + val proxy = HashMap() + return obj.toMap(proxy) + } + override fun fromProxy(proxy: HashMap): DifficultToSerialize.BrokenMapAbstract { + return DifficultToSerialize.BrokenMapAbstractImpl() + .also { it.putAll(proxy) } + } + } + + @Suppress("unused") + class TestFinalClassSerializer : + CheckpointCustomSerializer, HashMap> { + + override fun toProxy(obj: DifficultToSerialize.BrokenMapFinal): HashMap { + val proxy = HashMap() + return obj.toMap(proxy) + } + override fun fromProxy(proxy: HashMap): DifficultToSerialize.BrokenMapFinal { + return DifficultToSerialize.BrokenMapFinal() + .also { it.putAll(proxy) } + } + } + + @Suppress("unused") + class BrokenPublicKeySerializer : + CheckpointCustomSerializer { + override fun toProxy(obj: PublicKey): String { + throw FlowException("Broken on purpose") + } + + override fun fromProxy(proxy: String): PublicKey { + throw FlowException("Broken on purpose") + } + } + + @Suppress("unused") + class BrokenEdDSAPublicKeySerializer : + CheckpointCustomSerializer { + override fun toProxy(obj: EdDSAPublicKey): String { + throw FlowException("Broken on purpose") + } + + override fun fromProxy(proxy: String): EdDSAPublicKey { + throw FlowException("Broken on purpose") + } + } + +} diff --git a/node/src/main/kotlin/net/corda/node/internal/Node.kt b/node/src/main/kotlin/net/corda/node/internal/Node.kt index 77d95abacd..0c2552c37b 100644 --- a/node/src/main/kotlin/net/corda/node/internal/Node.kt +++ b/node/src/main/kotlin/net/corda/node/internal/Node.kt @@ -644,8 +644,8 @@ open class Node(configuration: NodeConfiguration, storageContext = AMQP_STORAGE_CONTEXT.withClassLoader(classloader), checkpointSerializer = KryoCheckpointSerializer, - checkpointContext = KRYO_CHECKPOINT_CONTEXT.withClassLoader(classloader) - ) + checkpointContext = KRYO_CHECKPOINT_CONTEXT.withClassLoader(classloader).withCheckpointCustomSerializers(cordappLoader.cordapps.flatMap { it.checkpointCustomSerializers }) + ) } /** Starts a blocking event loop for message dispatch. */ diff --git a/node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt b/node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt index 97a5672846..bb2fce1a58 100644 --- a/node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt +++ b/node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt @@ -18,6 +18,7 @@ import net.corda.core.internal.notary.NotaryService import net.corda.core.internal.notary.SinglePartyNotaryService import net.corda.core.node.services.CordaService import net.corda.core.schemas.MappedSchema +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.SerializationCustomSerializer import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken @@ -185,6 +186,7 @@ class JarScanningCordappLoader private constructor(private val cordappJarPaths: findServices(this), findWhitelists(url), findSerializers(this), + findCheckpointSerializers(this), findCustomSchemas(this), findAllFlows(this), url.url, @@ -334,6 +336,10 @@ class JarScanningCordappLoader private constructor(private val cordappJarPaths: return scanResult.getClassesImplementingWithClassVersionCheck(SerializationCustomSerializer::class) } + private fun findCheckpointSerializers(scanResult: RestrictedScanResult): List> { + return scanResult.getClassesImplementingWithClassVersionCheck(CheckpointCustomSerializer::class) + } + private fun findCustomSchemas(scanResult: RestrictedScanResult): Set { return scanResult.getClassesWithSuperclass(MappedSchema::class).instances().toSet() } diff --git a/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt b/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt index 3f9e3b85f9..5ad5add351 100644 --- a/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt +++ b/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt @@ -32,6 +32,7 @@ internal object VirtualCordapp { services = listOf(), serializationWhitelists = listOf(), serializationCustomSerializers = listOf(), + checkpointCustomSerializers = listOf(), customSchemas = setOf(), info = Cordapp.Info.Default("corda-core", versionInfo.vendor, versionInfo.releaseVersion, "Open Source (Apache 2)"), allFlows = listOf(), @@ -55,6 +56,7 @@ internal object VirtualCordapp { services = listOf(), serializationWhitelists = listOf(), serializationCustomSerializers = listOf(), + checkpointCustomSerializers = listOf(), customSchemas = setOf(NodeNotarySchemaV1), info = Cordapp.Info.Default("corda-notary", versionInfo.vendor, versionInfo.releaseVersion, "Open Source (Apache 2)"), allFlows = listOf(), @@ -78,6 +80,7 @@ internal object VirtualCordapp { services = listOf(), serializationWhitelists = listOf(), serializationCustomSerializers = listOf(), + checkpointCustomSerializers = listOf(), customSchemas = setOf(RaftNotarySchemaV1), info = Cordapp.Info.Default("corda-notary-raft", versionInfo.vendor, versionInfo.releaseVersion, "Open Source (Apache 2)"), allFlows = listOf(), @@ -101,6 +104,7 @@ internal object VirtualCordapp { services = listOf(), serializationWhitelists = listOf(), serializationCustomSerializers = listOf(), + checkpointCustomSerializers = listOf(), customSchemas = setOf(BFTSmartNotarySchemaV1), info = Cordapp.Info.Default("corda-notary-bft-smart", versionInfo.vendor, versionInfo.releaseVersion, "Open Source (Apache 2)"), allFlows = listOf(), diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt index b6c43ddc6d..f037e2dfbb 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt @@ -1,6 +1,7 @@ package net.corda.serialization.internal import net.corda.core.KeepForDJVM +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.ClassWhitelist import net.corda.core.serialization.EncodingWhitelist import net.corda.core.serialization.SerializationEncoding @@ -13,7 +14,8 @@ data class CheckpointSerializationContextImpl @JvmOverloads constructor( override val properties: Map, override val objectReferencesEnabled: Boolean, override val encoding: SerializationEncoding?, - override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist) : CheckpointSerializationContext { + override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist, + override val checkpointCustomSerializers: Iterable> = emptyList()) : CheckpointSerializationContext { override fun withProperty(property: Any, value: Any): CheckpointSerializationContext { return copy(properties = properties + (property to value)) } @@ -34,4 +36,6 @@ data class CheckpointSerializationContextImpl @JvmOverloads constructor( override fun withEncoding(encoding: SerializationEncoding?) = copy(encoding = encoding) override fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist) = copy(encodingWhitelist = encodingWhitelist) + override fun withCheckpointCustomSerializers(checkpointCustomSerializers : Iterable>) + = copy(checkpointCustomSerializers = checkpointCustomSerializers) } \ No newline at end of file diff --git a/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/InternalSerializationTestHelpers.kt b/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/InternalSerializationTestHelpers.kt index 61bf91aac9..116016b991 100644 --- a/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/InternalSerializationTestHelpers.kt +++ b/testing/core-test-utils/src/main/kotlin/net/corda/coretesting/internal/InternalSerializationTestHelpers.kt @@ -2,6 +2,7 @@ package net.corda.coretesting.internal import net.corda.nodeapi.internal.rpc.client.AMQPClientSerializationScheme import net.corda.core.internal.createInstancesOfClassesImplementing +import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.SerializationCustomSerializer import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.internal.SerializationEnvironment @@ -25,8 +26,11 @@ fun createTestSerializationEnv(): SerializationEnvironment { } fun createTestSerializationEnv(classLoader: ClassLoader?): SerializationEnvironment { + var customCheckpointSerializers: Set> = emptySet() val (clientSerializationScheme, serverSerializationScheme) = if (classLoader != null) { val customSerializers = createInstancesOfClassesImplementing(classLoader, SerializationCustomSerializer::class.java) + customCheckpointSerializers = createInstancesOfClassesImplementing(classLoader, CheckpointCustomSerializer::class.java) + val serializationWhitelists = ServiceLoader.load(SerializationWhitelist::class.java, classLoader).toSet() Pair(AMQPClientSerializationScheme(customSerializers, serializationWhitelists), @@ -44,7 +48,7 @@ fun createTestSerializationEnv(classLoader: ClassLoader?): SerializationEnvironm AMQP_RPC_SERVER_CONTEXT, AMQP_RPC_CLIENT_CONTEXT, AMQP_STORAGE_CONTEXT, - KRYO_CHECKPOINT_CONTEXT, + KRYO_CHECKPOINT_CONTEXT.withCheckpointCustomSerializers(customCheckpointSerializers), KryoCheckpointSerializer ) } From 0842ea26b717508b8078c8f08142f7150f5a598f Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Wed, 22 Jul 2020 19:43:08 +0100 Subject: [PATCH 32/48] NOTICK: Backport Jenkins configuration for nightly builds to release 4.4 --- .ci/dev/publish-branch/Jenkinsfile.nightly | 37 +++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly index b25549729e..485124ab66 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.nightly +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -1,11 +1,27 @@ #!groovy +/** + * Jenkins pipeline to build Corda OS nightly snapshots + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/* +** calculate the stage for NexusIQ evaluation +** * build for snapshots +*/ +def nexusIqStage = "build" + pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() @@ -28,6 +44,25 @@ pipeline { } stages { + stage('Sonatype Check') { + steps { + sh "./gradlew --no-daemon clean jar" + script { + sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" + } + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: manualApplication(nexusAppId), + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: nexusIqStage + ) + } + } + stage('Publish to Artifactory') { steps { rtServer ( From f60f06a85fc81febb65b8487a9bcd78df31fbbf5 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Wed, 22 Jul 2020 19:54:41 +0100 Subject: [PATCH 33/48] INFRA-508: Change appID in sonatype stage for Corda and Corda Enterprise --- .ci/dev/publish-branch/Jenkinsfile.nightly | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly index 485124ab66..471825bd2d 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.nightly +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -49,14 +49,15 @@ pipeline { sh "./gradlew --no-daemon clean jar" script { sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" - def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim() def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() def artifactId = 'corda' nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" } nexusPolicyEvaluation ( failBuildOnNetworkError: false, - iqApplication: manualApplication(nexusAppId), + iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], iqStage: nexusIqStage ) From 14279a30cc7b720031a2378124ac3f9800177214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Thu, 23 Jul 2020 10:36:48 +0100 Subject: [PATCH 34/48] INFRA-508: NexusIQ stage for GA releases updated to `release` (#6508) --- .ci/dev/regression/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 4678bbda6c..b079bab2e1 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -30,7 +30,7 @@ if (isReleaseTag) { switch (env.TAG_NAME) { case ~/.*-RC\d+(-.*)?/: nexusIqStage = "stage-release"; break; case ~/.*-HC\d+(-.*)?/: nexusIqStage = "stage-release"; break; - default: nexusIqStage = "operate" + default: nexusIqStage = "release" } } From 2e6bd97fe951c360bb574b7ca305bd71f818997b Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Thu, 23 Jul 2020 10:46:32 +0100 Subject: [PATCH 35/48] Updates * INFRA-508: NexusIQ stage for GA releases updated to `release` * do not truncate stdio when collecting JUnit tests --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 4 ++-- .ci/dev/regression/Jenkinsfile | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index b692a4d736..af53b9fa84 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -30,7 +30,7 @@ if (isReleaseTag) { switch (env.TAG_NAME) { case ~/.*-RC\d+(-.*)?/: nexusIqStage = "stage-release"; break; case ~/.*-HC\d+(-.*)?/: nexusIqStage = "stage-release"; break; - default: nexusIqStage = "operate" + default: nexusIqStage = "release" } } @@ -165,7 +165,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 7ba2ef2d6a..b9a952c1fb 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -174,7 +174,6 @@ pipeline { } } - post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false From 05c2428f2fb76ba73ac195821833d596deca677e Mon Sep 17 00:00:00 2001 From: Ross Nicoll Date: Thu, 23 Jul 2020 15:17:59 +0100 Subject: [PATCH 36/48] NOTICK Add more detail on signature build failures (#6510) Add details of the signature provider and key algorithm if `InvalidKeyException` is thrown when constructing a `ContentSigner`, in order to be able to usefully diagnose incorrect signature providers or similar errors. --- .../internal/crypto/ContentSignerBuilder.kt | 19 +++++++---- .../crypto/ContentSignerBuilderTest.kt | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) create mode 100644 node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilderTest.kt diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilder.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilder.kt index ac60f55764..bbee9e5d2a 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilder.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilder.kt @@ -6,6 +6,7 @@ import net.corda.core.crypto.internal.Instances import org.bouncycastle.asn1.x509.AlgorithmIdentifier import org.bouncycastle.operator.ContentSigner import java.io.OutputStream +import java.security.InvalidKeyException import java.security.PrivateKey import java.security.Provider import java.security.SecureRandom @@ -24,14 +25,18 @@ object ContentSignerBuilder { else Signature.getInstance(signatureScheme.signatureName, provider) - val sig = signatureInstance.apply { - // TODO special handling for Sphincs due to a known BouncyCastle's Sphincs bug we reported. - // It is fixed in BC 161b12, so consider updating the below if-statement after updating BouncyCastle. - if (random != null && signatureScheme != SPHINCS256_SHA256) { - initSign(privateKey, random) - } else { - initSign(privateKey) + val sig = try { + signatureInstance.apply { + // TODO special handling for Sphincs due to a known BouncyCastle's Sphincs bug we reported. + // It is fixed in BC 161b12, so consider updating the below if-statement after updating BouncyCastle. + if (random != null && signatureScheme != SPHINCS256_SHA256) { + initSign(privateKey, random) + } else { + initSign(privateKey) + } } + } catch(ex: InvalidKeyException) { + throw InvalidKeyException("Incorrect key type ${privateKey.algorithm} for signature scheme ${signatureInstance.algorithm}", ex) } return object : ContentSigner { private val stream = SignatureOutputStream(sig, optimised) diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilderTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilderTest.kt new file mode 100644 index 0000000000..6920c78093 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/ContentSignerBuilderTest.kt @@ -0,0 +1,33 @@ +package net.corda.nodeapi.internal.crypto + +import net.corda.core.crypto.Crypto +import org.assertj.core.api.Assertions.assertThatExceptionOfType +import org.junit.Test +import java.math.BigInteger +import java.security.InvalidKeyException + +class ContentSignerBuilderTest { + companion object { + private const val entropy = "20200723" + } + + @Test(timeout = 300_000) + fun `should build content signer for valid eddsa key`() { + val signatureScheme = Crypto.EDDSA_ED25519_SHA512 + val provider = Crypto.findProvider(signatureScheme.providerName) + val issuerKeyPair = Crypto.deriveKeyPairFromEntropy(signatureScheme, BigInteger(entropy)) + ContentSignerBuilder.build(signatureScheme, issuerKeyPair.private, provider) + } + + @Test(timeout = 300_000) + fun `should fail to build content signer for incorrect key type`() { + val signatureScheme = Crypto.EDDSA_ED25519_SHA512 + val provider = Crypto.findProvider(signatureScheme.providerName) + val issuerKeyPair = Crypto.deriveKeyPairFromEntropy(Crypto.ECDSA_SECP256R1_SHA256, BigInteger(entropy)) + assertThatExceptionOfType(InvalidKeyException::class.java) + .isThrownBy { + ContentSignerBuilder.build(signatureScheme, issuerKeyPair.private, provider) + } + .withMessage("Incorrect key type EC for signature scheme NONEwithEdDSA") + } +} \ No newline at end of file From 4acf41ea3d3d35825fa461b54e695bcc7316912d Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Thu, 23 Jul 2020 16:35:34 +0100 Subject: [PATCH 37/48] INFRA-477: Start nodes in parallel when possible (#6460) Co-authored-by: Ross Nicoll --- .../FlowExternalOperationInJavaTest.java | 48 ++++---- .../flows/FlowExternalAsyncOperationTest.kt | 61 ++++++---- .../FlowExternalOperationStartFlowTest.kt | 13 ++- .../flows/FlowExternalOperationTest.kt | 55 ++++++--- .../corda/coretests/flows/FlowIsKilledTest.kt | 15 ++- .../corda/coretests/flows/FlowSleepTest.kt | 7 +- .../endurance/NodesStartStopSingleVmTests.kt | 6 +- .../node/logging/IssueCashLoggingTests.kt | 7 +- .../StateMachineErrorHandlingTest.kt | 53 ++++++--- .../StateMachineFinalityErrorHandlingTest.kt | 12 +- .../StateMachineFlowInitErrorHandlingTest.kt | 30 ++--- .../StateMachineGeneralErrorHandlingTest.kt | 30 ++--- .../StateMachineKillFlowErrorHandlingTest.kt | 3 +- .../StateMachineSubFlowErrorHandlingTest.kt | 12 +- .../corda/node/flows/FlowEntityManagerTest.kt | 7 +- .../net/corda/node/flows/FlowOverrideTests.kt | 44 ++++---- .../net/corda/node/flows/FlowRetryTest.kt | 25 +++-- .../net/corda/node/flows/KillFlowTest.kt | 28 +++-- .../FlowsDrainingModeContentionTest.kt | 7 +- .../draining/P2PFlowsDrainingModeTest.kt | 20 +++- .../services/rpc/RpcExceptionHandlingTest.kt | 19 +++- .../services/statemachine/FlowHospitalTest.kt | 105 +++++++++--------- .../vault/VaultObserverExceptionTest.kt | 29 +++-- .../statemachine/FlowMetadataRecordingTest.kt | 37 ++++-- .../shell/InteractiveShellIntegrationTest.kt | 7 +- 25 files changed, 397 insertions(+), 283 deletions(-) diff --git a/core-tests/src/test/java/net/corda/coretests/flows/FlowExternalOperationInJavaTest.java b/core-tests/src/test/java/net/corda/coretests/flows/FlowExternalOperationInJavaTest.java index ec41941360..4ed32e0448 100644 --- a/core-tests/src/test/java/net/corda/coretests/flows/FlowExternalOperationInJavaTest.java +++ b/core-tests/src/test/java/net/corda/coretests/flows/FlowExternalOperationInJavaTest.java @@ -8,6 +8,7 @@ import net.corda.core.identity.Party; import net.corda.core.utilities.KotlinUtilsKt; import net.corda.testing.core.TestConstants; import net.corda.testing.core.TestUtils; +import net.corda.testing.driver.DriverDSL; import net.corda.testing.driver.DriverParameters; import net.corda.testing.driver.NodeHandle; import net.corda.testing.driver.NodeParameters; @@ -19,8 +20,11 @@ import org.slf4j.LoggerFactory; import java.io.Serializable; import java.time.Duration; import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.function.BiFunction; +import java.util.stream.Collectors; import static net.corda.testing.driver.Driver.driver; @@ -29,14 +33,9 @@ public class FlowExternalOperationInJavaTest extends AbstractFlowExternalOperati @Test public void awaitFlowExternalOperationInJava() { driver(new DriverParameters().withStartNodesInProcess(true), driver -> { - NodeHandle alice = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.ALICE_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); - NodeHandle bob = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.BOB_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); + List aliceAndBob = aliceAndBob(driver); + NodeHandle alice = aliceAndBob.get(0); + NodeHandle bob = aliceAndBob.get(1); return KotlinUtilsKt.getOrThrow(alice.getRpc().startFlowDynamic( FlowWithExternalOperationInJava.class, TestUtils.singleIdentity(bob.getNodeInfo()) @@ -47,14 +46,9 @@ public class FlowExternalOperationInJavaTest extends AbstractFlowExternalOperati @Test public void awaitFlowExternalAsyncOperationInJava() { driver(new DriverParameters().withStartNodesInProcess(true), driver -> { - NodeHandle alice = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.ALICE_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); - NodeHandle bob = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.BOB_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); + List aliceAndBob = aliceAndBob(driver); + NodeHandle alice = aliceAndBob.get(0); + NodeHandle bob = aliceAndBob.get(1); return KotlinUtilsKt.getOrThrow(alice.getRpc().startFlowDynamic( FlowWithExternalAsyncOperationInJava.class, TestUtils.singleIdentity(bob.getNodeInfo()) @@ -65,14 +59,9 @@ public class FlowExternalOperationInJavaTest extends AbstractFlowExternalOperati @Test public void awaitFlowExternalOperationInJavaCanBeRetried() { driver(new DriverParameters().withStartNodesInProcess(true), driver -> { - NodeHandle alice = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.ALICE_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); - NodeHandle bob = KotlinUtilsKt.getOrThrow( - driver.startNode(new NodeParameters().withProvidedName(TestConstants.BOB_NAME)), - Duration.of(1, ChronoUnit.MINUTES) - ); + List aliceAndBob = aliceAndBob(driver); + NodeHandle alice = aliceAndBob.get(0); + NodeHandle bob = aliceAndBob.get(1); KotlinUtilsKt.getOrThrow(alice.getRpc().startFlowDynamic( FlowWithExternalOperationThatGetsRetriedInJava.class, TestUtils.singleIdentity(bob.getNodeInfo()) @@ -190,4 +179,15 @@ public class FlowExternalOperationInJavaTest extends AbstractFlowExternalOperati return operation.apply(futureService, deduplicationId); } } + + private List aliceAndBob(DriverDSL driver) { + return Arrays.asList(TestConstants.ALICE_NAME, TestConstants.BOB_NAME) + .stream() + .map(nm -> driver.startNode(new NodeParameters().withProvidedName(nm))) + .collect(Collectors.toList()) + .stream() + .map(future -> KotlinUtilsKt.getOrThrow(future, + Duration.of(1, ChronoUnit.MINUTES))) + .collect(Collectors.toList()); + } } \ No newline at end of file diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt index 6b6cfb3891..13767431a0 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt @@ -4,6 +4,7 @@ import co.paralleluniverse.fibers.Suspendable import net.corda.core.flows.HospitalizeFlowException import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.minutes @@ -24,8 +25,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::FlowWithExternalAsyncOperation, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) @@ -35,8 +38,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that checks deduplicationId is not rerun when flow is retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationWithDeduplication, @@ -50,8 +55,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation propagates exception to calling flow`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationPropagatesException, @@ -66,8 +73,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation exception can be caught in flow`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val result = alice.rpc.startFlow( ::FlowWithExternalAsyncOperationThatThrowsExceptionAndCaughtInFlow, bob.nodeInfo.singleIdentity() @@ -80,8 +89,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation with exception that hospital keeps for observation does not fail`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationPropagatesException, @@ -96,8 +107,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation with exception that hospital discharges is retried and runs the future again`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationPropagatesException, @@ -112,8 +125,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that throws exception rather than completing future exceptionally fails with internal exception`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow(::FlowWithExternalAsyncOperationUnhandledException, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) @@ -125,8 +140,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that passes serviceHub into process can be retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationThatPassesInServiceHubCanRetry, @@ -140,8 +157,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that accesses serviceHub from flow directly will fail when retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalAsyncOperationThatDirectlyAccessesServiceHubFailsRetry, @@ -155,8 +174,10 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `starting multiple futures and joining on their results`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::FlowThatStartsMultipleFuturesAndJoins, bob.nodeInfo.singleIdentity()).returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationStartFlowTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationStartFlowTest.kt index bb80313c86..ded1074520 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationStartFlowTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationStartFlowTest.kt @@ -3,6 +3,7 @@ package net.corda.coretests.flows import co.paralleluniverse.fibers.Suspendable import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.minutes @@ -18,8 +19,10 @@ class FlowExternalOperationStartFlowTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `starting a flow inside of a flow that starts a future will succeed`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::FlowThatStartsAnotherFlowInAnExternalOperation, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) @@ -29,8 +32,10 @@ class FlowExternalOperationStartFlowTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `multiple flows can be started and their futures joined from inside a flow`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::ForkJoinFlows, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationTest.kt index 8e1a96aecd..f7dea8bcda 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalOperationTest.kt @@ -5,6 +5,7 @@ import net.corda.core.flows.FlowLogic import net.corda.core.flows.HospitalizeFlowException import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.internal.packageName import net.corda.core.messaging.startFlow import net.corda.core.node.services.queryBy @@ -29,8 +30,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::FlowWithExternalOperation, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) @@ -40,8 +43,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation that checks deduplicationId is not rerun when flow is retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalOperationWithDeduplication, @@ -55,8 +60,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation propagates exception to calling flow`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalOperationPropagatesException, @@ -71,8 +78,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation exception can be caught in flow`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.startFlow(::FlowWithExternalOperationThatThrowsExceptionAndCaughtInFlow, bob.nodeInfo.singleIdentity()) .returnValue.getOrThrow(1.minutes) assertHospitalCounters(0, 0) @@ -82,8 +91,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation with exception that hospital keeps for observation does not fail`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalOperationPropagatesException, @@ -98,8 +109,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation with exception that hospital discharges is retried and runs the external operation again`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalOperationPropagatesException, @@ -114,8 +127,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that passes serviceHub into process can be retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() blockUntilFlowKeptInForObservation { alice.rpc.startFlow( ::FlowWithExternalOperationThatPassesInServiceHubCanRetry, @@ -129,8 +144,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external async operation that accesses serviceHub from flow directly will fail when retried`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() assertFailsWith { alice.rpc.startFlow( ::FlowWithExternalOperationThatDirectlyAccessesServiceHubFailsRetry, @@ -199,8 +216,10 @@ class FlowExternalOperationTest : AbstractFlowExternalOperationTest() { @Test(timeout = 300_000) fun `external operation can be retried when an error occurs inside of database transaction`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val success = alice.rpc.startFlow( ::FlowWithExternalOperationThatErrorsInsideOfDatabaseTransaction, bob.nodeInfo.singleIdentity() diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt index b3cee7c1ca..35a1639714 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt @@ -10,6 +10,7 @@ import net.corda.core.flows.StartableByRPC import net.corda.core.flows.StateMachineRunId import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.minutes @@ -56,9 +57,10 @@ class FlowIsKilledTest { @Test(timeout = 300_000) fun `manually handled killed flows propagate error to counter parties`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() - val charlie = startNode(providedName = CHARLIE_NAME).getOrThrow() + val (alice, bob, charlie) = listOf(ALICE_NAME, BOB_NAME, CHARLIE_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.let { rpc -> val handle = rpc.startFlow( ::AFlowThatWantsToDieAndKillsItsFriends, @@ -85,8 +87,11 @@ class FlowIsKilledTest { @Test(timeout = 300_000) fun `a manually killed initiated flow will propagate the killed error to the initiator and its counter parties`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() + val handle = alice.rpc.startFlow( ::AFlowThatGetsMurderedByItsFriend, bob.nodeInfo.singleIdentity() diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowSleepTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowSleepTest.kt index 0c4a197d58..194c78bee8 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowSleepTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowSleepTest.kt @@ -7,6 +7,7 @@ import net.corda.core.flows.InitiatedBy import net.corda.core.flows.InitiatingFlow import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.minutes @@ -53,8 +54,10 @@ class FlowSleepTest { fun `flow can sleep and perform other suspending functions`() { // ensures that events received while the flow is sleeping are not processed driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val (start, finish) = alice.rpc.startFlow( ::SleepAndInteractWithPartyFlow, bob.nodeInfo.singleIdentity() diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/endurance/NodesStartStopSingleVmTests.kt b/node/src/integration-test-slow/kotlin/net/corda/node/endurance/NodesStartStopSingleVmTests.kt index dabf8379b6..a6e5b81c06 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/endurance/NodesStartStopSingleVmTests.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/endurance/NodesStartStopSingleVmTests.kt @@ -23,8 +23,10 @@ class NodesStartStopSingleVmTests(@Suppress("unused") private val iteration: Int @Test(timeout = 300_000) fun nodesStartStop() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - startNode(providedName = ALICE_NAME).getOrThrow() - startNode(providedName = BOB_NAME).getOrThrow() + val alice = startNode(providedName = ALICE_NAME) + val bob = startNode(providedName = BOB_NAME) + alice.getOrThrow() + bob.getOrThrow() } } } \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/logging/IssueCashLoggingTests.kt b/node/src/integration-test-slow/kotlin/net/corda/node/logging/IssueCashLoggingTests.kt index f07e564913..2700858086 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/logging/IssueCashLoggingTests.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/logging/IssueCashLoggingTests.kt @@ -1,5 +1,6 @@ package net.corda.node.logging +import net.corda.core.internal.concurrent.transpose import net.corda.core.internal.div import net.corda.core.messaging.startFlow import net.corda.core.utilities.OpaqueBytes @@ -22,8 +23,10 @@ class IssueCashLoggingTests { fun `issuing and sending cash as payment do not result in duplicate insertion warnings`() { val user = User("mark", "dadada", setOf(all())) driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS)) { - val nodeA = startNode(rpcUsers = listOf(user)).getOrThrow() - val nodeB = startNode().getOrThrow() + val (nodeA, nodeB) = listOf(startNode(rpcUsers = listOf(user)), + startNode()) + .transpose() + .getOrThrow() val amount = 1.DOLLARS val ref = OpaqueBytes.of(0) diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt index 8233cc79df..37d6cce6ac 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt @@ -62,30 +62,49 @@ abstract class StateMachineErrorHandlingTest { } } - internal fun DriverDSL.createBytemanNode( - providedName: CordaX500Name, + internal fun DriverDSL.createBytemanNode(nodeProvidedName: CordaX500Name): Pair { + val port = nextPort() + val bytemanNodeHandle = (this as InternalDriverDSL).startNode( + NodeParameters( + providedName = nodeProvidedName, + rpcUsers = listOf(rpcUser) + ), + bytemanPort = port + ) + return bytemanNodeHandle.getOrThrow() to port + } + + internal fun DriverDSL.createNode(nodeProvidedName: CordaX500Name): NodeHandle { + return (this as InternalDriverDSL).startNode( + NodeParameters( + providedName = nodeProvidedName, + rpcUsers = listOf(rpcUser) + ) + ).getOrThrow() + } + + internal fun DriverDSL.createNodeAndBytemanNode( + nodeProvidedName: CordaX500Name, + bytemanNodeProvidedName: CordaX500Name, additionalCordapps: Collection = emptyList() - ): Pair { + ): Triple { val port = nextPort() val nodeHandle = (this as InternalDriverDSL).startNode( NodeParameters( - providedName = providedName, + providedName = nodeProvidedName, + rpcUsers = listOf(rpcUser), + additionalCordapps = additionalCordapps + ) + ) + val bytemanNodeHandle = startNode( + NodeParameters( + providedName = bytemanNodeProvidedName, rpcUsers = listOf(rpcUser), additionalCordapps = additionalCordapps ), bytemanPort = port - ).getOrThrow() - return nodeHandle to port - } - - internal fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection = emptyList()): NodeHandle { - return startNode( - NodeParameters( - providedName = providedName, - rpcUsers = listOf(rpcUser), - additionalCordapps = additionalCordapps - ) - ).getOrThrow() + ) + return Triple(nodeHandle.getOrThrow(), bytemanNodeHandle.getOrThrow(), port) } internal fun submitBytemanRules(rules: String, port: Int) { @@ -285,4 +304,4 @@ abstract class StateMachineErrorHandlingTest { internal val stateMachineManagerClassName: String by lazy { Class.forName("net.corda.node.services.statemachine.SingleThreadedStateMachineManager").name } -} \ No newline at end of file +} diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt index 0613fd277e..d1db7445fb 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt @@ -35,8 +35,7 @@ class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) - val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME, FINANCE_CORDAPPS) // could not get rule for FinalityDoctor + observation counter to work val rules = """ @@ -97,8 +96,7 @@ class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) - val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME, FINANCE_CORDAPPS) // could not get rule for FinalityDoctor + observation counter to work val rules = """ @@ -161,8 +159,7 @@ class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) - val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME, FINANCE_CORDAPPS) val rules = """ RULE Create Counter @@ -229,8 +226,7 @@ class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) - val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME, FINANCE_CORDAPPS) val rules = """ RULE Create Counter diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt index c36d9750f0..304bf0818d 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt @@ -40,8 +40,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -88,8 +87,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `unexpected error during flow initialisation throws exception to client`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter CLASS ${FlowStateMachineImpl::class.java.name} @@ -134,8 +132,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during initialisation when trying to rollback the flow's database transaction the flow is able to retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -187,8 +184,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during initialisation when trying to close the flow's database transaction the flow is able to retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -242,8 +238,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -298,8 +293,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during retrying a flow that failed when committing its original checkpoint will retry the flow again and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Throw exception on executeCommitTransaction action after first suspend + commit @@ -351,8 +345,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() { startDriver { - val (charlie, port) = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME) val rules = """ RULE Create Counter @@ -400,8 +393,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() { startDriver { - val (charlie, port) = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME) val rules = """ RULE Create Counter @@ -464,8 +456,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `responding flow - session init can be retried when there is a transient connection error to the database`() { startDriver { - val (charlie, port) = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME) val rules = """ RULE Create Counter @@ -529,8 +520,7 @@ class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `responding flow - session init can be retried when there is a transient connection error to the database goes to observation if error persists`() { startDriver { - val (charlie, port) = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME) val rules = """ RULE Create Counter diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt index c1af1bce1a..79067ffae8 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt @@ -35,8 +35,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with SendInitial action is retried 3 times and kept for observation if error persists`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -87,8 +86,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with SendInitial action that does not persist will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -135,8 +133,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with AcknowledgeMessages action is swallowed and flow completes successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Set flag when inside executeAcknowledgeMessages @@ -230,8 +227,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during flow retry when executing retryFlowFromSafePoint the flow is able to retry and recover`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Set flag when executing first suspend @@ -296,8 +292,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action that occurs after the first suspend will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) // seems to be restarting the flow from the beginning every time val rules = """ @@ -362,8 +357,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) // seems to be restarting the flow from the beginning every time val rules = """ @@ -419,8 +413,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `error during transition with CommitTransaction action and ConstraintViolationException that occurs when completing a flow will retry and be kept for observation if error persists`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -488,8 +481,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `flow can be retried when there is a transient connection error to the database`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -552,8 +544,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `flow can be retried when there is a transient connection error to the database goes to observation if error persists`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -610,8 +601,7 @@ class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `responding flow - error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { startDriver { - val (charlie, port) = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) + val (alice, charlie, port) = createNodeAndBytemanNode(ALICE_NAME, CHARLIE_NAME) val rules = """ RULE Create Counter diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt index ee5699456d..f39005c476 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt @@ -103,8 +103,7 @@ class StateMachineKillFlowErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `flow killed when it is in the flow hospital for observation is removed correctly`() { startDriver { - val (alice, port) = createBytemanNode(ALICE_NAME) - val charlie = createNode(CHARLIE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt index 5a9335136b..020206962d 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt @@ -40,8 +40,7 @@ class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `initiating subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -119,8 +118,7 @@ class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `initiating subflow - error during transition with CommitTransaction action that occurs after the first receive will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -190,8 +188,7 @@ class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `inline subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter @@ -253,8 +250,7 @@ class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() { @Test(timeout = 300_000) fun `inline subflow - error during transition with CommitTransaction action that occurs during the first receive will retry and complete successfully`() { startDriver { - val charlie = createNode(CHARLIE_NAME) - val (alice, port) = createBytemanNode(ALICE_NAME) + val (charlie, alice, port) = createNodeAndBytemanNode(CHARLIE_NAME, ALICE_NAME) val rules = """ RULE Create Counter diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowEntityManagerTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowEntityManagerTest.kt index add2317a64..e6be1b1804 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowEntityManagerTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowEntityManagerTest.kt @@ -12,6 +12,7 @@ import net.corda.core.flows.ReceiveFinalityFlow import net.corda.core.flows.SignTransactionFlow import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.node.AppServiceHub import net.corda.core.node.services.CordaService @@ -318,8 +319,10 @@ class FlowEntityManagerTest : AbstractFlowEntityManagerTest() { StaffedFlowHospital.onFlowDischarged.add { _, _ -> ++counter } driver(DriverParameters(startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val txId = alice.rpc.startFlow(::EntityManagerWithFlushCatchAndInteractWithOtherPartyFlow, bob.nodeInfo.singleIdentity()) diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowOverrideTests.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowOverrideTests.kt index 1f9129b13a..5fb4afd3a4 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowOverrideTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowOverrideTests.kt @@ -3,6 +3,7 @@ package net.corda.node.flows import co.paralleluniverse.fibers.Suspendable import net.corda.core.flows.* import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap @@ -65,36 +66,35 @@ class FlowOverrideTests { private val nodeAClasses = setOf(Ping::class.java, Pong::class.java, Pongiest::class.java) private val nodeBClasses = setOf(Ping::class.java, Pong::class.java) - @Test(timeout=300_000) - fun `should use the most specific implementation of a responding flow`() { + @Test(timeout = 300_000) + fun `should use the most specific implementation of a responding flow`() { driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = emptySet())) { - val nodeA = startNode(NodeParameters( - providedName = ALICE_NAME, - additionalCordapps = setOf(cordappForClasses(*nodeAClasses.toTypedArray())) - )).getOrThrow() - val nodeB = startNode(NodeParameters( - providedName = BOB_NAME, - additionalCordapps = setOf(cordappForClasses(*nodeBClasses.toTypedArray())) - )).getOrThrow() + val (nodeA, nodeB) = listOf(ALICE_NAME, BOB_NAME) + .map { + NodeParameters(providedName = it, + additionalCordapps = setOf(cordappForClasses(*nodeAClasses.toTypedArray()))) + } + .map { startNode(it) } + .transpose() + .getOrThrow() assertThat(nodeB.rpc.startFlow(::Ping, nodeA.nodeInfo.singleIdentity()).returnValue.getOrThrow(), `is`(Pongiest.GORGONZOLA)) } } - @Test(timeout=300_000) - fun `should use the overriden implementation of a responding flow`() { + @Test(timeout = 300_000) + fun `should use the overriden implementation of a responding flow`() { val flowOverrides = mapOf(Ping::class.java to Pong::class.java) driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = emptySet())) { - val nodeA = startNode(NodeParameters( - providedName = ALICE_NAME, - additionalCordapps = setOf(cordappForClasses(*nodeAClasses.toTypedArray())), - flowOverrides = flowOverrides - )).getOrThrow() - val nodeB = startNode(NodeParameters( - providedName = BOB_NAME, - additionalCordapps = setOf(cordappForClasses(*nodeBClasses.toTypedArray())) - )).getOrThrow() + val (nodeA, nodeB) = listOf(ALICE_NAME, BOB_NAME) + .map { + NodeParameters(providedName = it, + flowOverrides = flowOverrides, + additionalCordapps = setOf(cordappForClasses(*nodeAClasses.toTypedArray()))) + } + .map { startNode(it) } + .transpose() + .getOrThrow() assertThat(nodeB.rpc.startFlow(::Ping, nodeA.nodeInfo.singleIdentity()).returnValue.getOrThrow(), `is`(Pong.PONG)) } } - } \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt index 939d755ad9..8d82b1a07d 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt @@ -7,6 +7,7 @@ import net.corda.core.CordaRuntimeException import net.corda.core.flows.* import net.corda.core.identity.Party import net.corda.core.internal.IdempotentFlow +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.serialization.CordaSerializable import net.corda.core.utilities.ProgressTracker @@ -66,8 +67,10 @@ class FlowRetryTest { startNodesInProcess = isQuasarAgentSpecified(), notarySpecs = emptyList() )) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val result = CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { it.proxy.startFlow(::InitiatorFlow, numSessions, numIterations, nodeBHandle.nodeInfo.singleIdentity()).returnValue.getOrThrow() @@ -134,8 +137,10 @@ class FlowRetryTest { val user = User("mark", "dadada", setOf(Permissions.all())) driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { assertFailsWith { it.proxy.startFlow(::TransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) @@ -152,8 +157,10 @@ class FlowRetryTest { val user = User("mark", "dadada", setOf(Permissions.all())) driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { assertFailsWith { it.proxy.startFlow(::WrappedTransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) @@ -170,8 +177,10 @@ class FlowRetryTest { val user = User("mark", "dadada", setOf(Permissions.all())) driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { assertFailsWith { diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt index eb219baf4e..81218f3b73 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt @@ -14,6 +14,7 @@ import net.corda.core.flows.StateMachineRunId import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.CordaRPCOps import net.corda.core.messaging.startFlow import net.corda.core.node.services.StatesNotAvailableException @@ -68,9 +69,10 @@ class KillFlowTest { @Test(timeout = 300_000) fun `a killed flow will propagate the killed error to counter parties when it reaches the next suspension point`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() - val charlie = startNode(providedName = CHARLIE_NAME).getOrThrow() + val (alice, bob, charlie) = listOf(ALICE_NAME, BOB_NAME, CHARLIE_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.let { rpc -> val handle = rpc.startFlow( ::AFlowThatGetsMurderedWhenItTriesToSuspendAndSomehowKillsItsFriends, @@ -118,8 +120,10 @@ class KillFlowTest { @Test(timeout = 300_000) fun `killing a flow suspended in send + receive + sendAndReceive ends the flow immediately`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = false)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val bobParty = bob.nodeInfo.singleIdentity() bob.stop() val terminated = (bob as OutOfProcess).process.waitFor(30, TimeUnit.SECONDS) @@ -192,9 +196,10 @@ class KillFlowTest { @Test(timeout = 300_000) fun `a killed flow will propagate the killed error to counter parties if it was suspended`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() - val charlie = startNode(providedName = CHARLIE_NAME).getOrThrow() + val (alice, bob, charlie) = listOf(ALICE_NAME, BOB_NAME, CHARLIE_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() alice.rpc.let { rpc -> val handle = rpc.startFlow( ::AFlowThatGetsMurderedAndSomehowKillsItsFriends, @@ -224,9 +229,10 @@ class KillFlowTest { @Test(timeout = 300_000) fun `a killed initiated flow will propagate the killed error to the initiator and its counter parties`() { driver(DriverParameters(notarySpecs = emptyList(), startNodesInProcess = true)) { - val alice = startNode(providedName = ALICE_NAME).getOrThrow() - val bob = startNode(providedName = BOB_NAME).getOrThrow() - val charlie = startNode(providedName = CHARLIE_NAME).getOrThrow() + val (alice, bob, charlie) = listOf(ALICE_NAME, BOB_NAME, CHARLIE_NAME) + .map { startNode(providedName = it) } + .transpose() + .getOrThrow() val handle = alice.rpc.startFlow( ::AFlowThatGetsMurderedByItsFriend, listOf(bob.nodeInfo.singleIdentity(), charlie.nodeInfo.singleIdentity()) diff --git a/node/src/integration-test/kotlin/net/corda/node/modes/draining/FlowsDrainingModeContentionTest.kt b/node/src/integration-test/kotlin/net/corda/node/modes/draining/FlowsDrainingModeContentionTest.kt index e059522f89..e4107d035c 100644 --- a/node/src/integration-test/kotlin/net/corda/node/modes/draining/FlowsDrainingModeContentionTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/modes/draining/FlowsDrainingModeContentionTest.kt @@ -7,6 +7,7 @@ import net.corda.core.contracts.Command import net.corda.core.contracts.StateAndContract import net.corda.core.flows.* import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.internal.packageName import net.corda.core.messaging.startFlow import net.corda.core.transactions.SignedTransaction @@ -57,8 +58,10 @@ class FlowsDrainingModeContentionTest { portAllocation = portAllocation, extraCordappPackagesToScan = listOf(MessageState::class.packageName) )) { - val nodeA = startNode(providedName = ALICE_NAME, rpcUsers = users).getOrThrow() - val nodeB = startNode(providedName = BOB_NAME, rpcUsers = users).getOrThrow() + val (nodeA, nodeB) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = users) } + .transpose() + .getOrThrow() val nodeARpcInfo = RpcInfo(nodeA.rpcAddress, user.username, user.password) val flow = nodeA.rpc.startFlow(::ProposeTransactionAndWaitForCommit, message, nodeARpcInfo, nodeB.nodeInfo.singleIdentity(), defaultNotaryIdentity) diff --git a/node/src/integration-test/kotlin/net/corda/node/modes/draining/P2PFlowsDrainingModeTest.kt b/node/src/integration-test/kotlin/net/corda/node/modes/draining/P2PFlowsDrainingModeTest.kt index ce70423592..a72ff9d4bf 100644 --- a/node/src/integration-test/kotlin/net/corda/node/modes/draining/P2PFlowsDrainingModeTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/modes/draining/P2PFlowsDrainingModeTest.kt @@ -4,6 +4,7 @@ import co.paralleluniverse.fibers.Suspendable import net.corda.core.flows.* import net.corda.core.identity.Party import net.corda.core.internal.concurrent.map +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.contextLogger import net.corda.core.utilities.getOrThrow @@ -53,8 +54,11 @@ class P2PFlowsDrainingModeTest { @Test(timeout=300_000) fun `flows draining mode suspends consumption of initial session messages`() { driver(DriverParameters(startNodesInProcess = false, portAllocation = portAllocation, notarySpecs = emptyList())) { - val initiatedNode = startNode(providedName = ALICE_NAME).getOrThrow() - val initiating = startNode(providedName = BOB_NAME, rpcUsers = users).getOrThrow().rpc + val (initiatedNode, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = users) } + .transpose() + .getOrThrow() + val initiating = bob.rpc val counterParty = initiatedNode.nodeInfo.singleIdentity() val initiated = initiatedNode.rpc @@ -85,8 +89,10 @@ class P2PFlowsDrainingModeTest { driver(DriverParameters(portAllocation = portAllocation, notarySpecs = emptyList())) { - val nodeA = startNode(providedName = ALICE_NAME, rpcUsers = users).getOrThrow() - val nodeB = startNode(providedName = BOB_NAME, rpcUsers = users).getOrThrow() + val (nodeA, nodeB) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = users) } + .transpose() + .getOrThrow() var successful = false val latch = CountDownLatch(1) @@ -133,8 +139,10 @@ class P2PFlowsDrainingModeTest { driver(DriverParameters(portAllocation = portAllocation, notarySpecs = emptyList())) { - val nodeA = startNode(providedName = ALICE_NAME, rpcUsers = users).getOrThrow() - val nodeB = startNode(providedName = BOB_NAME, rpcUsers = users).getOrThrow() + val (nodeA, nodeB) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = users) } + .transpose() + .getOrThrow() var successful = false val latch = CountDownLatch(1) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/rpc/RpcExceptionHandlingTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/rpc/RpcExceptionHandlingTest.kt index 3ae02898e5..e0e099646d 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/rpc/RpcExceptionHandlingTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/rpc/RpcExceptionHandlingTest.kt @@ -5,6 +5,7 @@ import net.corda.core.CordaRuntimeException import net.corda.core.flows.* import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap @@ -58,8 +59,10 @@ class RpcExceptionHandlingTest { } driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = listOf(enclosedCordapp()))) { - val devModeNode = startNode(params, BOB_NAME).getOrThrow() - val node = startNode(ALICE_NAME, devMode = false, parameters = params).getOrThrow() + val (devModeNode, node) = listOf(startNode(params, BOB_NAME), + startNode(ALICE_NAME, devMode = false, parameters = params)) + .transpose() + .getOrThrow() assertThatThrownExceptionIsReceivedUnwrapped(devModeNode) assertThatThrownExceptionIsReceivedUnwrapped(node) @@ -77,8 +80,10 @@ class RpcExceptionHandlingTest { } driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = listOf(enclosedCordapp()))) { - val devModeNode = startNode(params, BOB_NAME).getOrThrow() - val node = startNode(ALICE_NAME, devMode = false, parameters = params).getOrThrow() + val (devModeNode, node) = listOf(startNode(params, BOB_NAME), + startNode(ALICE_NAME, devMode = false, parameters = params)) + .transpose() + .getOrThrow() assertThatThrownBy { devModeNode.throwExceptionFromFlow() }.isInstanceOfSatisfying(FlowException::class.java) { exception -> assertThat(exception).hasNoCause() @@ -102,8 +107,10 @@ class RpcExceptionHandlingTest { fun DriverDSL.scenario(nameA: CordaX500Name, nameB: CordaX500Name, devMode: Boolean) { - val nodeA = startNode(nameA, devMode, params).getOrThrow() - val nodeB = startNode(nameB, devMode, params).getOrThrow() + val (nodeA, nodeB) = listOf(nameA, nameB) + .map { startNode(it, devMode, params) } + .transpose() + .getOrThrow() nodeA.rpc.startFlow(::InitFlow, nodeB.nodeInfo.singleIdentity()).returnValue.getOrThrow() } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowHospitalTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowHospitalTest.kt index 739e684623..c26a106910 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowHospitalTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowHospitalTest.kt @@ -15,6 +15,7 @@ import net.corda.core.flows.NotaryException import net.corda.core.flows.ReceiveFinalityFlow import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.StateMachineUpdate import net.corda.core.messaging.startFlow import net.corda.core.utilities.OpaqueBytes @@ -46,14 +47,20 @@ class FlowHospitalTest { private val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) - @Test(timeout=300_000) - fun `when double spend occurs, the flow is successfully deleted on the counterparty`() { + @Test(timeout = 300_000) + fun `when double spend occurs, the flow is successfully deleted on the counterparty`() { driver(DriverParameters(cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")))) { - val charlie = startNode(providedName = CHARLIE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() - val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() - - val charlieClient = CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val aliceClient = CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val (charlieClient, aliceClient) = listOf(CHARLIE_NAME, ALICE_NAME) + .map { + startNode(providedName = it, + rpcUsers = listOf(rpcUser)) + } + .transpose() + .getOrThrow() + .map { + CordaRPCClient(it.rpcAddress) + .start(rpcUser.username, rpcUser.password).proxy + } val aliceParty = aliceClient.nodeInfo().legalIdentities.first() @@ -80,7 +87,7 @@ class FlowHospitalTest { val secondStateAndRef = charlieClient.startFlow(::IssueFlow, defaultNotaryIdentity).returnValue.get() charlieClient.startFlow(::SpendFlowWithCustomException, secondStateAndRef, aliceParty).returnValue.get() - val secondSubscription = aliceClient.stateMachinesFeed().updates.subscribe{ + val secondSubscription = aliceClient.stateMachinesFeed().updates.subscribe { if (it is StateMachineUpdate.Removed && it.result.isFailure) secondLatch.countDown() } @@ -95,75 +102,75 @@ class FlowHospitalTest { } } - @Test(timeout=300_000) - fun `HospitalizeFlowException thrown`() { + @Test(timeout = 300_000) + fun `HospitalizeFlowException thrown`() { var observationCounter: Int = 0 StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> ++observationCounter } driver( - DriverParameters( - startNodesInProcess = true, - cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) - ) + DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) + ) ) { val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() val aliceClient = CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy assertFailsWith { aliceClient.startFlow(::ThrowingHospitalisedExceptionFlow, HospitalizeFlowException::class.java) - .returnValue.getOrThrow(5.seconds) + .returnValue.getOrThrow(5.seconds) } assertEquals(1, observationCounter) } } - @Test(timeout=300_000) - fun `Custom exception wrapping HospitalizeFlowException thrown`() { + @Test(timeout = 300_000) + fun `Custom exception wrapping HospitalizeFlowException thrown`() { var observationCounter: Int = 0 StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> ++observationCounter } driver( - DriverParameters( - startNodesInProcess = true, - cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) - ) + DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) + ) ) { val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() val aliceClient = CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy assertFailsWith { aliceClient.startFlow(::ThrowingHospitalisedExceptionFlow, WrappingHospitalizeFlowException::class.java) - .returnValue.getOrThrow(5.seconds) + .returnValue.getOrThrow(5.seconds) } assertEquals(1, observationCounter) } } - @Test(timeout=300_000) - fun `Custom exception extending HospitalizeFlowException thrown`() { + @Test(timeout = 300_000) + fun `Custom exception extending HospitalizeFlowException thrown`() { var observationCounter: Int = 0 StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> ++observationCounter } driver( - DriverParameters( - startNodesInProcess = true, - cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) - ) + DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) + ) ) { // one node will be enough for this testing val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() val aliceClient = CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy assertFailsWith { aliceClient.startFlow(::ThrowingHospitalisedExceptionFlow, ExtendingHospitalizeFlowException::class.java) - .returnValue.getOrThrow(5.seconds) + .returnValue.getOrThrow(5.seconds) } assertEquals(1, observationCounter) } } - @Test(timeout=300_000) - fun `HospitalizeFlowException cloaking an important exception thrown`() { + @Test(timeout = 300_000) + fun `HospitalizeFlowException cloaking an important exception thrown`() { var dischargedCounter = 0 var observationCounter: Int = 0 StaffedFlowHospital.onFlowDischarged.add { _, _ -> @@ -173,16 +180,16 @@ class FlowHospitalTest { ++observationCounter } driver( - DriverParameters( - startNodesInProcess = true, - cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) - ) + DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = listOf(enclosedCordapp(), findCordapp("net.corda.testing.contracts")) + ) ) { val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser)).getOrThrow() val aliceClient = CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy assertFailsWith { aliceClient.startFlow(::ThrowingHospitalisedExceptionFlow, CloakingHospitalizeFlowException::class.java) - .returnValue.getOrThrow(5.seconds) + .returnValue.getOrThrow(5.seconds) } assertEquals(0, observationCounter) // Since the flow will keep getting discharged from hospital dischargedCounter will be > 1. @@ -191,7 +198,7 @@ class FlowHospitalTest { } @StartableByRPC - class IssueFlow(val notary: Party): FlowLogic>() { + class IssueFlow(val notary: Party) : FlowLogic>() { @Suspendable override fun call(): StateAndRef { @@ -201,12 +208,11 @@ class FlowHospitalTest { val notarised = subFlow(FinalityFlow(signedTransaction, emptySet())) return notarised.coreTransaction.outRef(0) } - } @StartableByRPC @InitiatingFlow - class SpendFlow(private val stateAndRef: StateAndRef, private val newOwner: Party): FlowLogic() { + class SpendFlow(private val stateAndRef: StateAndRef, private val newOwner: Party) : FlowLogic() { @Suspendable override fun call() { @@ -216,11 +222,10 @@ class FlowHospitalTest { sessionWithCounterParty.sendAndReceive("initial-message") subFlow(FinalityFlow(signedTransaction, setOf(sessionWithCounterParty))) } - } @InitiatedBy(SpendFlow::class) - class AcceptSpendFlow(private val otherSide: FlowSession): FlowLogic() { + class AcceptSpendFlow(private val otherSide: FlowSession) : FlowLogic() { @Suspendable override fun call() { @@ -229,12 +234,11 @@ class FlowHospitalTest { subFlow(ReceiveFinalityFlow(otherSide)) } - } @StartableByRPC @InitiatingFlow - class SpendFlowWithCustomException(private val stateAndRef: StateAndRef, private val newOwner: Party): + class SpendFlowWithCustomException(private val stateAndRef: StateAndRef, private val newOwner: Party) : FlowLogic() { @Suspendable @@ -249,11 +253,10 @@ class FlowHospitalTest { throw DoubleSpendException("double spend!", e) } } - } @InitiatedBy(SpendFlowWithCustomException::class) - class AcceptSpendFlowWithCustomException(private val otherSide: FlowSession): FlowLogic() { + class AcceptSpendFlowWithCustomException(private val otherSide: FlowSession) : FlowLogic() { @Suspendable override fun call() { @@ -262,16 +265,15 @@ class FlowHospitalTest { subFlow(ReceiveFinalityFlow(otherSide)) } - } - class DoubleSpendException(message: String, cause: Throwable): FlowException(message, cause) + class DoubleSpendException(message: String, cause: Throwable) : FlowException(message, cause) @StartableByRPC class ThrowingHospitalisedExceptionFlow( - // Starting this Flow from an RPC client: if we pass in an encapsulated exception within another exception then the wrapping - // exception, when deserialized, will get grounded into a CordaRuntimeException (this happens in ThrowableSerializer#fromProxy). - private val hospitalizeFlowExceptionClass: Class<*>): FlowLogic() { + // Starting this Flow from an RPC client: if we pass in an encapsulated exception within another exception then the wrapping + // exception, when deserialized, will get grounded into a CordaRuntimeException (this happens in ThrowableSerializer#fromProxy). + private val hospitalizeFlowExceptionClass: Class<*>) : FlowLogic() { @Suspendable override fun call() { @@ -282,7 +284,7 @@ class FlowHospitalTest { } } - class WrappingHospitalizeFlowException(cause: HospitalizeFlowException = HospitalizeFlowException()) : Exception(cause) + class WrappingHospitalizeFlowException(cause: HospitalizeFlowException = HospitalizeFlowException()) : Exception(cause) class ExtendingHospitalizeFlowException : HospitalizeFlowException() @@ -294,5 +296,4 @@ class FlowHospitalTest { setCause(SQLException("deadlock")) } } - } \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt index 86bb3b2931..601134338f 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt @@ -16,6 +16,7 @@ import net.corda.core.flows.FlowLogic import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party import net.corda.core.internal.concurrent.openFuture +import net.corda.core.internal.concurrent.transpose import net.corda.core.messaging.startFlow import net.corda.core.node.services.Vault import net.corda.core.node.services.vault.QueryCriteria @@ -450,8 +451,11 @@ class VaultObserverExceptionTest { findCordapp("com.r3.dbfailure.schemas") ),inMemoryDB = false) ) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (aliceNode, bobNode) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, + rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val notary = defaultNotaryHandle.nodeHandles.getOrThrow().first() val startErrorInObservableWhenConsumingState = { @@ -540,8 +544,11 @@ class VaultObserverExceptionTest { ), inMemoryDB = false) ) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (aliceNode, bobNode) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, + rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val notary = defaultNotaryHandle.nodeHandles.getOrThrow().first() val startErrorInObservableWhenConsumingState = { @@ -622,8 +629,11 @@ class VaultObserverExceptionTest { ), inMemoryDB = false) ) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (aliceNode, bobNode) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, + rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val notary = defaultNotaryHandle.nodeHandles.getOrThrow().first() val startErrorInObservableWhenCreatingSecondState = { @@ -699,8 +709,11 @@ class VaultObserverExceptionTest { ), inMemoryDB = false) ) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (aliceNode, bobNode) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, + rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val notary = defaultNotaryHandle.nodeHandles.getOrThrow().first() val startErrorInObservableWhenConsumingState = { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index 0fae5c91bb..8d6fbf6c0e 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -21,6 +21,7 @@ import net.corda.core.flows.StartableByService import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.Party import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.internal.concurrent.transpose import net.corda.core.internal.uncheckedCast import net.corda.core.messaging.startFlow import net.corda.core.node.AppServiceHub @@ -74,8 +75,10 @@ class FlowMetadataRecordingTest { fun `rpc started flows have metadata recorded`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() var flowId: StateMachineRunId? = null var context: InvocationContext? = null @@ -162,8 +165,10 @@ class FlowMetadataRecordingTest { fun `rpc started flows have their arguments removed from in-memory checkpoint after zero'th checkpoint`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() var context: InvocationContext? = null var metadata: DBCheckpointStorage.DBFlowMetadata? = null @@ -214,8 +219,10 @@ class FlowMetadataRecordingTest { fun `initiated flows have metadata recorded`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() var flowId: StateMachineRunId? = null var context: InvocationContext? = null @@ -260,8 +267,10 @@ class FlowMetadataRecordingTest { fun `service started flows have metadata recorded`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() var flowId: StateMachineRunId? = null var context: InvocationContext? = null @@ -306,8 +315,10 @@ class FlowMetadataRecordingTest { fun `scheduled flows have metadata recorded`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() val lock = Semaphore(0) @@ -361,8 +372,10 @@ class FlowMetadataRecordingTest { fun `flows have their finish time recorded when completed`() { driver(DriverParameters(startNodesInProcess = true)) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() var flowId: StateMachineRunId? = null var metadata: DBCheckpointStorage.DBFlowMetadata? = null diff --git a/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt b/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt index 1a6a9fec81..d1a7df5eda 100644 --- a/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt +++ b/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt @@ -16,6 +16,7 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.* import net.corda.core.identity.AbstractParty import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.transpose import net.corda.core.internal.createDirectories import net.corda.core.internal.div import net.corda.core.internal.inputStream @@ -364,8 +365,10 @@ class InteractiveShellIntegrationTest { fun `dumpCheckpoints creates zip with json file for suspended flow`() { val user = User("u", "p", setOf(all())) driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()))) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + val (aliceNode, bobNode) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() bobNode.stop() // Create logs directory since the driver is not creating it From 2ca10464b78c0c4de2eb676cf368bf1c2db27c35 Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Thu, 23 Jul 2020 16:37:29 +0100 Subject: [PATCH 38/48] CORDA-3845: Update BC, log4j, slf4j (#6464) * CORDA-3845: Update BC to 1.64 * CORDA-3845: Upgraded log4j to 2.13.3 * We can remove the use of Manifests from the logging package so that when _it_ logs it doesn't error on the fact the stream was already closed by the default Java logger. * Some more tidy up * Remove the logging package as a plugin * latest BC version * Remove old test * fix up * Fix some rebased changes to log file handling * Fix some rebased changes to log file handling * Update slf4j too Co-authored-by: Adel El-Beik --- build.gradle | 4 +- .../common/logging/ErrorCodeRewritePolicy.kt | 28 -------- config/dev/log4j2.xml | 6 +- constants.properties | 2 +- .../DuplicateSerializerLogTest.kt | 2 +- ...cateSerializerLogWithSameSerializerTest.kt | 2 +- .../node/logging/ErrorCodeLoggingTests.kt | 68 ------------------- .../services/config/NodeConfigParsingTests.kt | 2 +- .../kotlin/net/corda/testing/driver/Driver.kt | 3 + .../src/main/resources/log4j2-test.xml | 5 +- .../src/test/resources/log4j2-test.xml | 5 +- 11 files changed, 12 insertions(+), 115 deletions(-) delete mode 100644 common/logging/src/main/kotlin/net/corda/common/logging/ErrorCodeRewritePolicy.kt delete mode 100644 node/src/integration-test/kotlin/net/corda/node/logging/ErrorCodeLoggingTests.kt diff --git a/build.gradle b/build.gradle index 6c421444de..1d8c28e160 100644 --- a/build.gradle +++ b/build.gradle @@ -68,8 +68,8 @@ buildscript { ext.jersey_version = '2.25' ext.servlet_version = '4.0.1' ext.assertj_version = '3.12.2' - ext.slf4j_version = '1.7.26' - ext.log4j_version = '2.11.2' + ext.slf4j_version = '1.7.30' + ext.log4j_version = '2.13.3' ext.bouncycastle_version = constants.getProperty("bouncycastleVersion") ext.guava_version = constants.getProperty("guavaVersion") ext.caffeine_version = constants.getProperty("caffeineVersion") diff --git a/common/logging/src/main/kotlin/net/corda/common/logging/ErrorCodeRewritePolicy.kt b/common/logging/src/main/kotlin/net/corda/common/logging/ErrorCodeRewritePolicy.kt deleted file mode 100644 index 5f0e24bef9..0000000000 --- a/common/logging/src/main/kotlin/net/corda/common/logging/ErrorCodeRewritePolicy.kt +++ /dev/null @@ -1,28 +0,0 @@ -package net.corda.common.logging - -import org.apache.logging.log4j.core.Core -import org.apache.logging.log4j.core.LogEvent -import org.apache.logging.log4j.core.appender.rewrite.RewritePolicy -import org.apache.logging.log4j.core.config.plugins.Plugin -import org.apache.logging.log4j.core.config.plugins.PluginFactory -import org.apache.logging.log4j.core.impl.Log4jLogEvent - -@Plugin(name = "ErrorCodeRewritePolicy", category = Core.CATEGORY_NAME, elementType = "rewritePolicy", printObject = false) -class ErrorCodeRewritePolicy : RewritePolicy { - override fun rewrite(source: LogEvent): LogEvent? { - val newMessage = source.message?.withErrorCodeFor(source.thrown, source.level) - return if (newMessage == source.message) { - source - } else { - Log4jLogEvent.Builder(source).setMessage(newMessage).build() - } - } - - companion object { - @JvmStatic - @PluginFactory - fun createPolicy(): ErrorCodeRewritePolicy { - return ErrorCodeRewritePolicy() - } - } -} \ No newline at end of file diff --git a/config/dev/log4j2.xml b/config/dev/log4j2.xml index 051518c4a6..02aa604cf4 100644 --- a/config/dev/log4j2.xml +++ b/config/dev/log4j2.xml @@ -1,5 +1,5 @@ - + ${sys:log-path:-logs} @@ -172,21 +172,17 @@ - - - - diff --git a/constants.properties b/constants.properties index 6cc3b7e4a2..e6fa88e69d 100644 --- a/constants.properties +++ b/constants.properties @@ -20,7 +20,7 @@ quasarClassifier=jdk8 quasarVersion11=0.8.0_r3 jdkClassifier11=jdk11 proguardVersion=6.1.1 -bouncycastleVersion=1.60 +bouncycastleVersion=1.66 classgraphVersion=4.8.78 disruptorVersion=3.4.2 typesafeConfigVersion=1.3.4 diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt index 2f87e1005f..0e96e84d3c 100644 --- a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogTest.kt @@ -6,8 +6,8 @@ import net.corda.core.flows.StartableByRPC import net.corda.core.messaging.startFlow import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.utilities.getOrThrow -import net.corda.node.logging.logFile import net.corda.testing.driver.driver +import net.corda.testing.driver.logFile import org.assertj.core.api.Assertions import org.junit.Test import java.time.Duration diff --git a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt index 598b1ed401..3608bc7a6b 100644 --- a/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/DuplicateSerializerLogWithSameSerializerTest.kt @@ -7,9 +7,9 @@ import net.corda.core.messaging.startFlow import net.corda.core.serialization.CheckpointCustomSerializer import net.corda.core.serialization.CordaSerializable import net.corda.core.utilities.getOrThrow -import net.corda.node.logging.logFile import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.driver +import net.corda.testing.driver.logFile import net.corda.testing.node.internal.enclosedCordapp import org.assertj.core.api.Assertions import org.junit.Test diff --git a/node/src/integration-test/kotlin/net/corda/node/logging/ErrorCodeLoggingTests.kt b/node/src/integration-test/kotlin/net/corda/node/logging/ErrorCodeLoggingTests.kt deleted file mode 100644 index e1db95e528..0000000000 --- a/node/src/integration-test/kotlin/net/corda/node/logging/ErrorCodeLoggingTests.kt +++ /dev/null @@ -1,68 +0,0 @@ -package net.corda.node.logging - -import net.corda.core.flows.FlowLogic -import net.corda.core.flows.InitiatingFlow -import net.corda.core.flows.StartableByRPC -import net.corda.core.internal.div -import net.corda.core.messaging.FlowHandle -import net.corda.core.messaging.startFlow -import net.corda.core.utilities.getOrThrow -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeHandle -import net.corda.testing.driver.driver -import org.assertj.core.api.Assertions.assertThat -import org.junit.Test -import java.io.File - -class ErrorCodeLoggingTests { - @Test(timeout=300_000) - fun `log entries with a throwable and ERROR or WARN get an error code appended`() { - driver(DriverParameters(notarySpecs = emptyList())) { - val node = startNode(startInSameProcess = false).getOrThrow() - node.rpc.startFlow(::MyFlow).waitForCompletion() - val logFile = node.logFile() - - val linesWithErrorCode = logFile.useLines { lines -> - lines.filter { line -> - line.contains("[errorCode=") - }.filter { line -> - line.contains("moreInformationAt=https://errors.corda.net/") - }.toList() - } - - assertThat(linesWithErrorCode).isNotEmpty - } - } - - // This is used to detect broken logging which can be caused by loggers being initialized - // before the initLogging() call is made - @Test(timeout=300_000) - fun `When logging is set to error level, there are no other levels logged after node startup`() { - driver(DriverParameters(notarySpecs = emptyList())) { - val node = startNode(startInSameProcess = false, logLevelOverride = "ERROR").getOrThrow() - val logFile = node.logFile() - val lengthAfterStart = logFile.length() - node.rpc.startFlow(::MyFlow).waitForCompletion() - // An exception thrown in a flow will log at the "INFO" level. - assertThat(logFile.length()).isEqualTo(lengthAfterStart) - } - } - - @StartableByRPC - @InitiatingFlow - class MyFlow : FlowLogic() { - override fun call(): String { - throw IllegalArgumentException("Mwahahahah") - } - } -} - -private fun FlowHandle<*>.waitForCompletion() { - try { - returnValue.getOrThrow() - } catch (e: Exception) { - // This is expected to throw an exception, using getOrThrow() just to wait until done. - } -} - -fun NodeHandle.logFile(): File = (baseDirectory / "logs").toFile().walk().filter { it.name.startsWith("node-") && it.extension == "log" }.single() \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt index 2f704bf630..5eec942efd 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt @@ -1,10 +1,10 @@ package net.corda.node.services.config import net.corda.core.utilities.getOrThrow -import net.corda.node.logging.logFile import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.driver import net.corda.testing.driver.internal.incrementalPortAllocation +import net.corda.testing.driver.logFile import org.junit.Assert.assertTrue import org.junit.Test diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt index 8c16c40a34..6481a29714 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt @@ -26,6 +26,7 @@ import net.corda.testing.node.internal.genericDriver import net.corda.testing.node.internal.getTimestampAsDirectoryName import net.corda.testing.node.internal.newContext import rx.Observable +import java.io.File import java.nio.file.Path import java.nio.file.Paths import java.util.concurrent.atomic.AtomicInteger @@ -66,6 +67,8 @@ interface NodeHandle : AutoCloseable { fun stop() } +fun NodeHandle.logFile(): File = (baseDirectory / "logs").toFile().walk().filter { it.name.startsWith("node-") && it.extension == "log" }.single() + /** Interface which represents an out of process node and exposes its process handle. **/ @DoNotImplement interface OutOfProcess : NodeHandle { diff --git a/testing/test-common/src/main/resources/log4j2-test.xml b/testing/test-common/src/main/resources/log4j2-test.xml index d8d489af1f..45910c8ca5 100644 --- a/testing/test-common/src/main/resources/log4j2-test.xml +++ b/testing/test-common/src/main/resources/log4j2-test.xml @@ -1,5 +1,5 @@ - + ${sys:log-path:-logs} @@ -63,17 +63,14 @@ - - - diff --git a/testing/test-db/src/test/resources/log4j2-test.xml b/testing/test-db/src/test/resources/log4j2-test.xml index d0941363a9..35b51709ed 100644 --- a/testing/test-db/src/test/resources/log4j2-test.xml +++ b/testing/test-db/src/test/resources/log4j2-test.xml @@ -1,5 +1,5 @@ - + ${sys:log-path:-logs} @@ -65,17 +65,14 @@ - - - From 13073c300f4db305722946e35bba9828863d041c Mon Sep 17 00:00:00 2001 From: Ross Nicoll Date: Thu, 23 Jul 2020 16:59:42 +0100 Subject: [PATCH 39/48] NOTICK: OS 4.3 to OS 4.4 merge (#6506) * CORDA-3917 Update to Jackson 2.9.8 (#6493) * Update to Jackson 2.9.8 to address multiple security issues, and update warning note about updates to clarify that it refers to 2.10+. When the note was added 2.9.7 as the highest available version in the 2.9.x series. * Add PR code checks Jenkinsfile * CORDA-3916 Update to BouncyCastle 1.61 (#6492) Update to BouncyCastle 1.61. Updating one version at a time to mitigate risk of a complex breaking change being introduced. * Added missing collection of JUnit tests and logs Co-authored-by: Waldemar Zurowski --- .ci/dev/compatibility/JenkinsfileJDK11Compile | 4 ++++ build.gradle | 4 ++-- constants.properties | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index f6e9c43195..d84a91db54 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -32,6 +32,10 @@ pipeline { } post { + always { + archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/build.gradle b/build.gradle index 85fc16a9da..241c85b5a6 100644 --- a/build.gradle +++ b/build.gradle @@ -62,8 +62,8 @@ buildscript { ext.asm_version = '7.1' ext.artemis_version = '2.6.2' - // TODO Upgrade Jackson only when corda is using kotlin 1.3.10 - ext.jackson_version = '2.9.7' + // TODO Upgrade to Jackson 2.10+ only when corda is using kotlin 1.3.10 + ext.jackson_version = '2.9.8' ext.jetty_version = '9.4.19.v20190610' ext.jersey_version = '2.25' ext.servlet_version = '4.0.1' diff --git a/constants.properties b/constants.properties index 27a3493ced..1ca16d18ee 100644 --- a/constants.properties +++ b/constants.properties @@ -20,7 +20,7 @@ quasarClassifier=jdk8 quasarVersion11=0.8.0_r3 jdkClassifier11=jdk11 proguardVersion=6.1.1 -bouncycastleVersion=1.60 +bouncycastleVersion=1.61 classgraphVersion=4.8.58 disruptorVersion=3.4.2 typesafeConfigVersion=1.3.4 From d4189c4f375ebafc2d7a39bb8d6962fe83031052 Mon Sep 17 00:00:00 2001 From: Ryan Fowler Date: Mon, 27 Jul 2020 14:09:26 +0100 Subject: [PATCH 40/48] CORDA-3918: Port of ENT-5417: Allow exceptions to propagate when shutdown commands are called (#6516) --- .../rpc/internal/ReconnectingCordaRPCOps.kt | 19 ++++++++++--------- .../net/corda/tools/shell/InteractiveShell.kt | 10 ++++++---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt index ff833dd03b..641d2323ca 100644 --- a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt +++ b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt @@ -292,6 +292,7 @@ class ReconnectingCordaRPCOps private constructor( } private class ErrorInterceptingHandler(val reconnectingRPCConnection: ReconnectingRPCConnection) : InvocationHandler { private fun Method.isStartFlow() = name.startsWith("startFlow") || name.startsWith("startTrackedFlow") + private fun Method.isShutdown() = name == "shutdown" || name == "gracefulShutdown" || name == "terminate" private fun checkIfIsStartFlow(method: Method, e: InvocationTargetException) { if (method.isStartFlow()) { @@ -306,7 +307,7 @@ class ReconnectingCordaRPCOps private constructor( * * A negative number for [maxNumberOfAttempts] means an unlimited number of retries will be performed. */ - @Suppress("ThrowsCount", "ComplexMethod") + @Suppress("ThrowsCount", "ComplexMethod", "NestedBlockDepth") private fun doInvoke(method: Method, args: Array?, maxNumberOfAttempts: Int): Any? { checkIfClosed() var remainingAttempts = maxNumberOfAttempts @@ -318,20 +319,20 @@ class ReconnectingCordaRPCOps private constructor( log.debug { "RPC $method invoked successfully." } } } catch (e: InvocationTargetException) { - if (method.name.equals("shutdown", true)) { - log.debug("Shutdown invoked, stop reconnecting.", e) - reconnectingRPCConnection.notifyServerAndClose() - break - } when (e.targetException) { is RejectedCommandException -> { log.warn("Node is being shutdown. Operation ${method.name} rejected. Shutting down...", e) throw e.targetException } is ConnectionFailureException -> { - log.warn("Failed to perform operation ${method.name}. Connection dropped. Retrying....", e) - reconnectingRPCConnection.reconnectOnError(e) - checkIfIsStartFlow(method, e) + if (method.isShutdown()) { + log.debug("Shutdown invoked, stop reconnecting.", e) + reconnectingRPCConnection.notifyServerAndClose() + } else { + log.warn("Failed to perform operation ${method.name}. Connection dropped. Retrying....", e) + reconnectingRPCConnection.reconnectOnError(e) + checkIfIsStartFlow(method, e) + } } is RPCException -> { rethrowIfUnrecoverable(e.targetException as RPCException) diff --git a/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt b/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt index bc00b7b53a..d78f4be24f 100644 --- a/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt +++ b/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt @@ -111,6 +111,8 @@ object InteractiveShell { YAML } + private fun isShutdownCmd(cmd: String) = cmd == "shutdown" || cmd == "gracefulShutdown" || cmd == "terminate" + fun startShell(configuration: ShellConfiguration, classLoader: ClassLoader? = null, standalone: Boolean = false) { makeRPCConnection = { username: String, password: String -> val connection = if (standalone) { @@ -623,6 +625,10 @@ object InteractiveShell { throw e.rootCause } } + if (isShutdownCmd(cmd)) { + out.println("Called 'shutdown' on the node.\nQuitting the shell now.").also { out.flush() } + onExit.invoke() + } } catch (e: StringToMethodCallParser.UnparseableCallException) { out.println(e.message, Decoration.bold, Color.red) if (e !is StringToMethodCallParser.UnparseableCallException.NoSuchFile) { @@ -634,10 +640,6 @@ object InteractiveShell { InputStreamSerializer.invokeContext = null InputStreamDeserializer.closeAll() } - if (cmd == "shutdown") { - out.println("Called 'shutdown' on the node.\nQuitting the shell now.").also { out.flush() } - onExit.invoke() - } return result } From 7f94f6cb39fd7579ec917919e8aa98d41c668bf3 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Tue, 28 Jul 2020 09:36:11 +0100 Subject: [PATCH 41/48] Added missing buildDiscarder to docs API publisher Jenkins configuration * updated regex for recognised tag to match JenkinsUI configuration --- .ci/dev/publish-api-docs/Jenkinsfile | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.ci/dev/publish-api-docs/Jenkinsfile b/.ci/dev/publish-api-docs/Jenkinsfile index d99d17ef44..b45aa95e95 100644 --- a/.ci/dev/publish-api-docs/Jenkinsfile +++ b/.ci/dev/publish-api-docs/Jenkinsfile @@ -1,5 +1,15 @@ -@Library('corda-shared-build-pipeline-steps') +#!groovy +/** + * Jenkins pipeline to build Corda OS KDoc & Javadoc archive + */ +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ +@Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) @@ -10,6 +20,7 @@ pipeline { ansiColor('xterm') timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { @@ -20,7 +31,7 @@ pipeline { stages { stage('Publish Archived API Docs to Artifactory') { - when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' } + when { tag pattern: /^docs-release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' } steps { sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs" } From f2336f397d9ec139903d54c7cf7dfcf22f91bf35 Mon Sep 17 00:00:00 2001 From: Dimos Raptis Date: Tue, 28 Jul 2020 10:20:24 +0100 Subject: [PATCH 42/48] CORDA-3506 - Add test for session close API (#6512) --- .../corda/node/flows/FlowSessionCloseTest.kt | 59 +++++++++++++------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt index a7e0cf877e..b7abe4249f 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowSessionCloseTest.kt @@ -43,7 +43,7 @@ class FlowSessionCloseTest { ).transpose().getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { - assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), true, null, false).returnValue.getOrThrow() } + assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), true, null, InitiatorFlow.ResponderReaction.NORMAL_CLOSE).returnValue.getOrThrow() } .isInstanceOf(CordaRuntimeException::class.java) .hasMessageContaining(PrematureSessionCloseException::class.java.name) .hasMessageContaining("The following session was closed before it was initialised") @@ -52,18 +52,26 @@ class FlowSessionCloseTest { } @Test(timeout=300_000) - fun `flow cannot access closed session`() { + fun `flow cannot access closed session, unless it's a duplicate close which is handled gracefully`() { driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = listOf(enclosedCordapp()), notarySpecs = emptyList())) { val (nodeAHandle, nodeBHandle) = listOf( startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)), startNode(providedName = BOB_NAME, rpcUsers = listOf(user)) ).transpose().getOrThrow() - InitiatorFlow.SessionAPI.values().forEach { sessionAPI -> - CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { - assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, sessionAPI, false).returnValue.getOrThrow() } - .isInstanceOf(UnexpectedFlowEndException::class.java) - .hasMessageContaining("Tried to access ended session") + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + InitiatorFlow.SessionAPI.values().forEach { sessionAPI -> + when (sessionAPI) { + InitiatorFlow.SessionAPI.CLOSE -> { + it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, sessionAPI, InitiatorFlow.ResponderReaction.NORMAL_CLOSE).returnValue.getOrThrow() + } + else -> { + assertThatThrownBy { it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, sessionAPI, InitiatorFlow.ResponderReaction.NORMAL_CLOSE).returnValue.getOrThrow() } + .isInstanceOf(UnexpectedFlowEndException::class.java) + .hasMessageContaining("Tried to access ended session") + } + } } } @@ -79,7 +87,7 @@ class FlowSessionCloseTest { ).transpose().getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { - it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, false).returnValue.getOrThrow() + it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, InitiatorFlow.ResponderReaction.NORMAL_CLOSE).returnValue.getOrThrow() } } } @@ -93,7 +101,7 @@ class FlowSessionCloseTest { ).transpose().getOrThrow() CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { - it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, true).returnValue.getOrThrow() + it.proxy.startFlow(::InitiatorFlow, nodeBHandle.nodeInfo.legalIdentities.first(), false, null, InitiatorFlow.ResponderReaction.RETRY_CLOSE_FROM_CHECKPOINT).returnValue.getOrThrow() } } } @@ -151,14 +159,21 @@ class FlowSessionCloseTest { @StartableByRPC class InitiatorFlow(val party: Party, private val prematureClose: Boolean = false, private val accessClosedSessionWithApi: SessionAPI? = null, - private val retryClose: Boolean = false): FlowLogic() { + private val responderReaction: ResponderReaction): FlowLogic() { @CordaSerializable enum class SessionAPI { SEND, SEND_AND_RECEIVE, RECEIVE, - GET_FLOW_INFO + GET_FLOW_INFO, + CLOSE + } + + @CordaSerializable + enum class ResponderReaction { + NORMAL_CLOSE, + RETRY_CLOSE_FROM_CHECKPOINT } @Suspendable @@ -169,7 +184,7 @@ class FlowSessionCloseTest { session.close() } - session.send(retryClose) + session.send(responderReaction) sleep(1.seconds) if (accessClosedSessionWithApi != null) { @@ -178,6 +193,7 @@ class FlowSessionCloseTest { SessionAPI.RECEIVE -> session.receive() SessionAPI.SEND_AND_RECEIVE -> session.sendAndReceive("dummy payload") SessionAPI.GET_FLOW_INFO -> session.getCounterpartyFlowInfo() + SessionAPI.CLOSE -> session.close() } } } @@ -192,16 +208,21 @@ class FlowSessionCloseTest { @Suspendable override fun call() { - val retryClose = otherSideSession.receive() + val responderReaction = otherSideSession.receive() .unwrap{ it } - otherSideSession.close() + when(responderReaction) { + InitiatorFlow.ResponderReaction.NORMAL_CLOSE -> { + otherSideSession.close() + } + InitiatorFlow.ResponderReaction.RETRY_CLOSE_FROM_CHECKPOINT -> { + otherSideSession.close() - // failing with a transient exception to force a replay of the close. - if (retryClose) { - if (!thrown) { - thrown = true - throw SQLTransientConnectionException("Connection is not available") + // failing with a transient exception to force a replay of the close. + if (!thrown) { + thrown = true + throw SQLTransientConnectionException("Connection is not available") + } } } } From 1e6be340eb3516fa8b477a6071a2cf813065c3ff Mon Sep 17 00:00:00 2001 From: Dimitris Gounaris <17044221+dgounaris@users.noreply.github.com> Date: Tue, 28 Jul 2020 17:02:53 +0300 Subject: [PATCH 43/48] CORDA-3844: bulk node infos request (#6411) * CORDA-3844: Add new functions to network map client * CORDA-3844: Apply new fetch logic to nm updater * CORDA-3844: Fix base url and warnings * CORDA-3844: Change response object and response validation In order to make sure that the returned node infos are not maliciously modified, either a signed list response or a signed reference object would need to be provided. As providing a signed list requires a lot of effort from NM and Signer services, the signed network map is provided instead, allowing nodes to validate that the list provided conforms to the entries of the signed network map. * CORDA-3844: Add clarifications and comments * CORDA-3844: Add error handling for bulk request * CORDA-3844: Enhance testing * CORDA-3844: Fix detekt issues * EG-3844: Apply pr suggestions --- .../node/services/network/NetworkMapClient.kt | 24 ++++++- .../services/network/NetworkMapUpdater.kt | 70 +++++++++++++----- .../HTTPNetworkRegistrationService.kt | 5 ++ .../services/network/NetworkMapClientTest.kt | 23 ++++++ .../services/network/NetworkMapUpdaterTest.kt | 71 +++++++++++++++++++ .../node/internal/network/NetworkMapServer.kt | 16 ++++- 6 files changed, 189 insertions(+), 20 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt index 91a0e159c6..de1cccac8e 100644 --- a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapClient.kt @@ -2,6 +2,7 @@ package net.corda.node.services.network import net.corda.core.crypto.SecureHash import net.corda.core.crypto.SignedData +import net.corda.core.crypto.sha256 import net.corda.core.internal.openHttpConnection import net.corda.core.internal.post import net.corda.core.internal.responseAs @@ -13,6 +14,7 @@ import net.corda.core.utilities.seconds import net.corda.core.utilities.trace import net.corda.node.VersionInfo import net.corda.node.utilities.registration.cacheControl +import net.corda.node.utilities.registration.cordaServerVersion import net.corda.nodeapi.internal.SignedNodeInfo import net.corda.nodeapi.internal.network.NetworkMap import net.corda.nodeapi.internal.network.SignedNetworkMap @@ -61,8 +63,9 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val versionInfo: Versi val signedNetworkMap = connection.responseAs() val networkMap = signedNetworkMap.verifiedNetworkMapCert(trustRoot) val timeout = connection.cacheControl.maxAgeSeconds().seconds + val version = connection.cordaServerVersion logger.trace { "Fetched network map update from $url successfully: $networkMap" } - return NetworkMapResponse(networkMap, timeout) + return NetworkMapResponse(networkMap, timeout, version) } fun getNodeInfo(nodeInfoHash: SecureHash): NodeInfo { @@ -81,6 +84,23 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val versionInfo: Versi return networkParameter } + fun getNodeInfos(): List { + val url = URL("$networkMapUrl/node-infos") + logger.trace { "Fetching node infos from $url." } + val verifiedNodeInfo = url.openHttpConnection().responseAs>>() + .also { + val verifiedNodeInfoHashes = it.first.verifiedNetworkMapCert(trustRoot).nodeInfoHashes + val nodeInfoHashes = it.second.map { signedNodeInfo -> signedNodeInfo.verified().serialize().sha256() } + require( + verifiedNodeInfoHashes.containsAll(nodeInfoHashes) && + verifiedNodeInfoHashes.size == nodeInfoHashes.size + ) + } + .second.map { it.verified() } + logger.trace { "Fetched node infos successfully. Node Infos size: ${verifiedNodeInfo.size}" } + return verifiedNodeInfo + } + fun myPublicHostname(): String { val url = URL("$networkMapUrl/my-hostname") logger.trace { "Resolving public hostname from '$url'." } @@ -90,4 +110,4 @@ class NetworkMapClient(compatibilityZoneURL: URL, private val versionInfo: Versi } } -data class NetworkMapResponse(val payload: NetworkMap, val cacheMaxAge: Duration) +data class NetworkMapResponse(val payload: NetworkMap, val cacheMaxAge: Duration, val serverVersion: String) diff --git a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapUpdater.kt b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapUpdater.kt index 7ff18232a2..712efce341 100644 --- a/node/src/main/kotlin/net/corda/node/services/network/NetworkMapUpdater.kt +++ b/node/src/main/kotlin/net/corda/node/services/network/NetworkMapUpdater.kt @@ -4,6 +4,7 @@ import com.google.common.util.concurrent.MoreExecutors import net.corda.core.CordaRuntimeException import net.corda.core.crypto.SecureHash import net.corda.core.crypto.SignedData +import net.corda.core.crypto.sha256 import net.corda.core.internal.NetworkParametersStorage import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.copyTo @@ -65,6 +66,7 @@ class NetworkMapUpdater(private val networkMapCache: NetworkMapCacheInternal, companion object { private val logger = contextLogger() private val defaultRetryInterval = 1.minutes + private const val bulkNodeInfoFetchThreshold = 50 } private val parametersUpdatesTrack = PublishSubject.create() @@ -173,17 +175,9 @@ class NetworkMapUpdater(private val networkMapCache: NetworkMapCacheInternal, if (networkMapClient == null) { throw CordaRuntimeException("Network map cache can be updated only if network map/compatibility zone URL is specified") } - val (globalNetworkMap, cacheTimeout) = networkMapClient.getNetworkMap() + val (globalNetworkMap, cacheTimeout, version) = networkMapClient.getNetworkMap() globalNetworkMap.parametersUpdate?.let { handleUpdateNetworkParameters(networkMapClient, it) } - val additionalHashes = extraNetworkMapKeys.flatMap { - try { - networkMapClient.getNetworkMap(it).payload.nodeInfoHashes - } catch (e: Exception) { - // Failure to retrieve one network map using UUID shouldn't stop the whole update. - logger.warn("Error encountered when downloading network map with uuid '$it', skipping...", e) - emptyList() - } - } + val additionalHashes = getPrivateNetworkNodeHashes(version) val allHashesFromNetworkMap = (globalNetworkMap.nodeInfoHashes + additionalHashes).toSet() if (currentParametersHash != globalNetworkMap.networkParameterHash) { exitOnParametersMismatch(globalNetworkMap) @@ -194,6 +188,37 @@ class NetworkMapUpdater(private val networkMapCache: NetworkMapCacheInternal, val allNodeHashes = networkMapCache.allNodeHashes val nodeHashesToBeDeleted = (allNodeHashes - allHashesFromNetworkMap - nodeInfoWatcher.processedNodeInfoHashes) .filter { it != ourNodeInfoHash } + // enforce bulk fetch when no other nodes are known or unknown nodes count is less than threshold + if (version == "1" || (allNodeHashes.size > 1 && (allHashesFromNetworkMap - allNodeHashes).size < bulkNodeInfoFetchThreshold)) + updateNodeInfosV1(allHashesFromNetworkMap, allNodeHashes, networkMapClient) + else + updateNodeInfos(allHashesFromNetworkMap) + // NOTE: We remove nodes after any new/updates because updated nodes will have a new hash and, therefore, any + // nodes that we can actually pull out of the cache (with the old hashes) should be a truly removed node. + nodeHashesToBeDeleted.mapNotNull { networkMapCache.getNodeByHash(it) }.forEach(networkMapCache::removeNode) + + // Mark the network map cache as ready on a successful poll of the HTTP network map, even on the odd chance that + // it's empty + networkMapCache.nodeReady.set(null) + return cacheTimeout + } + + private fun updateNodeInfos(allHashesFromNetworkMap: Set) { + val networkMapDownloadStartTime = System.currentTimeMillis() + val nodeInfos = try { + networkMapClient!!.getNodeInfos() + } catch (e: Exception) { + logger.warn("Error encountered when downloading node infos", e) + emptyList() + } + (allHashesFromNetworkMap - nodeInfos.map { it.serialize().sha256() }).forEach { + logger.warn("Error encountered when downloading node info '$it', skipping...") + } + networkMapCache.addOrUpdateNodes(nodeInfos) + logger.info("Fetched: ${nodeInfos.size} using 1 bulk request in ${System.currentTimeMillis() - networkMapDownloadStartTime}ms") + } + + private fun updateNodeInfosV1(allHashesFromNetworkMap: Set, allNodeHashes: List, networkMapClient: NetworkMapClient) { //at the moment we use a blocking HTTP library - but under the covers, the OS will interleave threads waiting for IO //as HTTP GET is mostly IO bound, use more threads than CPU's //maximum threads to use = 24, as if we did not limit this on large machines it could result in 100's of concurrent requests @@ -230,14 +255,25 @@ class NetworkMapUpdater(private val networkMapCache: NetworkMapCacheInternal, executorToUseForInsertionIntoDB.shutdown() }.getOrThrow() } - // NOTE: We remove nodes after any new/updates because updated nodes will have a new hash and, therefore, any - // nodes that we can actually pull out of the cache (with the old hashes) should be a truly removed node. - nodeHashesToBeDeleted.mapNotNull { networkMapCache.getNodeByHash(it) }.forEach(networkMapCache::removeNode) + } - // Mark the network map cache as ready on a successful poll of the HTTP network map, even on the odd chance that - // it's empty - networkMapCache.nodeReady.set(null) - return cacheTimeout + private fun getPrivateNetworkNodeHashes(version: String): List { + // private networks are not supported by latest versions of Network Map + // for compatibility reasons, this call is still present for new nodes that communicate with old Network Map service versions + // but can be omitted if we know that the version of the Network Map is recent enough + return if (version == "1") { + extraNetworkMapKeys.flatMap { + try { + networkMapClient!!.getNetworkMap(it).payload.nodeInfoHashes + } catch (e: Exception) { + // Failure to retrieve one network map using UUID shouldn't stop the whole update. + logger.warn("Error encountered when downloading network map with uuid '$it', skipping...", e) + emptyList() + } + } + } else { + emptyList() + } } private fun exitOnParametersMismatch(networkMap: NetworkMap) { diff --git a/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt b/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt index 41ea0edd25..653f6fdaf9 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/registration/HTTPNetworkRegistrationService.kt @@ -69,3 +69,8 @@ val HttpURLConnection.cacheControl: CacheControl get() { return CacheControl.parse(Headers.of(headerFields.filterKeys { it != null }.mapValues { it.value[0] })) } + +val HttpURLConnection.cordaServerVersion: String + get() { + return headerFields["X-Corda-Server-Version"]?.singleOrNull() ?: "1" + } \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt index 9fa13bf67e..ab42bd19fd 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt @@ -72,6 +72,29 @@ class NetworkMapClientTest { assertEquals(nodeInfo2, networkMapClient.getNodeInfo(nodeInfoHash2)) } + @Test(timeout=300_000) + fun `registered node is added to the network map v2`() { + server.version = "2" + val (nodeInfo, signedNodeInfo) = createNodeInfoAndSigned(ALICE_NAME) + + networkMapClient.publish(signedNodeInfo) + + val nodeInfoHash = nodeInfo.serialize().sha256() + + assertThat(networkMapClient.getNetworkMap().payload.nodeInfoHashes).containsExactly(nodeInfoHash) + assertEquals(nodeInfo, networkMapClient.getNodeInfos().single()) + + val (nodeInfo2, signedNodeInfo2) = createNodeInfoAndSigned(BOB_NAME) + + networkMapClient.publish(signedNodeInfo2) + + val nodeInfoHash2 = nodeInfo2.serialize().sha256() + assertThat(networkMapClient.getNetworkMap().payload.nodeInfoHashes).containsExactly(nodeInfoHash, nodeInfoHash2) + assertEquals(cacheTimeout, networkMapClient.getNetworkMap().cacheMaxAge) + assertEquals("2", networkMapClient.getNetworkMap().serverVersion) + assertThat(networkMapClient.getNodeInfos()).containsExactlyInAnyOrder(nodeInfo, nodeInfo2) + } + @Test(timeout=300_000) fun `negative test - registered invalid node is added to the network map`() { val invalidLongNodeName = CordaX500Name( diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt index d2689ce039..a406bd9be6 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt @@ -3,6 +3,7 @@ package net.corda.node.services.network import com.google.common.jimfs.Configuration.unix import com.google.common.jimfs.Jimfs import com.nhaarman.mockito_kotlin.any +import com.nhaarman.mockito_kotlin.atLeast import com.nhaarman.mockito_kotlin.mock import com.nhaarman.mockito_kotlin.never import com.nhaarman.mockito_kotlin.times @@ -10,6 +11,7 @@ import com.nhaarman.mockito_kotlin.verify import net.corda.core.crypto.Crypto import net.corda.core.crypto.SecureHash import net.corda.core.crypto.generateKeyPair +import net.corda.core.crypto.sha256 import net.corda.core.crypto.sign import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party @@ -383,6 +385,75 @@ class NetworkMapUpdaterTest { assertEquals(aliceInfo, networkMapClient.getNodeInfo(aliceHash)) } + @Test(timeout=300_000) + fun `update nodes is successful for network map supporting bulk operations but with only a few nodes requested`() { + server.version = "2" + setUpdater() + // on first update, bulk request is used + val (nodeInfo1, signedNodeInfo1) = createNodeInfoAndSigned("info1") + val nodeInfoHash1 = nodeInfo1.serialize().sha256() + val (nodeInfo2, signedNodeInfo2) = createNodeInfoAndSigned("info2") + val nodeInfoHash2 = nodeInfo2.serialize().sha256() + networkMapClient.publish(signedNodeInfo1) + networkMapClient.publish(signedNodeInfo2) + + startUpdater() + + Thread.sleep(2L * cacheExpiryMs) + verify(networkMapCache, times(1)).addOrUpdateNodes(listOf(nodeInfo1, nodeInfo2)) + assertThat(networkMapCache.allNodeHashes).containsExactlyInAnyOrder(nodeInfoHash1, nodeInfoHash2) + + // on subsequent updates, single requests are used + val (nodeInfo3, signedNodeInfo3) = createNodeInfoAndSigned("info3") + val nodeInfoHash3 = nodeInfo3.serialize().sha256() + val (nodeInfo4, signedNodeInfo4) = createNodeInfoAndSigned("info4") + val nodeInfoHash4 = nodeInfo4.serialize().sha256() + networkMapClient.publish(signedNodeInfo3) + networkMapClient.publish(signedNodeInfo4) + + Thread.sleep(2L * cacheExpiryMs) + verify(networkMapCache, times(1)).addOrUpdateNodes(listOf(nodeInfo3)) + verify(networkMapCache, times(1)).addOrUpdateNodes(listOf(nodeInfo4)) + assertThat(networkMapCache.allNodeHashes).containsExactlyInAnyOrder(nodeInfoHash1, nodeInfoHash2, nodeInfoHash3, nodeInfoHash4) + } + + @Test(timeout=300_000) + @SuppressWarnings("SpreadOperator") + fun `update nodes is successful for network map supporting bulk operations when high number of nodes is requested`() { + server.version = "2" + setUpdater() + val nodeInfos = (1..51).map { createNodeInfoAndSigned("info$it") + .also { nodeInfoAndSigned -> networkMapClient.publish(nodeInfoAndSigned.signed) } + .nodeInfo + } + val nodeInfoHashes = nodeInfos.map { it.serialize().sha256() } + + startUpdater() + Thread.sleep(2L * cacheExpiryMs) + + verify(networkMapCache, times(1)).addOrUpdateNodes(nodeInfos) + assertThat(networkMapCache.allNodeHashes).containsExactlyInAnyOrder(*(nodeInfoHashes.toTypedArray())) + } + + @Test(timeout=300_000) + @SuppressWarnings("SpreadOperator") + fun `update nodes is successful for network map not supporting bulk operations`() { + setUpdater() + val nodeInfos = (1..51).map { createNodeInfoAndSigned("info$it") + .also { nodeInfoAndSigned -> networkMapClient.publish(nodeInfoAndSigned.signed) } + .nodeInfo + } + val nodeInfoHashes = nodeInfos.map { it.serialize().sha256() } + + startUpdater() + Thread.sleep(2L * cacheExpiryMs) + + // we can't be sure about the number of requests (and updates), as it depends on the machine and the threads created + // but if they are more than 1 it's enough to deduct that the parallel way was favored + verify(networkMapCache, atLeast(2)).addOrUpdateNodes(any()) + assertThat(networkMapCache.allNodeHashes).containsExactlyInAnyOrder(*(nodeInfoHashes.toTypedArray())) + } + @Test(timeout=300_000) fun `remove node from filesystem deletes it from network map cache`() { setUpdater(netMapClient = null) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt index 0aa00f832c..88620bb1d7 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/network/NetworkMapServer.kt @@ -49,6 +49,8 @@ class NetworkMapServer(private val pollInterval: Duration, private val service = InMemoryNetworkMapService() private var parametersUpdate: ParametersUpdate? = null private var nextNetworkParameters: NetworkParameters? = null + // version toggle allowing to easily test behaviour of different version without spinning up a whole new server + var version: String = "1" init { server = Server(InetSocketAddress(hostAndPort.host, hostAndPort.port)).apply { @@ -171,7 +173,10 @@ class NetworkMapServer(private val pollInterval: Duration, private fun networkMapResponse(nodeInfoHashes: List): Response { val networkMap = NetworkMap(nodeInfoHashes, signedNetParams.raw.hash, parametersUpdate) val signedNetworkMap = networkMapCertAndKeyPair.sign(networkMap) - return Response.ok(signedNetworkMap.serialize().bytes).header("Cache-Control", "max-age=${pollInterval.seconds}").build() + return Response.ok(signedNetworkMap.serialize().bytes) + .header("Cache-Control", "max-age=${pollInterval.seconds}") + .apply { if (version != "1") this.header("X-Corda-Server-Version", version)} + .build() } // Remove nodeInfo for testing. @@ -205,6 +210,15 @@ class NetworkMapServer(private val pollInterval: Duration, }.build() } + @GET + @Path("node-infos") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + fun getNodeInfos(): Response { + val networkMap = NetworkMap(nodeInfoMap.keys.toList(), signedNetParams.raw.hash, parametersUpdate) + val signedNetworkMap = networkMapCertAndKeyPair.sign(networkMap) + return Response.ok(Pair(signedNetworkMap, nodeInfoMap.values.toList()).serialize().bytes).build() + } + @GET @Path("network-parameters/{var}") @Produces(MediaType.APPLICATION_OCTET_STREAM) From 52cbe04b8c439e7b2624e2dda7eca3853b61ffba Mon Sep 17 00:00:00 2001 From: Stefan Iliev <46542846+StefanIliev545@users.noreply.github.com> Date: Tue, 28 Jul 2020 15:50:19 +0100 Subject: [PATCH 44/48] EG-2375 - batching notary open sourcing. (#6507) --- .../vault/VaultObserverExceptionTest.kt | 6 +- .../net/corda/node/internal/AbstractNode.kt | 5 - .../node/internal/cordapp/VirtualCordapps.kt | 15 +- .../node/services/schema/NodeSchemaService.kt | 1 - .../transactions/SimpleNotaryService.kt | 49 --- .../net/corda/node/utilities/NotaryLoader.kt | 6 +- .../net/corda/notary/common/BatchSigning.kt | 54 +++ .../notary/jpa/JPANotaryConfiguration.kt | 9 + .../net/corda/notary/jpa/JPANotaryService.kt | 55 +++ .../corda/notary/jpa/JPAUniquenessProvider.kt | 408 ++++++++++++++++++ .../kotlin/net/corda/notary/jpa/Schema.kt | 18 + .../node-notary.changelog-master.xml | 2 + .../migration/node-notary.changelog-v3.xml | 48 +++ .../node-notary.changelog-worker-logging.xml | 14 + .../transactions/UniquenessProviderTests.kt | 51 ++- 15 files changed, 658 insertions(+), 83 deletions(-) delete mode 100644 node/src/main/kotlin/net/corda/node/services/transactions/SimpleNotaryService.kt create mode 100644 node/src/main/kotlin/net/corda/notary/common/BatchSigning.kt create mode 100644 node/src/main/kotlin/net/corda/notary/jpa/JPANotaryConfiguration.kt create mode 100644 node/src/main/kotlin/net/corda/notary/jpa/JPANotaryService.kt create mode 100644 node/src/main/kotlin/net/corda/notary/jpa/JPAUniquenessProvider.kt create mode 100644 node/src/main/kotlin/net/corda/notary/jpa/Schema.kt create mode 100644 node/src/main/resources/migration/node-notary.changelog-v3.xml create mode 100644 node/src/main/resources/migration/node-notary.changelog-worker-logging.xml diff --git a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt index 601134338f..d1d4cd7b23 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt @@ -25,7 +25,7 @@ import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.seconds import net.corda.node.services.Permissions import net.corda.node.services.statemachine.StaffedFlowHospital -import net.corda.node.services.transactions.PersistentUniquenessProvider +import net.corda.notary.jpa.JPAUniquenessProvider import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.singleIdentity @@ -856,8 +856,8 @@ class VaultObserverExceptionTest { override fun call(): List { return serviceHub.withEntityManager { val criteriaQuery = this.criteriaBuilder.createQuery(String::class.java) - val root = criteriaQuery.from(PersistentUniquenessProvider.CommittedTransaction::class.java) - criteriaQuery.select(root.get(PersistentUniquenessProvider.CommittedTransaction::transactionId.name)) + val root = criteriaQuery.from(JPAUniquenessProvider.CommittedTransaction::class.java) + criteriaQuery.select(root.get(JPAUniquenessProvider.CommittedTransaction::transactionId.name)) val query = this.createQuery(criteriaQuery) query.resultList } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 59b6b4fca7..2c1c9563fa 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -133,7 +133,6 @@ import net.corda.node.services.statemachine.StateMachineManager import net.corda.node.services.transactions.BasicVerifierFactoryService import net.corda.node.services.transactions.DeterministicVerifierFactoryService import net.corda.node.services.transactions.InMemoryTransactionVerifierService -import net.corda.node.services.transactions.SimpleNotaryService import net.corda.node.services.transactions.VerifierFactoryService import net.corda.node.services.upgrade.ContractUpgradeServiceImpl import net.corda.node.services.vault.NodeVaultService @@ -792,10 +791,6 @@ abstract class AbstractNode(val configuration: NodeConfiguration, ) } - private fun isRunningSimpleNotaryService(configuration: NodeConfiguration): Boolean { - return configuration.notary != null && configuration.notary?.className == SimpleNotaryService::class.java.name - } - private class ServiceInstantiationException(cause: Throwable?) : CordaException("Service Instantiation Error", cause) private fun installCordaServices() { diff --git a/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt b/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt index 5ad5add351..72c9b0a90a 100644 --- a/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt +++ b/node/src/main/kotlin/net/corda/node/internal/cordapp/VirtualCordapps.kt @@ -6,12 +6,12 @@ import net.corda.core.flows.ContractUpgradeFlow import net.corda.core.internal.cordapp.CordappImpl import net.corda.core.internal.location import net.corda.node.VersionInfo -import net.corda.node.services.transactions.NodeNotarySchemaV1 -import net.corda.node.services.transactions.SimpleNotaryService import net.corda.notary.experimental.bftsmart.BFTSmartNotarySchemaV1 import net.corda.notary.experimental.bftsmart.BFTSmartNotaryService import net.corda.notary.experimental.raft.RaftNotarySchemaV1 import net.corda.notary.experimental.raft.RaftNotaryService +import net.corda.notary.jpa.JPANotarySchemaV1 +import net.corda.notary.jpa.JPANotaryService internal object VirtualCordapp { /** A list of the core RPC flows present in Corda */ @@ -46,7 +46,7 @@ internal object VirtualCordapp { } /** A Cordapp for the built-in notary service implementation. */ - fun generateSimpleNotary(versionInfo: VersionInfo): CordappImpl { + fun generateJPANotary(versionInfo: VersionInfo): CordappImpl { return CordappImpl( contractClassNames = listOf(), initiatedFlows = listOf(), @@ -57,15 +57,16 @@ internal object VirtualCordapp { serializationWhitelists = listOf(), serializationCustomSerializers = listOf(), checkpointCustomSerializers = listOf(), - customSchemas = setOf(NodeNotarySchemaV1), + customSchemas = setOf(JPANotarySchemaV1), info = Cordapp.Info.Default("corda-notary", versionInfo.vendor, versionInfo.releaseVersion, "Open Source (Apache 2)"), allFlows = listOf(), - jarPath = SimpleNotaryService::class.java.location, + jarPath = JPANotaryService::class.java.location, jarHash = SecureHash.allOnesHash, minimumPlatformVersion = versionInfo.platformVersion, targetPlatformVersion = versionInfo.platformVersion, - notaryService = SimpleNotaryService::class.java, - isLoaded = false + notaryService = JPANotaryService::class.java, + isLoaded = false, + isVirtual = true ) } diff --git a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt index d38c6371ef..3244385d04 100644 --- a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt +++ b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt @@ -66,7 +66,6 @@ class NodeSchemaService(private val extraSchemas: Set = emptySet() // when mapped schemas from the finance module are present, they are considered as internal ones schema::class.qualifiedName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.qualifiedName == "net.corda.finance.schemas.CommercialPaperSchemaV1" || - schema::class.qualifiedName == "net.corda.node.services.transactions.NodeNotarySchemaV1" || schema::class.qualifiedName?.startsWith("net.corda.notary.") ?: false } diff --git a/node/src/main/kotlin/net/corda/node/services/transactions/SimpleNotaryService.kt b/node/src/main/kotlin/net/corda/node/services/transactions/SimpleNotaryService.kt deleted file mode 100644 index 055cadab84..0000000000 --- a/node/src/main/kotlin/net/corda/node/services/transactions/SimpleNotaryService.kt +++ /dev/null @@ -1,49 +0,0 @@ -package net.corda.node.services.transactions - -import net.corda.core.flows.FlowSession -import net.corda.core.internal.notary.SinglePartyNotaryService -import net.corda.core.internal.notary.NotaryServiceFlow -import net.corda.core.schemas.MappedSchema -import net.corda.core.utilities.seconds -import net.corda.node.services.api.ServiceHubInternal -import java.security.PublicKey - -/** An embedded notary service that uses the node's database to store committed states. */ -class SimpleNotaryService(override val services: ServiceHubInternal, override val notaryIdentityKey: PublicKey) : SinglePartyNotaryService() { - private val notaryConfig = services.configuration.notary - ?: throw IllegalArgumentException("Failed to register ${this::class.java}: notary configuration not present") - - init { - val mode = if (notaryConfig.validating) "validating" else "non-validating" - log.info("Starting notary in $mode mode") - } - - override val uniquenessProvider = PersistentUniquenessProvider( - services.clock, - services.database, - services.cacheFactory, - ::signTransaction) - - override fun createServiceFlow(otherPartySession: FlowSession): NotaryServiceFlow { - return if (notaryConfig.validating) { - ValidatingNotaryFlow(otherPartySession, this, notaryConfig.etaMessageThresholdSeconds.seconds) - } else { - NonValidatingNotaryFlow(otherPartySession, this, notaryConfig.etaMessageThresholdSeconds.seconds) - } - } - - override fun start() {} - override fun stop() {} -} - -// Entities used by a Notary -object NodeNotarySchema - -object NodeNotarySchemaV1 : MappedSchema(schemaFamily = NodeNotarySchema.javaClass, version = 1, - mappedTypes = listOf(PersistentUniquenessProvider.BaseComittedState::class.java, - PersistentUniquenessProvider.Request::class.java, - PersistentUniquenessProvider.CommittedState::class.java, - PersistentUniquenessProvider.CommittedTransaction::class.java - )) { - override val migrationResource = "node-notary.changelog-master" -} diff --git a/node/src/main/kotlin/net/corda/node/utilities/NotaryLoader.kt b/node/src/main/kotlin/net/corda/node/utilities/NotaryLoader.kt index 749440431f..75fa2efb24 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/NotaryLoader.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/NotaryLoader.kt @@ -9,10 +9,10 @@ import net.corda.node.VersionInfo import net.corda.node.internal.cordapp.VirtualCordapp import net.corda.node.services.api.ServiceHubInternal import net.corda.node.services.config.NotaryConfig -import net.corda.node.services.transactions.SimpleNotaryService import net.corda.nodeapi.internal.cordapp.CordappLoader import net.corda.notary.experimental.bftsmart.BFTSmartNotaryService import net.corda.notary.experimental.raft.RaftNotaryService +import net.corda.notary.jpa.JPANotaryService import java.lang.reflect.InvocationTargetException import java.security.PublicKey @@ -44,8 +44,8 @@ class NotaryLoader( RaftNotaryService::class.java } else -> { - builtInNotary = VirtualCordapp.generateSimpleNotary(versionInfo) - SimpleNotaryService::class.java + builtInNotary = VirtualCordapp.generateJPANotary(versionInfo) + JPANotaryService::class.java } } } else { diff --git a/node/src/main/kotlin/net/corda/notary/common/BatchSigning.kt b/node/src/main/kotlin/net/corda/notary/common/BatchSigning.kt new file mode 100644 index 0000000000..4fc3bb4e25 --- /dev/null +++ b/node/src/main/kotlin/net/corda/notary/common/BatchSigning.kt @@ -0,0 +1,54 @@ +package net.corda.notary.common + +import net.corda.core.crypto.Crypto +import net.corda.core.crypto.MerkleTree +import net.corda.core.crypto.PartialMerkleTree +import net.corda.core.crypto.SecureHash +import net.corda.core.crypto.SignableData +import net.corda.core.crypto.SignatureMetadata +import net.corda.core.crypto.TransactionSignature +import net.corda.core.crypto.sha256 +import net.corda.core.flows.NotaryError +import net.corda.core.node.ServiceHub +import java.security.PublicKey + +typealias BatchSigningFunction = (Iterable) -> BatchSignature + +/** Generates a signature over the bach of [txIds]. */ +fun signBatch( + txIds: Iterable, + notaryIdentityKey: PublicKey, + services: ServiceHub +): BatchSignature { + val merkleTree = MerkleTree.getMerkleTree(txIds.map { it.sha256() }) + val merkleTreeRoot = merkleTree.hash + val signableData = SignableData( + merkleTreeRoot, + SignatureMetadata( + services.myInfo.platformVersion, + Crypto.findSignatureScheme(notaryIdentityKey).schemeNumberID + ) + ) + val sig = services.keyManagementService.sign(signableData, notaryIdentityKey) + return BatchSignature(sig, merkleTree) +} + +/** The outcome of just committing a transaction. */ +sealed class InternalResult { + object Success : InternalResult() + data class Failure(val error: NotaryError) : InternalResult() +} + +data class BatchSignature( + val rootSignature: TransactionSignature, + val fullMerkleTree: MerkleTree) { + /** Extracts a signature with a partial Merkle tree for the specified leaf in the batch signature. */ + fun forParticipant(txId: SecureHash): TransactionSignature { + return TransactionSignature( + rootSignature.bytes, + rootSignature.by, + rootSignature.signatureMetadata, + PartialMerkleTree.build(fullMerkleTree, listOf(txId.sha256())) + ) + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryConfiguration.kt b/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryConfiguration.kt new file mode 100644 index 0000000000..39c8f4dff9 --- /dev/null +++ b/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryConfiguration.kt @@ -0,0 +1,9 @@ +package net.corda.notary.jpa + +data class JPANotaryConfiguration( + val batchSize: Int = 32, + val batchTimeoutMs: Long = 200L, + val maxInputStates: Int = 2000, + val maxDBTransactionRetryCount: Int = 10, + val backOffBaseMs: Long = 20L +) diff --git a/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryService.kt b/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryService.kt new file mode 100644 index 0000000000..6db10a8333 --- /dev/null +++ b/node/src/main/kotlin/net/corda/notary/jpa/JPANotaryService.kt @@ -0,0 +1,55 @@ +package net.corda.notary.jpa + +import net.corda.core.crypto.SecureHash +import net.corda.core.flows.FlowSession +import net.corda.core.internal.notary.NotaryServiceFlow +import net.corda.core.internal.notary.SinglePartyNotaryService +import net.corda.core.utilities.seconds +import net.corda.node.services.api.ServiceHubInternal +import net.corda.node.services.transactions.NonValidatingNotaryFlow +import net.corda.node.services.transactions.ValidatingNotaryFlow +import net.corda.nodeapi.internal.config.parseAs +import net.corda.notary.common.signBatch +import java.security.PublicKey + +/** Notary service backed by a relational database. */ +class JPANotaryService( + override val services: ServiceHubInternal, + override val notaryIdentityKey: PublicKey) : SinglePartyNotaryService() { + + private val notaryConfig = services.configuration.notary + ?: throw IllegalArgumentException("Failed to register ${this::class.java}: notary configuration not present") + + + @Suppress("TooGenericExceptionCaught") + override val uniquenessProvider = with(services) { + val jpaNotaryConfig = try { + notaryConfig.extraConfig?.parseAs() ?: JPANotaryConfiguration() + } catch (e: Exception) { + throw IllegalArgumentException("Failed to register ${JPANotaryService::class.java}: extra notary configuration parameters invalid") + } + JPAUniquenessProvider( + clock, + database, + jpaNotaryConfig, + configuration.myLegalName, + ::signTransactionBatch + ) + } + + private fun signTransactionBatch(txIds: Iterable) + = signBatch(txIds, notaryIdentityKey, services) + + override fun createServiceFlow(otherPartySession: FlowSession): NotaryServiceFlow { + return if (notaryConfig.validating) { + ValidatingNotaryFlow(otherPartySession, this, notaryConfig.etaMessageThresholdSeconds.seconds) + } else NonValidatingNotaryFlow(otherPartySession, this, notaryConfig.etaMessageThresholdSeconds.seconds) + } + + override fun start() { + } + + override fun stop() { + uniquenessProvider.stop() + } +} diff --git a/node/src/main/kotlin/net/corda/notary/jpa/JPAUniquenessProvider.kt b/node/src/main/kotlin/net/corda/notary/jpa/JPAUniquenessProvider.kt new file mode 100644 index 0000000000..53ea1749fd --- /dev/null +++ b/node/src/main/kotlin/net/corda/notary/jpa/JPAUniquenessProvider.kt @@ -0,0 +1,408 @@ +package net.corda.notary.jpa + +import com.google.common.collect.Queues +import net.corda.core.concurrent.CordaFuture +import net.corda.core.contracts.StateRef +import net.corda.core.contracts.TimeWindow +import net.corda.core.crypto.SecureHash +import net.corda.core.crypto.sha256 +import net.corda.core.flows.NotarisationRequestSignature +import net.corda.core.flows.NotaryError +import net.corda.core.flows.StateConsumptionDetails +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.OpenFuture +import net.corda.core.internal.concurrent.openFuture +import net.corda.notary.common.BatchSigningFunction +import net.corda.core.internal.notary.NotaryInternalException +import net.corda.core.internal.notary.UniquenessProvider +import net.corda.core.internal.notary.isConsumedByTheSameTx +import net.corda.core.internal.notary.validateTimeWindow +import net.corda.core.schemas.PersistentStateRef +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializationDefaults +import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.serialization.serialize +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.nodeapi.internal.persistence.CordaPersistence +import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX +import net.corda.notary.common.InternalResult +import net.corda.serialization.internal.CordaSerializationEncoding +import org.hibernate.Session +import java.sql.SQLException +import java.time.Clock +import java.time.Instant +import java.util.* +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import javax.annotation.concurrent.ThreadSafe +import javax.persistence.Column +import javax.persistence.EmbeddedId +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Lob +import javax.persistence.NamedQuery +import kotlin.concurrent.thread + +/** A JPA backed Uniqueness provider */ +@Suppress("MagicNumber") // database column length +@ThreadSafe +class JPAUniquenessProvider( + val clock: Clock, + val database: CordaPersistence, + val config: JPANotaryConfiguration = JPANotaryConfiguration(), + val notaryWorkerName: CordaX500Name, + val signBatch: BatchSigningFunction +) : UniquenessProvider, SingletonSerializeAsToken() { + + // This is the prefix of the ID in the request log table, to allow running multiple instances that access the + // same table. + private val instanceId = UUID.randomUUID() + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}notary_request_log") + @CordaSerializable + class Request( + @Id + @Column(nullable = true, length = 76) + var id: String? = null, + + @Column(name = "consuming_transaction_id", nullable = true, length = 64) + val consumingTxHash: String?, + + @Column(name = "requesting_party_name", nullable = true, length = 255) + var partyName: String?, + + @Lob + @Column(name = "request_signature", nullable = false) + val requestSignature: ByteArray, + + @Column(name = "request_timestamp", nullable = false) + var requestDate: Instant, + + @Column(name = "worker_node_x500_name", nullable = true, length = 255) + val workerNodeX500Name: String? + ) + + private data class CommitRequest( + val states: List, + val txId: SecureHash, + val callerIdentity: Party, + val requestSignature: NotarisationRequestSignature, + val timeWindow: TimeWindow?, + val references: List, + val future: OpenFuture, + val requestEntity: Request, + val committedStatesEntities: List) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}notary_committed_states") + @NamedQuery(name = "CommittedState.select", query = "SELECT c from JPAUniquenessProvider\$CommittedState c WHERE c.id in :ids") + class CommittedState( + @EmbeddedId + val id: PersistentStateRef, + @Column(name = "consuming_transaction_id", nullable = false, length = 64) + val consumingTxHash: String) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}notary_committed_txs") + class CommittedTransaction( + @Id + @Column(name = "transaction_id", nullable = false, length = 64) + val transactionId: String + ) + + private val requestQueue = LinkedBlockingQueue(requestQueueSize) + + /** A requestEntity processor thread. */ + private val processorThread = thread(name = "Notary request queue processor", isDaemon = true) { + try { + val buffer = LinkedList() + while (!Thread.interrupted()) { + val drainedSize = Queues.drain(requestQueue, buffer, config.batchSize, config.batchTimeoutMs, TimeUnit.MILLISECONDS) + if (drainedSize == 0) continue + processRequests(buffer) + buffer.clear() + } + } catch (_: InterruptedException) { + log.debug { "Process interrupted."} + } + log.debug { "Shutting down with ${requestQueue.size} in-flight requests unprocessed." } + } + + fun stop() { + processorThread.interrupt() + } + + companion object { + private const val requestQueueSize = 100_000 + private const val jdbcBatchSize = 100_000 + private val log = contextLogger() + + fun encodeStateRef(s: StateRef): PersistentStateRef { + return PersistentStateRef(s.txhash.toString(), s.index) + } + + fun decodeStateRef(s: PersistentStateRef): StateRef { + return StateRef(txhash = SecureHash.parse(s.txId), index = s.index) + } + } + + /** + * Generates and adds a [CommitRequest] to the requestEntity queue. If the requestEntity queue is full, this method will block + * until space is available. + * + * Returns a future that will complete once the requestEntity is processed, containing the commit [Result]. + */ + override fun commit( + states: List, + txId: SecureHash, + callerIdentity: Party, + requestSignature: NotarisationRequestSignature, + timeWindow: TimeWindow?, + references: List + ): CordaFuture { + val future = openFuture() + val requestEntities = Request(consumingTxHash = txId.toString(), + partyName = callerIdentity.name.toString(), + requestSignature = requestSignature.serialize(context = SerializationDefaults.STORAGE_CONTEXT.withEncoding(CordaSerializationEncoding.SNAPPY)).bytes, + requestDate = clock.instant(), + workerNodeX500Name = notaryWorkerName.toString()) + val stateEntities = states.map { + CommittedState( + encodeStateRef(it), + txId.toString() + ) + } + val request = CommitRequest(states, txId, callerIdentity, requestSignature, timeWindow, references, future, requestEntities, stateEntities) + + requestQueue.put(request) + + return future + } + + // Safe up to 100k requests per second. + private var nextRequestId = System.currentTimeMillis() * 100 + + private fun logRequests(requests: List) { + database.transaction { + for (request in requests) { + request.requestEntity.id = "$instanceId:${(nextRequestId++).toString(16)}" + session.persist(request.requestEntity) + } + } + } + + private fun commitRequests(session: Session, requests: List) { + for (request in requests) { + for (cs in request.committedStatesEntities) { + session.persist(cs) + } + session.persist(CommittedTransaction(request.txId.toString())) + } + } + + private fun findAlreadyCommitted(session: Session, states: List, references: List): Map { + val persistentStateRefs = (states + references).map { encodeStateRef(it) }.toSet() + val committedStates = mutableListOf() + + for (idsBatch in persistentStateRefs.chunked(config.maxInputStates)) { + @Suppress("UNCHECKED_CAST") + val existing = session + .createNamedQuery("CommittedState.select") + .setParameter("ids", idsBatch) + .resultList as List + committedStates.addAll(existing) + } + + return committedStates.map { + val stateRef = StateRef(txhash = SecureHash.parse(it.id.txId), index = it.id.index) + val consumingTxId = SecureHash.parse(it.consumingTxHash) + if (stateRef in references) { + stateRef to StateConsumptionDetails(consumingTxId.sha256(), type = StateConsumptionDetails.ConsumedStateType.REFERENCE_INPUT_STATE) + } else { + stateRef to StateConsumptionDetails(consumingTxId.sha256()) + } + }.toMap() + } + + private fun withRetry(block: () -> T): T { + var retryCount = 0 + var backOff = config.backOffBaseMs + var exceptionCaught: SQLException? = null + while (retryCount <= config.maxDBTransactionRetryCount) { + try { + val res = block() + return res + } catch (e: SQLException) { + retryCount++ + Thread.sleep(backOff) + backOff *= 2 + exceptionCaught = e + } + } + throw exceptionCaught!! + } + + private fun findAllConflicts(session: Session, requests: List): MutableMap { + log.info("Processing notarization requests with ${requests.sumBy { it.states.size }} input states and ${requests.sumBy { it.references.size }} references") + + val allStates = requests.flatMap { it.states } + val allReferences = requests.flatMap { it.references } + return findAlreadyCommitted(session, allStates, allReferences).toMutableMap() + } + + private fun processRequest( + session: Session, + request: CommitRequest, + consumedStates: MutableMap, + processedTxIds: MutableMap, + toCommit: MutableList + ): InternalResult { + val conflicts = (request.states + request.references).mapNotNull { + if (consumedStates.containsKey(it)) it to consumedStates[it]!! + else null + }.toMap() + + return if (conflicts.isNotEmpty()) { + handleStateConflicts(request, conflicts, session) + } else { + handleNoStateConflicts(request, toCommit, consumedStates, processedTxIds, session) + } + } + + /** + * Process the [request] given there are conflicting states already present in the DB or current batch. + * + * To ensure idempotency, if the request's transaction matches a previously consumed transaction then the + * same result (success) can be returned without committing it to the DB. Failure is only returned in the + * case where the request is not a duplicate of a previously processed request and hence it is a genuine + * double spend attempt. + */ + private fun handleStateConflicts( + request: CommitRequest, + stateConflicts: Map, + session: Session + ): InternalResult { + return when { + isConsumedByTheSameTx(request.txId.sha256(), stateConflicts) -> { + InternalResult.Success + } + request.states.isEmpty() && isPreviouslyNotarised(session, request.txId) -> { + InternalResult.Success + } + else -> { + InternalResult.Failure(NotaryError.Conflict(request.txId, stateConflicts)) + } + } + } + + /** + * Process the [request] given there are no conflicting states already present in the DB or current batch. + * + * This method performs time window validation and adds the request to the commit list if applicable. + * It also checks the [processedTxIds] map to ensure that any time-window only duplicates within the batch + * are only committed once. + */ + private fun handleNoStateConflicts( + request: CommitRequest, + toCommit: MutableList, + consumedStates: MutableMap, + processedTxIds: MutableMap, + session: Session + ): InternalResult { + return when { + request.states.isEmpty() && isPreviouslyNotarised(session, request.txId) -> { + InternalResult.Success + } + processedTxIds.containsKey(request.txId) -> { + processedTxIds[request.txId]!! + } + else -> { + val outsideTimeWindowError = validateTimeWindow(clock.instant(), request.timeWindow) + val internalResult = if (outsideTimeWindowError != null) { + InternalResult.Failure(outsideTimeWindowError) + } else { + // Mark states as consumed to capture conflicting transactions in the same batch + request.states.forEach { + consumedStates[it] = StateConsumptionDetails(request.txId.sha256()) + } + toCommit.add(request) + InternalResult.Success + } + // Store transaction result to capture conflicting time-window only transactions in the same batch + processedTxIds[request.txId] = internalResult + internalResult + } + } + } + + private fun isPreviouslyNotarised(session: Session, txId: SecureHash): Boolean { + return session.find(CommittedTransaction::class.java, txId.toString()) != null + } + + @Suppress("TooGenericExceptionCaught") + private fun processRequests(requests: List) { + try { + // Note that there is an additional retry mechanism within the transaction itself. + val res = withRetry { + database.transaction { + val em = session.entityManagerFactory.createEntityManager() + em.unwrap(Session::class.java).jdbcBatchSize = jdbcBatchSize + + val toCommit = mutableListOf() + val consumedStates = findAllConflicts(session, requests) + val processedTxIds = mutableMapOf() + + val results = requests.map { request -> + processRequest(session, request, consumedStates, processedTxIds, toCommit) + } + + logRequests(requests) + commitRequests(session, toCommit) + + results + } + } + completeResponses(requests, res) + } catch (e: Exception) { + log.warn("Error processing commit requests", e) + for (request in requests) { + respondWithError(request, e) + } + } + } + + private fun completeResponses(requests: List, results: List): Int { + val zippedResults = requests.zip(results) + val successfulRequests = zippedResults + .filter { it.second is InternalResult.Success } + .map { it.first.txId } + .distinct() + val signature = if (successfulRequests.isNotEmpty()) + signBatch(successfulRequests) + else null + + var inputStateCount = 0 + for ((request, result) in zippedResults) { + val resultToSet = when { + result is InternalResult.Failure -> UniquenessProvider.Result.Failure(result.error) + signature != null -> UniquenessProvider.Result.Success(signature.forParticipant(request.txId)) + else -> throw IllegalStateException("Signature is required but not found") + } + + request.future.set(resultToSet) + inputStateCount += request.states.size + } + return inputStateCount + } + + private fun respondWithError(request: CommitRequest, exception: Exception) { + if (exception is NotaryInternalException) { + request.future.set(UniquenessProvider.Result.Failure(exception.error)) + } else { + request.future.setException(NotaryInternalException(NotaryError.General(Exception("Internal service error.")))) + } + } +} diff --git a/node/src/main/kotlin/net/corda/notary/jpa/Schema.kt b/node/src/main/kotlin/net/corda/notary/jpa/Schema.kt new file mode 100644 index 0000000000..209e894e48 --- /dev/null +++ b/node/src/main/kotlin/net/corda/notary/jpa/Schema.kt @@ -0,0 +1,18 @@ +package net.corda.notary.jpa + +import net.corda.core.schemas.MappedSchema + +object JPANotarySchema + +object JPANotarySchemaV1 : MappedSchema( + schemaFamily = JPANotarySchema.javaClass, + version = 1, + mappedTypes = listOf( + JPAUniquenessProvider.CommittedState::class.java, + JPAUniquenessProvider.Request::class.java, + JPAUniquenessProvider.CommittedTransaction::class.java + ) +) { + override val migrationResource: String? + get() = "node-notary.changelog-master" +} diff --git a/node/src/main/resources/migration/node-notary.changelog-master.xml b/node/src/main/resources/migration/node-notary.changelog-master.xml index d8a5a61b8c..506cd551ad 100644 --- a/node/src/main/resources/migration/node-notary.changelog-master.xml +++ b/node/src/main/resources/migration/node-notary.changelog-master.xml @@ -9,5 +9,7 @@ + + diff --git a/node/src/main/resources/migration/node-notary.changelog-v3.xml b/node/src/main/resources/migration/node-notary.changelog-v3.xml new file mode 100644 index 0000000000..596d0876d4 --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-v3.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + UPDATE node_notary_request_log SET id_temp = id + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-notary.changelog-worker-logging.xml b/node/src/main/resources/migration/node-notary.changelog-worker-logging.xml new file mode 100644 index 0000000000..096e6478f7 --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-worker-logging.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + diff --git a/node/src/test/kotlin/net/corda/node/services/transactions/UniquenessProviderTests.kt b/node/src/test/kotlin/net/corda/node/services/transactions/UniquenessProviderTests.kt index 07679e0f77..29d347b427 100644 --- a/node/src/test/kotlin/net/corda/node/services/transactions/UniquenessProviderTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/transactions/UniquenessProviderTests.kt @@ -4,6 +4,7 @@ import com.codahale.metrics.MetricRegistry import net.corda.core.contracts.TimeWindow import net.corda.core.crypto.Crypto import net.corda.core.crypto.DigitalSignature +import net.corda.core.crypto.MerkleTree import net.corda.core.crypto.NullKeys import net.corda.core.crypto.SecureHash import net.corda.core.crypto.SignableData @@ -21,9 +22,13 @@ import net.corda.node.services.schema.NodeSchemaService import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig +import net.corda.notary.common.BatchSignature import net.corda.notary.experimental.raft.RaftConfig import net.corda.notary.experimental.raft.RaftNotarySchemaV1 import net.corda.notary.experimental.raft.RaftUniquenessProvider +import net.corda.notary.jpa.JPANotaryConfiguration +import net.corda.notary.jpa.JPANotarySchemaV1 +import net.corda.notary.jpa.JPAUniquenessProvider import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity import net.corda.testing.core.generateStateRef @@ -52,7 +57,7 @@ class UniquenessProviderTests( @JvmStatic @Parameterized.Parameters(name = "{0}") fun data(): Collection = listOf( - PersistentUniquenessProviderFactory(), + JPAUniquenessProviderFactory(), RaftUniquenessProviderFactory() ) } @@ -599,20 +604,6 @@ interface UniquenessProviderFactory { fun cleanUp() {} } -class PersistentUniquenessProviderFactory : UniquenessProviderFactory { - private var database: CordaPersistence? = null - - override fun create(clock: Clock): UniquenessProvider { - database?.close() - database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, NodeSchemaService(extraSchemas = setOf(NodeNotarySchemaV1))) - return PersistentUniquenessProvider(clock, database!!, TestingNamedCacheFactory(), ::signSingle) - } - - override fun cleanUp() { - database?.close() - } -} - class RaftUniquenessProviderFactory : UniquenessProviderFactory { private var database: CordaPersistence? = null private var provider: RaftUniquenessProvider? = null @@ -645,6 +636,36 @@ class RaftUniquenessProviderFactory : UniquenessProviderFactory { } } +fun signBatch(it: Iterable): BatchSignature { + val root = MerkleTree.getMerkleTree(it.map { it.sha256() }) + + val signableMetadata = SignatureMetadata(4, Crypto.findSignatureScheme(pubKey).schemeNumberID) + val signature = keyService.sign(SignableData(root.hash, signableMetadata), pubKey) + return BatchSignature(signature, root) +} + +class JPAUniquenessProviderFactory : UniquenessProviderFactory { + private var database: CordaPersistence? = null + private val notaryConfig = JPANotaryConfiguration(maxInputStates = 10) + private val notaryWorkerName = CordaX500Name.parse("CN=NotaryWorker, O=Corda, L=London, C=GB") + + override fun create(clock: Clock): UniquenessProvider { + database?.close() + database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, NodeSchemaService(extraSchemas = setOf(JPANotarySchemaV1))) + return JPAUniquenessProvider( + clock, + database!!, + notaryConfig, + notaryWorkerName, + ::signBatch + ) + } + + override fun cleanUp() { + database?.close() + } +} + var ourKeyPair: KeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) val keyService = MockKeyManagementService(makeTestIdentityService(), ourKeyPair) val pubKey = keyService.freshKey() From c2fd8253ea545437c7faa4ee6ce8934866f84a42 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Tue, 28 Jul 2020 16:27:51 +0100 Subject: [PATCH 45/48] CORDA-3777 Reload after every checkpoint (#6494) Enable reloading of a flow after every checkpoint is saved. This includes reloading the checkpoint from the database and recreating the fiber. When a flow and its `StateMachineState` is created it checks the node's config to see if the `reloadCheckpointAfterSuspend` is set to true. If it is it initialises `StateMachineState.reloadCheckpointAfterSuspendCount` with the value 0. Otherwise, it remains `null`. This count represents how many times the flow has reloaded from its checkpoint (not the same as retrying). It is incremented every time the flow is reloaded. When a flow suspends, it processes the suspend event like usual, but it will now also check if `reloadCheckpointAfterSuspendCount` is not `null` (that it is activated) and process a `ReloadFlowFromCheckpointAfterSuspend`event, if and only if `reloadCheckpointAfterSuspendCount` is greater than `CheckpointState.numberOfSuspends`. This means idempotent flows can reload from the start and not reload again until reaching a new suspension point. Flows that skip checkpoints can reload from a previously saved checkpoint (or from the initial checkpoint) and will continue reloading on reaching the next new suspension point (not the suspension point that it skipped saving). If the flow fails to deserialize the checkpoint from the database upon reloading a `ReloadFlowFromCheckpointException` is throw. This causes the flow to be kept for observation. --- .../flows/FlowReloadAfterCheckpointTest.kt | 511 ++++++++++++++++++ .../net/corda/node/flows/FlowRetryTest.kt | 232 ++++---- .../node/services/config/NodeConfiguration.kt | 16 +- .../services/config/NodeConfigurationImpl.kt | 5 +- .../schema/v1/V1NodeConfigurationSpec.kt | 5 +- .../corda/node/services/statemachine/Event.kt | 31 +- .../node/services/statemachine/FlowCreator.kt | 80 ++- .../statemachine/FlowStateMachineImpl.kt | 24 +- .../SingleThreadedStateMachineManager.kt | 28 +- .../statemachine/StaffedFlowHospital.kt | 1 + .../statemachine/StateMachineState.kt | 3 +- .../statemachine/StateTransitionExceptions.kt | 13 +- ...FiberDeserializationCheckingInterceptor.kt | 101 ---- .../transitions/TopLevelTransition.kt | 27 +- .../config/NodeConfigurationImplTest.kt | 8 - .../node/internal/InternalMockNetwork.kt | 1 + 16 files changed, 779 insertions(+), 307 deletions(-) create mode 100644 node/src/integration-test/kotlin/net/corda/node/flows/FlowReloadAfterCheckpointTest.kt delete mode 100644 node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/FiberDeserializationCheckingInterceptor.kt diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowReloadAfterCheckpointTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowReloadAfterCheckpointTest.kt new file mode 100644 index 0000000000..1fd0ca76a7 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowReloadAfterCheckpointTest.kt @@ -0,0 +1,511 @@ +package net.corda.node.flows + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.HospitalizeFlowException +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.flows.StateMachineRunId +import net.corda.core.identity.Party +import net.corda.core.internal.FlowIORequest +import net.corda.core.internal.IdempotentFlow +import net.corda.core.internal.TimedFlow +import net.corda.core.internal.concurrent.transpose +import net.corda.core.messaging.StateMachineTransactionMapping +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.core.utilities.unwrap +import net.corda.finance.DOLLARS +import net.corda.finance.flows.CashIssueAndPaymentFlow +import net.corda.node.services.config.NodeConfiguration +import net.corda.node.services.statemachine.FlowStateMachineImpl +import net.corda.node.services.statemachine.FlowTimeoutException +import net.corda.node.services.statemachine.StaffedFlowHospital +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.internal.FINANCE_CORDAPPS +import net.corda.testing.node.internal.enclosedCordapp +import org.junit.Test +import java.sql.SQLTransientConnectionException +import java.util.concurrent.Semaphore +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class FlowReloadAfterCheckpointTest { + + private companion object { + val cordapps = listOf(enclosedCordapp()) + } + + @Test(timeout = 300_000) + fun `flow will reload from its checkpoint after suspending when reloadCheckpointAfterSuspend is true`() { + val reloadCounts = mutableMapOf() + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { id -> + reloadCounts.compute(id) { _, value -> value?.plus(1) ?: 1 } + } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { + startNode( + providedName = it, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ) + } + .transpose() + .getOrThrow() + + val handle = alice.rpc.startFlow(::ReloadFromCheckpointFlow, bob.nodeInfo.singleIdentity(), false, false, false) + val flowStartedByAlice = handle.id + handle.returnValue.getOrThrow() + assertEquals(5, reloadCounts[flowStartedByAlice]) + assertEquals(6, reloadCounts[ReloadFromCheckpointResponder.flowId]) + } + } + + @Test(timeout = 300_000) + fun `flow will not reload from its checkpoint after suspending when reloadCheckpointAfterSuspend is false`() { + val reloadCounts = mutableMapOf() + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { id -> + reloadCounts.compute(id) { _, value -> value?.plus(1) ?: 1 } + } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { + startNode( + providedName = it, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to false) + ) + } + .transpose() + .getOrThrow() + + val handle = alice.rpc.startFlow(::ReloadFromCheckpointFlow, bob.nodeInfo.singleIdentity(), false, false, false) + val flowStartedByAlice = handle.id + handle.returnValue.getOrThrow() + assertNull(reloadCounts[flowStartedByAlice]) + assertNull(reloadCounts[ReloadFromCheckpointResponder.flowId]) + } + } + + @Test(timeout = 300_000) + fun `flow will reload from its checkpoint after suspending when reloadCheckpointAfterSuspend is true and be kept for observation due to failed deserialization`() { + val reloadCounts = mutableMapOf() + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { id -> + reloadCounts.compute(id) { _, value -> value?.plus(1) ?: 1 } + } + lateinit var flowKeptForObservation: StateMachineRunId + val lock = Semaphore(0) + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { id, _ -> + flowKeptForObservation = id + lock.release() + } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { + startNode( + providedName = it, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ) + } + .transpose() + .getOrThrow() + + val handle = alice.rpc.startFlow(::ReloadFromCheckpointFlow, bob.nodeInfo.singleIdentity(), true, false, false) + val flowStartedByAlice = handle.id + lock.acquire() + assertEquals(flowStartedByAlice, flowKeptForObservation) + assertEquals(4, reloadCounts[flowStartedByAlice]) + assertEquals(4, reloadCounts[ReloadFromCheckpointResponder.flowId]) + } + } + + @Test(timeout = 300_000) + fun `flow will reload from a previous checkpoint after calling suspending function and skipping the persisting the current checkpoint when reloadCheckpointAfterSuspend is true`() { + val reloadCounts = mutableMapOf() + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { id -> + reloadCounts.compute(id) { _, value -> value?.plus(1) ?: 1 } + } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { + startNode( + providedName = it, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ) + } + .transpose() + .getOrThrow() + + val handle = alice.rpc.startFlow(::ReloadFromCheckpointFlow, bob.nodeInfo.singleIdentity(), false, false, true) + val flowStartedByAlice = handle.id + handle.returnValue.getOrThrow() + assertEquals(5, reloadCounts[flowStartedByAlice]) + assertEquals(6, reloadCounts[ReloadFromCheckpointResponder.flowId]) + } + } + + @Test(timeout = 300_000) + fun `idempotent flow will reload from initial checkpoint after calling a suspending function when reloadCheckpointAfterSuspend is true`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::MyIdempotentFlow, false).returnValue.getOrThrow() + assertEquals(5, reloadCount) + } + } + + @Test(timeout = 300_000) + fun `idempotent flow will reload from initial checkpoint after calling a suspending function when reloadCheckpointAfterSuspend is true but can't throw deserialization error from objects in the call function`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::MyIdempotentFlow, true).returnValue.getOrThrow() + assertEquals(5, reloadCount) + } + } + + @Test(timeout = 300_000) + fun `timed flow will reload from initial checkpoint after calling a suspending function when reloadCheckpointAfterSuspend is true`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::MyTimedFlow).returnValue.getOrThrow() + assertEquals(5, reloadCount) + } + } + + @Test(timeout = 300_000) + fun `flow will correctly retry after an error when reloadCheckpointAfterSuspend is true`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + var timesDischarged = 0 + StaffedFlowHospital.onFlowDischarged.add { _, _ -> timesDischarged += 1 } + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::TransientConnectionFailureFlow).returnValue.getOrThrow() + assertEquals(5, reloadCount) + assertEquals(3, timesDischarged) + } + } + + @Test(timeout = 300_000) + fun `flow continues reloading from checkpoints after node restart when reloadCheckpointAfterSuspend is true`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + driver( + DriverParameters( + inMemoryDB = false, + startNodesInProcess = true, + notarySpecs = emptyList(), + cordappsForAllNodes = cordapps + ) + ) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::MyHospitalizingFlow) + Thread.sleep(10.seconds.toMillis()) + + alice.stop() + + startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + Thread.sleep(20.seconds.toMillis()) + + assertEquals(5, reloadCount) + } + } + + @Test(timeout = 300_000) + fun `idempotent flow continues reloading from checkpoints after node restart when reloadCheckpointAfterSuspend is true`() { + var reloadCount = 0 + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { _ -> reloadCount += 1 } + driver( + DriverParameters( + inMemoryDB = false, + startNodesInProcess = true, + notarySpecs = emptyList(), + cordappsForAllNodes = cordapps + ) + ) { + + val alice = startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + alice.rpc.startFlow(::IdempotentHospitalizingFlow) + Thread.sleep(10.seconds.toMillis()) + + alice.stop() + + startNode( + providedName = ALICE_NAME, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ).getOrThrow() + + Thread.sleep(20.seconds.toMillis()) + + // restarts completely from the beginning and forgets the in-memory reload count therefore + // it reloads an extra 2 times for checkpoints it had already reloaded before the node shutdown + assertEquals(7, reloadCount) + } + } + + @Test(timeout = 300_000) + fun `more complicated flow will reload from its checkpoint after suspending when reloadCheckpointAfterSuspend is true`() { + val reloadCounts = mutableMapOf() + FlowStateMachineImpl.onReloadFlowFromCheckpoint = { id -> + reloadCounts.compute(id) { _, value -> value?.plus(1) ?: 1 } + } + driver(DriverParameters(startNodesInProcess = true, cordappsForAllNodes = FINANCE_CORDAPPS)) { + + val (alice, bob) = listOf(ALICE_NAME, BOB_NAME) + .map { + startNode( + providedName = it, + customOverrides = mapOf(NodeConfiguration::reloadCheckpointAfterSuspend.name to true) + ) + } + .transpose() + .getOrThrow() + + val handle = alice.rpc.startFlow( + ::CashIssueAndPaymentFlow, + 500.DOLLARS, + OpaqueBytes.of(0x01), + bob.nodeInfo.singleIdentity(), + false, + defaultNotaryIdentity + ) + val flowStartedByAlice = handle.id + handle.returnValue.getOrThrow(30.seconds) + val flowStartedByBob = bob.rpc.stateMachineRecordedTransactionMappingSnapshot() + .map(StateMachineTransactionMapping::stateMachineRunId) + .toSet() + .single() + Thread.sleep(10.seconds.toMillis()) + assertEquals(7, reloadCounts[flowStartedByAlice]) + assertEquals(6, reloadCounts[flowStartedByBob]) + } + } + + /** + * Has 4 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 5. + * Therefore this flow should reload 5 times when completed without errors or restarts. + */ + @StartableByRPC + @InitiatingFlow + class ReloadFromCheckpointFlow( + private val party: Party, + private val shouldHaveDeserializationError: Boolean, + private val counterPartyHasDeserializationError: Boolean, + private val skipCheckpoints: Boolean + ) : FlowLogic() { + + @Suspendable + override fun call() { + val session = initiateFlow(party) + session.send(counterPartyHasDeserializationError, skipCheckpoints) + session.receive(String::class.java, skipCheckpoints).unwrap { it } + stateMachine.suspend(FlowIORequest.ForceCheckpoint, skipCheckpoints) + val map = if (shouldHaveDeserializationError) { + BrokenMap(mutableMapOf("i dont want" to "this to work")) + } else { + mapOf("i dont want" to "this to work") + } + logger.info("I need to use my variable to pass the build!: $map") + session.sendAndReceive("hey I made it this far") + } + } + + /** + * Has 5 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 6. + * Therefore this flow should reload 6 times when completed without errors or restarts. + */ + @InitiatedBy(ReloadFromCheckpointFlow::class) + class ReloadFromCheckpointResponder(private val session: FlowSession) : FlowLogic() { + + companion object { + var flowId: StateMachineRunId? = null + } + + @Suspendable + override fun call() { + flowId = runId + val counterPartyHasDeserializationError = session.receive().unwrap { it } + session.send("hello there 12312311") + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + val map = if (counterPartyHasDeserializationError) { + BrokenMap(mutableMapOf("i dont want" to "this to work")) + } else { + mapOf("i dont want" to "this to work") + } + logger.info("I need to use my variable to pass the build!: $map") + session.receive().unwrap { it } + session.send("sending back a message") + } + } + + /** + * Has 4 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 5. + * Therefore this flow should reload 5 times when completed without errors or restarts. + */ + @StartableByRPC + @InitiatingFlow + class MyIdempotentFlow(private val shouldHaveDeserializationError: Boolean) : FlowLogic(), IdempotentFlow { + + @Suspendable + override fun call() { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + val map = if (shouldHaveDeserializationError) { + BrokenMap(mutableMapOf("i dont want" to "this to work")) + } else { + mapOf("i dont want" to "this to work") + } + logger.info("I need to use my variable to pass the build!: $map") + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + } + + /** + * Has 4 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 5. + * Therefore this flow should reload 5 times when completed without errors or restarts. + */ + @StartableByRPC + @InitiatingFlow + class MyTimedFlow : FlowLogic(), TimedFlow { + + companion object { + var thrown = false + } + + override val isTimeoutEnabled: Boolean = true + + @Suspendable + override fun call() { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + if (!thrown) { + thrown = true + throw FlowTimeoutException() + } + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + } + + @StartableByRPC + @InitiatingFlow + class TransientConnectionFailureFlow : FlowLogic() { + + companion object { + var retryCount = 0 + } + + @Suspendable + override fun call() { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + if (retryCount < 3) { + retryCount += 1 + throw SQLTransientConnectionException("Connection is not available") + + } + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + } + + /** + * Has 4 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 5. + * Therefore this flow should reload 5 times when completed without errors or restarts. + */ + @StartableByRPC + @InitiatingFlow + class MyHospitalizingFlow : FlowLogic() { + + companion object { + var thrown = false + } + + @Suspendable + override fun call() { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + if (!thrown) { + thrown = true + throw HospitalizeFlowException("i want to try again") + } + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + } + + /** + * Has 4 suspension points inside the flow and 1 in [FlowStateMachineImpl.run] totaling 5. + * Therefore this flow should reload 5 times when completed without errors or restarts. + */ + @StartableByRPC + @InitiatingFlow + class IdempotentHospitalizingFlow : FlowLogic(), IdempotentFlow { + + companion object { + var thrown = false + } + + @Suspendable + override fun call() { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + if (!thrown) { + thrown = true + throw HospitalizeFlowException("i want to try again") + } + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt index 8d82b1a07d..1dda43c691 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt @@ -1,10 +1,13 @@ package net.corda.node.flows import co.paralleluniverse.fibers.Suspendable -import net.corda.client.rpc.CordaRPCClient -import net.corda.client.rpc.CordaRPCClientConfiguration import net.corda.core.CordaRuntimeException -import net.corda.core.flows.* +import net.corda.core.flows.FlowExternalAsyncOperation +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC import net.corda.core.identity.Party import net.corda.core.internal.IdempotentFlow import net.corda.core.internal.concurrent.transpose @@ -23,6 +26,7 @@ import net.corda.testing.core.singleIdentity import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.driver import net.corda.testing.node.User +import net.corda.testing.node.internal.enclosedCordapp import org.assertj.core.api.Assertions.assertThatExceptionOfType import org.hibernate.exception.ConstraintViolationException import org.junit.After @@ -33,7 +37,8 @@ import java.sql.SQLException import java.sql.SQLTransientConnectionException import java.time.Duration import java.time.temporal.ChronoUnit -import java.util.* +import java.util.Collections +import java.util.HashSet import java.util.concurrent.CompletableFuture import java.util.concurrent.TimeoutException import kotlin.test.assertEquals @@ -41,7 +46,11 @@ import kotlin.test.assertFailsWith import kotlin.test.assertNotNull class FlowRetryTest { - val config = CordaRPCClientConfiguration.DEFAULT.copy(connectionRetryIntervalMultiplier = 1.1) + + private companion object { + val user = User("mark", "dadada", setOf(Permissions.all())) + val cordapps = listOf(enclosedCordapp()) + } @Before fun resetCounters() { @@ -58,154 +67,134 @@ class FlowRetryTest { StaffedFlowHospital.DatabaseEndocrinologist.customConditions.clear() } - @Test(timeout=300_000) - fun `flows continue despite errors`() { + @Test(timeout = 300_000) + fun `flows continue despite errors`() { val numSessions = 2 val numIterations = 10 - val user = User("mark", "dadada", setOf(Permissions.startFlow())) - val result: Any? = driver(DriverParameters( - startNodesInProcess = isQuasarAgentSpecified(), - notarySpecs = emptyList() - )) { - val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) - .map { startNode(providedName = it, rpcUsers = listOf(user)) } - .transpose() - .getOrThrow() + val result: Any? = driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { - val result = CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - it.proxy.startFlow(::InitiatorFlow, numSessions, numIterations, nodeBHandle.nodeInfo.singleIdentity()).returnValue.getOrThrow() - } + val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() + + val result = nodeAHandle.rpc.startFlow( + ::InitiatorFlow, + numSessions, + numIterations, + nodeBHandle.nodeInfo.singleIdentity() + ).returnValue.getOrThrow() result } assertNotNull(result) assertEquals("$numSessions:$numIterations", result) } - @Test(timeout=300_000) - fun `async operation deduplication id is stable accross retries`() { - val user = User("mark", "dadada", setOf(Permissions.startFlow())) - driver(DriverParameters( - startNodesInProcess = isQuasarAgentSpecified(), - notarySpecs = emptyList() - )) { + @Test(timeout = 300_000) + fun `async operation deduplication id is stable accross retries`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + nodeAHandle.rpc.startFlow(::AsyncRetryFlow).returnValue.getOrThrow() + } + } - CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - it.proxy.startFlow(::AsyncRetryFlow).returnValue.getOrThrow() + @Test(timeout = 300_000) + fun `flow gives up after number of exceptions, even if this is the first line of the flow`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + assertFailsWith { + nodeAHandle.rpc.startFlow(::RetryFlow).returnValue.getOrThrow() } } } - @Test(timeout=300_000) - fun `flow gives up after number of exceptions, even if this is the first line of the flow`() { - val user = User("mark", "dadada", setOf(Permissions.startFlow())) - assertThatExceptionOfType(CordaRuntimeException::class.java).isThrownBy { - driver(DriverParameters( - startNodesInProcess = isQuasarAgentSpecified(), - notarySpecs = emptyList() - )) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - - val result = CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - it.proxy.startFlow(::RetryFlow).returnValue.getOrThrow() - } - result + @Test(timeout = 300_000) + fun `flow that throws in constructor throw for the RPC client that attempted to start them`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + assertFailsWith { + nodeAHandle.rpc.startFlow(::ThrowingFlow).returnValue.getOrThrow() } } } - @Test(timeout=300_000) - fun `flow that throws in constructor throw for the RPC client that attempted to start them`() { - val user = User("mark", "dadada", setOf(Permissions.startFlow())) - assertThatExceptionOfType(CordaRuntimeException::class.java).isThrownBy { - driver(DriverParameters( - startNodesInProcess = isQuasarAgentSpecified(), - notarySpecs = emptyList() - )) { - val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - - val result = CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - it.proxy.startFlow(::ThrowingFlow).returnValue.getOrThrow() - } - result - } - } - } - - @Test(timeout=300_000) - fun `SQLTransientConnectionExceptions thrown by hikari are retried 3 times and then kept in the checkpoints table`() { - val user = User("mark", "dadada", setOf(Permissions.all())) - driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { + @Test(timeout = 300_000) + fun `SQLTransientConnectionExceptions thrown by hikari are retried 3 times and then kept in the checkpoints table`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) - .map { startNode(providedName = it, rpcUsers = listOf(user)) } - .transpose() - .getOrThrow() - CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - assertFailsWith { - it.proxy.startFlow(::TransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) - .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) - } - assertEquals(3, TransientConnectionFailureFlow.retryCount) - assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get()) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() + + assertFailsWith { + nodeAHandle.rpc.startFlow(::TransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) + .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) } + assertEquals(3, TransientConnectionFailureFlow.retryCount) + assertEquals( + 1, + nodeAHandle.rpc.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get() + ) } } - @Test(timeout=300_000) - fun `Specific exception still detected even if it is nested inside another exception`() { - val user = User("mark", "dadada", setOf(Permissions.all())) - driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { + @Test(timeout = 300_000) + fun `Specific exception still detected even if it is nested inside another exception`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) - .map { startNode(providedName = it, rpcUsers = listOf(user)) } - .transpose() - .getOrThrow() - CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - assertFailsWith { - it.proxy.startFlow(::WrappedTransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) - .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) - } - assertEquals(3, WrappedTransientConnectionFailureFlow.retryCount) - assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get()) + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() + + assertFailsWith { + nodeAHandle.rpc.startFlow(::WrappedTransientConnectionFailureFlow, nodeBHandle.nodeInfo.singleIdentity()) + .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) } + assertEquals(3, WrappedTransientConnectionFailureFlow.retryCount) + assertEquals( + 1, + nodeAHandle.rpc.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get() + ) } } - @Test(timeout=300_000) - fun `General external exceptions are not retried and propagate`() { - val user = User("mark", "dadada", setOf(Permissions.all())) - driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { + @Test(timeout = 300_000) + fun `General external exceptions are not retried and propagate`() { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { val (nodeAHandle, nodeBHandle) = listOf(ALICE_NAME, BOB_NAME) - .map { startNode(providedName = it, rpcUsers = listOf(user)) } - .transpose() - .getOrThrow() + .map { startNode(providedName = it, rpcUsers = listOf(user)) } + .transpose() + .getOrThrow() - CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - assertFailsWith { - it.proxy.startFlow(::GeneralExternalFailureFlow, nodeBHandle.nodeInfo.singleIdentity()).returnValue.getOrThrow() - } - assertEquals(0, GeneralExternalFailureFlow.retryCount) - assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.FAILED).returnValue.get()) + assertFailsWith { + nodeAHandle.rpc.startFlow( + ::GeneralExternalFailureFlow, + nodeBHandle.nodeInfo.singleIdentity() + ).returnValue.getOrThrow() } + assertEquals(0, GeneralExternalFailureFlow.retryCount) + assertEquals( + 1, + nodeAHandle.rpc.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.FAILED).returnValue.get() + ) } } - @Test(timeout=300_000) - fun `Permission exceptions are not retried and propagate`() { + @Test(timeout = 300_000) + fun `Permission exceptions are not retried and propagate`() { val user = User("mark", "dadada", setOf()) - driver(DriverParameters(isDebug = true, startNodesInProcess = isQuasarAgentSpecified())) { + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), cordappsForAllNodes = cordapps)) { val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() - CordaRPCClient(nodeAHandle.rpcAddress, config).start(user.username, user.password).use { - assertThatExceptionOfType(CordaRuntimeException::class.java).isThrownBy { - it.proxy.startFlow(::AsyncRetryFlow).returnValue.getOrThrow() - }.withMessageStartingWith("User not authorized to perform RPC call") - // This stays at -1 since the flow never even got called - assertEquals(-1, GeneralExternalFailureFlow.retryCount) - } + assertThatExceptionOfType(CordaRuntimeException::class.java).isThrownBy { + nodeAHandle.rpc.startFlow(::AsyncRetryFlow).returnValue.getOrThrow() + }.withMessageStartingWith("User not authorized to perform RPC call") + // This stays at -1 since the flow never even got called + assertEquals(-1, GeneralExternalFailureFlow.retryCount) } } } @@ -315,6 +304,10 @@ enum class Step { First, BeforeInitiate, AfterInitiate, AfterInitiateSendReceive data class Visited(val sessionNum: Int, val iterationNum: Int, val step: Step) +class BrokenMap(delegate: MutableMap = mutableMapOf()) : MutableMap by delegate { + override fun put(key: K, value: V): V? = throw IllegalStateException("Broken on purpose") +} + @StartableByRPC class RetryFlow() : FlowLogic(), IdempotentFlow { companion object { @@ -342,7 +335,7 @@ class AsyncRetryFlow() : FlowLogic(), IdempotentFlow { val deduplicationIds = mutableSetOf() } - class RecordDeduplicationId: FlowExternalAsyncOperation { + class RecordDeduplicationId : FlowExternalAsyncOperation { override fun execute(deduplicationId: String): CompletableFuture { val dedupeIdIsNew = deduplicationIds.add(deduplicationId) if (dedupeIdIsNew) { @@ -423,8 +416,9 @@ class WrappedTransientConnectionFailureFlow(private val party: Party) : FlowLogi // checkpoint will restart the flow after the send retryCount += 1 throw IllegalStateException( - "wrapped error message", - IllegalStateException("another layer deep", SQLTransientConnectionException("Connection is not available"))) + "wrapped error message", + IllegalStateException("another layer deep", SQLTransientConnectionException("Connection is not available")) + ) } } @@ -465,12 +459,14 @@ class GeneralExternalFailureResponder(private val session: FlowSession) : FlowLo @StartableByRPC class GetCheckpointNumberOfStatusFlow(private val flowStatus: Checkpoint.FlowStatus) : FlowLogic() { + + @Suspendable override fun call(): Long { val sqlStatement = - "select count(*) " + - "from node_checkpoints " + - "where status = ${flowStatus.ordinal} " + - "and flow_id != '${runId.uuid}' " // don't count in the checkpoint of the current flow + "select count(*) " + + "from node_checkpoints " + + "where status = ${flowStatus.ordinal} " + + "and flow_id != '${runId.uuid}' " // don't count in the checkpoint of the current flow return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> ps.executeQuery().use { rs -> diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt index a12989e169..9c3d5f9741 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt @@ -93,6 +93,8 @@ interface NodeConfiguration : ConfigurationWithOptionsContainer { val quasarExcludePackages: List + val reloadCheckpointAfterSuspend: Boolean + companion object { // default to at least 8MB and a bit extra for larger heap sizes val defaultTransactionCacheSize: Long = 8.MB + getAdditionalCacheMemory() @@ -125,9 +127,13 @@ enum class JmxReporterType { } data class DevModeOptions( - val disableCheckpointChecker: Boolean = Defaults.disableCheckpointChecker, - val allowCompatibilityZone: Boolean = Defaults.allowCompatibilityZone, - val djvm: DJVMOptions? = null + @Deprecated( + "The checkpoint checker has been replaced by the ability to reload a checkpoint from the database after every suspend" + + "Use [NodeConfiguration.disableReloadCheckpointAfterSuspend] instead." + ) + val disableCheckpointChecker: Boolean = Defaults.disableCheckpointChecker, + val allowCompatibilityZone: Boolean = Defaults.allowCompatibilityZone, + val djvm: DJVMOptions? = null ) { internal object Defaults { val disableCheckpointChecker = false @@ -140,10 +146,6 @@ data class DJVMOptions( val cordaSource: List ) -fun NodeConfiguration.shouldCheckCheckpoints(): Boolean { - return this.devMode && this.devModeOptions?.disableCheckpointChecker != true -} - fun NodeConfiguration.shouldStartSSHDaemon() = this.sshd != null fun NodeConfiguration.shouldStartLocalShell() = !this.noLocalShell && System.console() != null && this.devMode fun NodeConfiguration.shouldInitCrashShell() = shouldStartLocalShell() || shouldStartSSHDaemon() diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt index e1dcc86903..6106441279 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt @@ -84,7 +84,9 @@ data class NodeConfigurationImpl( override val blacklistedAttachmentSigningKeys: List = Defaults.blacklistedAttachmentSigningKeys, override val configurationWithOptions: ConfigurationWithOptions, override val flowExternalOperationThreadPoolSize: Int = Defaults.flowExternalOperationThreadPoolSize, - override val quasarExcludePackages: List = Defaults.quasarExcludePackages + override val quasarExcludePackages: List = Defaults.quasarExcludePackages, + override val reloadCheckpointAfterSuspend: Boolean = Defaults.reloadCheckpointAfterSuspend + ) : NodeConfiguration { internal object Defaults { val jmxMonitoringHttpPort: Int? = null @@ -123,6 +125,7 @@ data class NodeConfigurationImpl( val blacklistedAttachmentSigningKeys: List = emptyList() const val flowExternalOperationThreadPoolSize: Int = 1 val quasarExcludePackages: List = emptyList() + val reloadCheckpointAfterSuspend: Boolean = System.getProperty("reloadCheckpointAfterSuspend", "false")!!.toBoolean() fun cordappsDirectories(baseDirectory: Path) = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT) diff --git a/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt b/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt index d396c466c0..a10c15870b 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt @@ -8,6 +8,7 @@ import net.corda.common.validation.internal.Validated.Companion.invalid import net.corda.common.validation.internal.Validated.Companion.valid import net.corda.node.services.config.* import net.corda.node.services.config.NodeConfigurationImpl.Defaults +import net.corda.node.services.config.NodeConfigurationImpl.Defaults.reloadCheckpointAfterSuspend import net.corda.node.services.config.schema.parsers.* internal object V1NodeConfigurationSpec : Configuration.Specification("NodeConfiguration") { @@ -66,6 +67,7 @@ internal object V1NodeConfigurationSpec : Configuration.Specification, - val maySkipCheckpoint: Boolean, - val fiber: SerializedBytes>, - var progressStep: ProgressTracker.Step? + val ioRequest: FlowIORequest<*>, + val maySkipCheckpoint: Boolean, + val fiber: SerializedBytes>, + var progressStep: ProgressTracker.Step? ) : Event() { override fun toString() = - "Suspend(" + - "ioRequest=$ioRequest, " + - "maySkipCheckpoint=$maySkipCheckpoint, " + - "fiber=${fiber.hash}, " + - "currentStep=${progressStep?.label}" + - ")" + "Suspend(" + + "ioRequest=$ioRequest, " + + "maySkipCheckpoint=$maySkipCheckpoint, " + + "fiber=${fiber.hash}, " + + "currentStep=${progressStep?.label}" + + ")" } /** @@ -148,12 +148,21 @@ sealed class Event { data class AsyncOperationThrows(val throwable: Throwable) : Event() /** - * Retry a flow from the last checkpoint, or if there is no checkpoint, restart the flow with the same invocation details. + * Retry a flow from its last checkpoint, or if there is no checkpoint, restart the flow with the same invocation details. */ object RetryFlowFromSafePoint : Event() { override fun toString() = "RetryFlowFromSafePoint" } + /** + * Reload a flow from its last checkpoint, or if there is no checkpoint, restart the flow with the same invocation details. + * This is separate from [RetryFlowFromSafePoint] which is used for error handling within the state machine. + * [ReloadFlowFromCheckpointAfterSuspend] is only used when [NodeConfiguration.reloadCheckpointAfterSuspend] is true. + */ + object ReloadFlowFromCheckpointAfterSuspend : Event() { + override fun toString() = "ReloadFlowFromCheckpointAfterSuspend" + } + /** * Keeps a flow for overnight observation. Overnight observation practically sends the fiber to get suspended, * in [FlowStateMachineImpl.processEventsUntilFlowIsResumed]. Since the fiber's channel will have no more events to process, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt index 08a006c345..8e388b8d35 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt @@ -19,6 +19,7 @@ import net.corda.core.utilities.contextLogger import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.ServiceHubInternal import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.statemachine.FlowStateMachineImpl.Companion.currentStateMachine import net.corda.node.services.statemachine.transitions.StateMachine import net.corda.node.utilities.isEnabledTimedFlow import net.corda.nodeapi.internal.persistence.CordaPersistence @@ -36,21 +37,23 @@ class NonResidentFlow(val runId: StateMachineRunId, val checkpoint: Checkpoint) } class FlowCreator( - val checkpointSerializationContext: CheckpointSerializationContext, + private val checkpointSerializationContext: CheckpointSerializationContext, private val checkpointStorage: CheckpointStorage, - val scheduler: FiberScheduler, - val database: CordaPersistence, - val transitionExecutor: TransitionExecutor, - val actionExecutor: ActionExecutor, - val secureRandom: SecureRandom, - val serviceHub: ServiceHubInternal, - val unfinishedFibers: ReusableLatch, - val resetCustomTimeout: (StateMachineRunId, Long) -> Unit) { + private val scheduler: FiberScheduler, + private val database: CordaPersistence, + private val transitionExecutor: TransitionExecutor, + private val actionExecutor: ActionExecutor, + private val secureRandom: SecureRandom, + private val serviceHub: ServiceHubInternal, + private val unfinishedFibers: ReusableLatch, + private val resetCustomTimeout: (StateMachineRunId, Long) -> Unit) { companion object { private val logger = contextLogger() } + private val reloadCheckpointAfterSuspend = serviceHub.configuration.reloadCheckpointAfterSuspend + fun createFlowFromNonResidentFlow(nonResidentFlow: NonResidentFlow): Flow<*>? { // As for paused flows we don't extract the serialized flow state we need to re-extract the checkpoint from the database. val checkpoint = when (nonResidentFlow.checkpoint.status) { @@ -65,13 +68,23 @@ class FlowCreator( return createFlowFromCheckpoint(nonResidentFlow.runId, checkpoint) } - fun createFlowFromCheckpoint(runId: StateMachineRunId, oldCheckpoint: Checkpoint): Flow<*>? { + fun createFlowFromCheckpoint( + runId: StateMachineRunId, + oldCheckpoint: Checkpoint, + reloadCheckpointAfterSuspendCount: Int? = null + ): Flow<*>? { val checkpoint = oldCheckpoint.copy(status = Checkpoint.FlowStatus.RUNNABLE) val fiber = checkpoint.getFiberFromCheckpoint(runId) ?: return null val resultFuture = openFuture() fiber.logic.stateMachine = fiber verifyFlowLogicIsSuspendable(fiber.logic) - val state = createStateMachineState(checkpoint, fiber, true) + val state = createStateMachineState( + checkpoint = checkpoint, + fiber = fiber, + anyCheckpointPersisted = true, + reloadCheckpointAfterSuspendCount = reloadCheckpointAfterSuspendCount + ?: if (reloadCheckpointAfterSuspend) checkpoint.checkpointState.numberOfSuspends else null + ) fiber.transientValues = createTransientValues(runId, resultFuture) fiber.transientState = state return Flow(fiber, resultFuture) @@ -108,11 +121,13 @@ class FlowCreator( ).getOrThrow() val state = createStateMachineState( - checkpoint, - flowStateMachineImpl, - existingCheckpoint != null, - deduplicationHandler, - senderUUID) + checkpoint = checkpoint, + fiber = flowStateMachineImpl, + anyCheckpointPersisted = existingCheckpoint != null, + reloadCheckpointAfterSuspendCount = if (reloadCheckpointAfterSuspend) 0 else null, + deduplicationHandler = deduplicationHandler, + senderUUID = senderUUID + ) flowStateMachineImpl.transientState = state return Flow(flowStateMachineImpl, resultFuture) } @@ -125,9 +140,7 @@ class FlowCreator( } is FlowState.Started -> tryCheckpointDeserialize(this.flowState.frozenFiber, runId) ?: return null // Places calling this function is rely on it to return null if the flow cannot be created from the checkpoint. - else -> { - return null - } + else -> null } } @@ -136,8 +149,16 @@ class FlowCreator( return try { bytes.checkpointDeserialize(context = checkpointSerializationContext) } catch (e: Exception) { - logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) - null + if (reloadCheckpointAfterSuspend && currentStateMachine() != null) { + logger.error( + "Unable to deserialize checkpoint for flow $flowId. [reloadCheckpointAfterSuspend] is turned on, throwing exception", + e + ) + throw ReloadFlowFromCheckpointException(e) + } else { + logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) + null + } } } @@ -169,12 +190,15 @@ class FlowCreator( ) } + @Suppress("LongParameterList") private fun createStateMachineState( - checkpoint: Checkpoint, - fiber: FlowStateMachineImpl<*>, - anyCheckpointPersisted: Boolean, - deduplicationHandler: DeduplicationHandler? = null, - senderUUID: String? = null): StateMachineState { + checkpoint: Checkpoint, + fiber: FlowStateMachineImpl<*>, + anyCheckpointPersisted: Boolean, + reloadCheckpointAfterSuspendCount: Int?, + deduplicationHandler: DeduplicationHandler? = null, + senderUUID: String? = null + ): StateMachineState { return StateMachineState( checkpoint = checkpoint, pendingDeduplicationHandlers = deduplicationHandler?.let { listOf(it) } ?: emptyList(), @@ -186,6 +210,8 @@ class FlowCreator( isRemoved = false, isKilled = false, flowLogic = fiber.logic, - senderUUID = senderUUID) + senderUUID = senderUUID, + reloadCheckpointAfterSuspendCount = reloadCheckpointAfterSuspendCount + ) } } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index ce4fdea2bd..6b0ad10698 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -29,6 +29,7 @@ import net.corda.core.internal.DeclaredField import net.corda.core.internal.FlowIORequest import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.IdempotentFlow +import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.concurrent.OpenFuture import net.corda.core.internal.isIdempotentFlow import net.corda.core.internal.isRegularFile @@ -87,6 +88,9 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, private val log: Logger = LoggerFactory.getLogger("net.corda.flow") private val SERIALIZER_BLOCKER = Fiber::class.java.getDeclaredField("SERIALIZER_BLOCKER").apply { isAccessible = true }.get(null) + + @VisibleForTesting + var onReloadFlowFromCheckpoint: ((id: StateMachineRunId) -> Unit)? = null } data class TransientValues( @@ -504,10 +508,10 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, contextTransactionOrNull = transaction.value val event = try { Event.Suspend( - ioRequest = ioRequest, - maySkipCheckpoint = skipPersistingCheckpoint, - fiber = this.checkpointSerialize(context = serializationContext.value), - progressStep = logic.progressTracker?.currentStep + ioRequest = ioRequest, + maySkipCheckpoint = skipPersistingCheckpoint, + fiber = this.checkpointSerialize(context = serializationContext.value), + progressStep = logic.progressTracker?.currentStep ) } catch (exception: Exception) { Event.Error(exception) @@ -529,6 +533,18 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, unpark(SERIALIZER_BLOCKER) } } + + transientState.reloadCheckpointAfterSuspendCount?.let { count -> + if (count < transientState.checkpoint.checkpointState.numberOfSuspends) { + onReloadFlowFromCheckpoint?.invoke(id) + processEventImmediately( + Event.ReloadFlowFromCheckpointAfterSuspend, + isDbTransactionOpenOnEntry = false, + isDbTransactionOpenOnExit = false + ) + } + } + return uncheckedCast(processEventsUntilFlowIsResumed( isDbTransactionOpenOnEntry = false, isDbTransactionOpenOnExit = true diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index fba5833661..d1aa9ee6aa 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -30,11 +30,9 @@ import net.corda.core.utilities.debug import net.corda.node.internal.InitiatedFlowFactory import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.ServiceHubInternal -import net.corda.node.services.config.shouldCheckCheckpoints import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.statemachine.FlowStateMachineImpl.Companion.currentStateMachine import net.corda.node.services.statemachine.interceptors.DumpHistoryOnErrorInterceptor -import net.corda.node.services.statemachine.interceptors.FiberDeserializationChecker -import net.corda.node.services.statemachine.interceptors.FiberDeserializationCheckingInterceptor import net.corda.node.services.statemachine.interceptors.HospitalisingInterceptor import net.corda.node.services.statemachine.interceptors.PrintingInterceptor import net.corda.node.utilities.AffinityExecutor @@ -89,7 +87,6 @@ internal class SingleThreadedStateMachineManager( private val flowMessaging: FlowMessaging = FlowMessagingImpl(serviceHub) private val actionFutureExecutor = ActionFutureExecutor(innerState, serviceHub, scheduledFutureExecutor) private val flowTimeoutScheduler = FlowTimeoutScheduler(innerState, scheduledFutureExecutor, serviceHub) - private val fiberDeserializationChecker = if (serviceHub.configuration.shouldCheckCheckpoints()) FiberDeserializationChecker() else null private val ourSenderUUID = serviceHub.networkService.ourSenderUUID private var checkpointSerializationContext: CheckpointSerializationContext? = null @@ -97,6 +94,7 @@ internal class SingleThreadedStateMachineManager( override val flowHospital: StaffedFlowHospital = makeFlowHospital() private val transitionExecutor = makeTransitionExecutor() + private val reloadCheckpointAfterSuspend = serviceHub.configuration.reloadCheckpointAfterSuspend override val allStateMachines: List> get() = innerState.withLock { flows.values.map { it.fiber.logic } } @@ -124,7 +122,6 @@ internal class SingleThreadedStateMachineManager( ) this.checkpointSerializationContext = checkpointSerializationContext val actionExecutor = makeActionExecutor(checkpointSerializationContext) - fiberDeserializationChecker?.start(checkpointSerializationContext) when (startMode) { StateMachineManager.StartMode.ExcludingPaused -> {} StateMachineManager.StartMode.Safe -> markAllFlowsAsPaused() @@ -207,10 +204,6 @@ internal class SingleThreadedStateMachineManager( // Account for any expected Fibers in a test scenario. liveFibers.countDown(allowedUnsuspendedFiberCount) liveFibers.await() - fiberDeserializationChecker?.let { - val foundUnrestorableFibers = it.stop() - check(!foundUnrestorableFibers) { "Unrestorable checkpoints were created, please check the logs for details." } - } flowHospital.close() scheduledFutureExecutor.shutdown() scheduler.shutdown() @@ -397,7 +390,7 @@ internal class SingleThreadedStateMachineManager( val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) ?: return // Resurrect flow - flowCreator.createFlowFromCheckpoint(flowId, checkpoint) ?: return + flowCreator.createFlowFromCheckpoint(flowId, checkpoint, currentState.reloadCheckpointAfterSuspendCount) ?: return } else { // Just flow initiation message null @@ -632,8 +625,16 @@ internal class SingleThreadedStateMachineManager( return try { serializedCheckpoint.deserialize(checkpointSerializationContext!!) } catch (e: Exception) { - logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) - null + if (reloadCheckpointAfterSuspend && currentStateMachine() != null) { + logger.error( + "Unable to deserialize checkpoint for flow $flowId. [reloadCheckpointAfterSuspend] is turned on, throwing exception", + e + ) + throw ReloadFlowFromCheckpointException(e) + } else { + logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) + null + } } } @@ -700,9 +701,6 @@ internal class SingleThreadedStateMachineManager( if (serviceHub.configuration.devMode) { interceptors.add { DumpHistoryOnErrorInterceptor(it) } } - if (serviceHub.configuration.shouldCheckCheckpoints()) { - interceptors.add { FiberDeserializationCheckingInterceptor(fiberDeserializationChecker!!, it) } - } if (logger.isDebugEnabled) { interceptors.add { PrintingInterceptor(it) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt index 4d6e73bfbe..f74497d48a 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt @@ -589,6 +589,7 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, return if (newError.mentionsThrowable(StateTransitionException::class.java)) { when { newError.mentionsThrowable(InterruptedException::class.java) -> Diagnosis.TERMINAL + newError.mentionsThrowable(ReloadFlowFromCheckpointException::class.java) -> Diagnosis.OVERNIGHT_OBSERVATION newError.mentionsThrowable(AsyncOperationTransitionException::class.java) -> Diagnosis.NOT_MY_SPECIALTY history.notDischargedForTheSameThingMoreThan(2, this, currentState) -> Diagnosis.DISCHARGE else -> Diagnosis.OVERNIGHT_OBSERVATION diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 6c124d41e6..5d8326b668 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -59,7 +59,8 @@ data class StateMachineState( val isRemoved: Boolean, @Volatile var isKilled: Boolean, - val senderUUID: String? + val senderUUID: String?, + val reloadCheckpointAfterSuspendCount: Int? ) : KryoSerializable { override fun write(kryo: Kryo?, output: Output?) { throw IllegalStateException("${StateMachineState::class.qualifiedName} should never be serialized") diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt index 2e37261c04..2c3fc5b641 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt @@ -1,6 +1,6 @@ package net.corda.node.services.statemachine -import net.corda.core.CordaException +import net.corda.core.CordaRuntimeException import net.corda.core.serialization.ConstructorForDeserialization // CORDA-3353 - These exceptions should not be propagated up to rpc as they suppress the real exceptions @@ -9,12 +9,17 @@ class StateTransitionException( val transitionAction: Action?, val transitionEvent: Event?, val exception: Exception -) : CordaException(exception.message, exception) { +) : CordaRuntimeException(exception.message, exception) { @ConstructorForDeserialization constructor(exception: Exception): this(null, null, exception) } -class AsyncOperationTransitionException(exception: Exception) : CordaException(exception.message, exception) +class AsyncOperationTransitionException(exception: Exception) : CordaRuntimeException(exception.message, exception) -class ErrorStateTransitionException(val exception: Exception) : CordaException(exception.message, exception) \ No newline at end of file +class ErrorStateTransitionException(val exception: Exception) : CordaRuntimeException(exception.message, exception) + +class ReloadFlowFromCheckpointException(cause: Exception) : CordaRuntimeException( + "Could not reload flow from checkpoint. This is likely due to a discrepancy " + + "between the serialization and deserialization of an object in the flow's checkpoint", cause +) \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/FiberDeserializationCheckingInterceptor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/FiberDeserializationCheckingInterceptor.kt deleted file mode 100644 index 57c742b9ff..0000000000 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/FiberDeserializationCheckingInterceptor.kt +++ /dev/null @@ -1,101 +0,0 @@ -package net.corda.node.services.statemachine.interceptors - -import co.paralleluniverse.fibers.Suspendable -import net.corda.core.serialization.SerializedBytes -import net.corda.core.serialization.internal.CheckpointSerializationContext -import net.corda.core.serialization.internal.checkpointDeserialize -import net.corda.core.utilities.contextLogger -import net.corda.node.services.statemachine.ActionExecutor -import net.corda.node.services.statemachine.Event -import net.corda.node.services.statemachine.FlowFiber -import net.corda.node.services.statemachine.FlowState -import net.corda.node.services.statemachine.FlowStateMachineImpl -import net.corda.node.services.statemachine.StateMachineState -import net.corda.node.services.statemachine.TransitionExecutor -import net.corda.node.services.statemachine.transitions.FlowContinuation -import net.corda.node.services.statemachine.transitions.TransitionResult -import java.util.concurrent.LinkedBlockingQueue -import kotlin.concurrent.thread - -/** - * This interceptor checks whether a checkpointed fiber state can be deserialised in a separate thread. - */ -class FiberDeserializationCheckingInterceptor( - val fiberDeserializationChecker: FiberDeserializationChecker, - val delegate: TransitionExecutor -) : TransitionExecutor { - - @Suspendable - override fun executeTransition( - fiber: FlowFiber, - previousState: StateMachineState, - event: Event, - transition: TransitionResult, - actionExecutor: ActionExecutor - ): Pair { - val (continuation, nextState) = delegate.executeTransition(fiber, previousState, event, transition, actionExecutor) - val previousFlowState = previousState.checkpoint.flowState - val nextFlowState = nextState.checkpoint.flowState - if (nextFlowState is FlowState.Started) { - if (previousFlowState !is FlowState.Started || previousFlowState.frozenFiber != nextFlowState.frozenFiber) { - fiberDeserializationChecker.submitCheck(nextFlowState.frozenFiber) - } - } - return Pair(continuation, nextState) - } -} - -/** - * A fiber deserialisation checker thread. It checks the queued up serialised checkpoints to see if they can be - * deserialised. This is only run in development mode to allow detecting of corrupt serialised checkpoints before they - * are actually used. - */ -class FiberDeserializationChecker { - companion object { - val log = contextLogger() - } - - private sealed class Job { - class Check(val serializedFiber: SerializedBytes>) : Job() - object Finish : Job() - } - - private var checkerThread: Thread? = null - private val jobQueue = LinkedBlockingQueue() - private var foundUnrestorableFibers: Boolean = false - - fun start(checkpointSerializationContext: CheckpointSerializationContext) { - require(checkerThread == null){"Checking thread must not already be started"} - checkerThread = thread(name = "FiberDeserializationChecker") { - while (true) { - val job = jobQueue.take() - when (job) { - is Job.Check -> { - try { - job.serializedFiber.checkpointDeserialize(context = checkpointSerializationContext) - } catch (exception: Exception) { - log.error("Encountered unrestorable checkpoint!", exception) - foundUnrestorableFibers = true - } - } - Job.Finish -> { - return@thread - } - } - } - } - } - - fun submitCheck(serializedFiber: SerializedBytes>) { - jobQueue.add(Job.Check(serializedFiber)) - } - - /** - * Returns true if some unrestorable checkpoints were encountered, false otherwise - */ - fun stop(): Boolean { - jobQueue.add(Job.Finish) - checkerThread?.join() - return foundUnrestorableFibers - } -} diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 4846ee101d..169148108e 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -58,7 +58,8 @@ class TopLevelTransition( is Event.InitiateFlow -> initiateFlowTransition(event) is Event.AsyncOperationCompletion -> asyncOperationCompletionTransition(event) is Event.AsyncOperationThrows -> asyncOperationThrowsTransition(event) - is Event.RetryFlowFromSafePoint -> retryFlowFromSafePointTransition(startingState) + is Event.RetryFlowFromSafePoint -> retryFlowFromSafePointTransition() + is Event.ReloadFlowFromCheckpointAfterSuspend -> reloadFlowFromCheckpointAfterSuspendTransition() is Event.OvernightObservation -> overnightObservationTransition() is Event.WakeUpFromSleep -> wakeUpFromSleepTransition() } @@ -198,8 +199,8 @@ class TopLevelTransition( Action.ScheduleEvent(Event.DoRemainingWork) )) currentState = currentState.copy( - checkpoint = newCheckpoint, - isFlowResumed = false + checkpoint = newCheckpoint, + isFlowResumed = false ) } else { actions.addAll(arrayOf( @@ -210,10 +211,10 @@ class TopLevelTransition( Action.ScheduleEvent(Event.DoRemainingWork) )) currentState = currentState.copy( - checkpoint = newCheckpoint, - pendingDeduplicationHandlers = emptyList(), - isFlowResumed = false, - isAnyCheckpointPersisted = true + checkpoint = newCheckpoint, + pendingDeduplicationHandlers = emptyList(), + isFlowResumed = false, + isAnyCheckpointPersisted = true ) } FlowContinuation.ProcessEvents @@ -315,10 +316,18 @@ class TopLevelTransition( } } - private fun retryFlowFromSafePointTransition(startingState: StateMachineState): TransitionResult { + private fun retryFlowFromSafePointTransition(): TransitionResult { return builder { // Need to create a flow from the prior checkpoint or flow initiation. - actions.add(Action.RetryFlowFromSafePoint(startingState)) + actions.add(Action.RetryFlowFromSafePoint(currentState)) + FlowContinuation.Abort + } + } + + private fun reloadFlowFromCheckpointAfterSuspendTransition(): TransitionResult { + return builder { + currentState = currentState.copy(reloadCheckpointAfterSuspendCount = currentState.reloadCheckpointAfterSuspendCount!! + 1) + actions.add(Action.RetryFlowFromSafePoint(currentState)) FlowContinuation.Abort } } diff --git a/node/src/test/kotlin/net/corda/node/services/config/NodeConfigurationImplTest.kt b/node/src/test/kotlin/net/corda/node/services/config/NodeConfigurationImplTest.kt index 8cea3cebbb..069901080f 100644 --- a/node/src/test/kotlin/net/corda/node/services/config/NodeConfigurationImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/config/NodeConfigurationImplTest.kt @@ -60,14 +60,6 @@ class NodeConfigurationImplTest { assertThat(configValidationResult.first()).contains("crlCheckSoftFail") } - @Test(timeout=3_000) - fun `check devModeOptions flag helper`() { - assertTrue { configDebugOptions(true, null).shouldCheckCheckpoints() } - assertTrue { configDebugOptions(true, DevModeOptions()).shouldCheckCheckpoints() } - assertTrue { configDebugOptions(true, DevModeOptions(false)).shouldCheckCheckpoints() } - assertFalse { configDebugOptions(true, DevModeOptions(true)).shouldCheckCheckpoints() } - } - @Test(timeout=3_000) fun `check crashShell flags helper`() { assertFalse { testConfiguration.copy(sshd = null).shouldStartSSHDaemon() } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt index 9ae6f1f9d2..681325d382 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt @@ -639,6 +639,7 @@ private fun mockNodeConfiguration(certificatesDirectory: Path): NodeConfiguratio doReturn(NetworkParameterAcceptanceSettings()).whenever(it).networkParameterAcceptanceSettings doReturn(rigorousMock()).whenever(it).configurationWithOptions doReturn(2).whenever(it).flowExternalOperationThreadPoolSize + doReturn(false).whenever(it).reloadCheckpointAfterSuspend } } From 0bedbd8c754789eb865a6c614e5978e530fcd6d6 Mon Sep 17 00:00:00 2001 From: Yiftach Kaplan <67583323+yift-r3@users.noreply.github.com> Date: Wed, 29 Jul 2020 15:47:45 +0100 Subject: [PATCH 46/48] INFRA-530: Start notary node in process (#6521) --- .../corda/node/AddressBindingFailureTests.kt | 2 +- .../net/corda/testing/driver/DriverTests.kt | 4 ++- .../net/corda/testing/node/NotarySpec.kt | 35 +++++++++++++++++-- .../testing/node/internal/DriverDSLImpl.kt | 10 ++++-- 4 files changed, 44 insertions(+), 7 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt b/node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt index 0bef72552d..3e549d6581 100644 --- a/node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt @@ -41,7 +41,7 @@ class AddressBindingFailureTests { assertThatThrownBy { driver(DriverParameters(startNodesInProcess = false, - notarySpecs = listOf(NotarySpec(notaryName)), + notarySpecs = listOf(NotarySpec(notaryName, startInProcess = false)), notaryCustomOverrides = mapOf("p2pAddress" to address.toString()), portAllocation = portAllocation, cordappsForAllNodes = emptyList()) diff --git a/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt b/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt index 05a4cfbba2..5e5f10f74a 100644 --- a/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt +++ b/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt @@ -16,7 +16,9 @@ import net.corda.testing.common.internal.ProjectStructure.projectRootDir import net.corda.testing.core.BOB_NAME import net.corda.testing.core.DUMMY_BANK_A_NAME import net.corda.testing.core.DUMMY_BANK_B_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME import net.corda.testing.http.HttpApi +import net.corda.testing.node.NotarySpec import net.corda.testing.node.internal.addressMustBeBound import net.corda.testing.node.internal.addressMustNotBeBound import org.assertj.core.api.Assertions.assertThat @@ -118,7 +120,7 @@ class DriverTests { fun `started node, which is not waited for in the driver, is shutdown when the driver exits`() { // First check that the process-id file is created by the node on startup, so that we can be sure our check that // it's deleted on shutdown isn't a false-positive. - val baseDirectory = driver { + val baseDirectory = driver(DriverParameters(notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, startInProcess = false)))) { val baseDirectory = defaultNotaryNode.getOrThrow().baseDirectory assertThat(baseDirectory / "process-id").exists() baseDirectory diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt index 584c70d94f..1487065d30 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/NotarySpec.kt @@ -13,24 +13,55 @@ import net.corda.testing.driver.VerifierType * @property rpcUsers A list of users able to instigate RPC for this node or cluster of nodes. * @property verifierType How the notary will verify transactions. * @property cluster [ClusterSpec] if this is a distributed cluster notary. If null then this is a single-node notary. + * @property startInProcess Should the notary be started in process. */ data class NotarySpec( val name: CordaX500Name, val validating: Boolean = true, val rpcUsers: List = emptyList(), val verifierType: VerifierType = VerifierType.InMemory, - val cluster: ClusterSpec? = null + val cluster: ClusterSpec? = null, + val startInProcess: Boolean = true ) { + constructor(name: CordaX500Name, + validating: Boolean = true, + rpcUsers: List = emptyList(), + verifierType: VerifierType = VerifierType.InMemory, + cluster: ClusterSpec? = null): this(name, validating, rpcUsers, verifierType, cluster, "512m", true) + + constructor(name: CordaX500Name, + validating: Boolean = true, + rpcUsers: List = emptyList(), + verifierType: VerifierType = VerifierType.InMemory, + cluster: ClusterSpec? = null, + maximumHeapSize: String): this(name, validating, rpcUsers, verifierType, cluster, maximumHeapSize, true) + // These extra fields are handled this way to preserve Kotlin wire compatibility wrt additional parameters with default values. constructor(name: CordaX500Name, validating: Boolean = true, rpcUsers: List = emptyList(), verifierType: VerifierType = VerifierType.InMemory, cluster: ClusterSpec? = null, - maximumHeapSize: String = "512m"): this(name, validating, rpcUsers, verifierType, cluster) { + maximumHeapSize: String = "512m", + startInProcess: Boolean = true): this(name, validating, rpcUsers, verifierType, cluster, startInProcess) { this.maximumHeapSize = maximumHeapSize } + fun copy( + name: CordaX500Name, + validating: Boolean = true, + rpcUsers: List = emptyList(), + verifierType: VerifierType = VerifierType.InMemory, + cluster: ClusterSpec? = null + ) = this.copy( + name = name, + validating = validating, + rpcUsers = rpcUsers, + verifierType = verifierType, + cluster = cluster, + startInProcess = true + ) + var maximumHeapSize: String = "512m" } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index e3822522cd..71fb8ae72e 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -571,9 +571,13 @@ class DriverDSLImpl( private fun startSingleNotary(spec: NotarySpec, localNetworkMap: LocalNetworkMap?, customOverrides: Map): CordaFuture> { val notaryConfig = mapOf("notary" to mapOf("validating" to spec.validating)) return startRegisteredNode( - spec.name, - localNetworkMap, - NodeParameters(rpcUsers = spec.rpcUsers, verifierType = spec.verifierType, customOverrides = notaryConfig + customOverrides, maximumHeapSize = spec.maximumHeapSize) + spec.name, + localNetworkMap, + NodeParameters(rpcUsers = spec.rpcUsers, + verifierType = spec.verifierType, + startInSameProcess = spec.startInProcess, + customOverrides = notaryConfig + customOverrides, + maximumHeapSize = spec.maximumHeapSize) ).map { listOf(it) } } From a81e8713569edc2f4a1b25198f64c33be3478ccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Thu, 30 Jul 2020 05:25:01 +0100 Subject: [PATCH 47/48] NOTICK: Reverted use of Artifactory cache mechanism (#6526) --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 2 -- .ci/dev/nightly-regression/Jenkinsfile | 2 -- .ci/dev/regression/Jenkinsfile | 2 -- Jenkinsfile | 2 -- 4 files changed, 8 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index af53b9fa84..391a2b5fc6 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -48,7 +48,6 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") - CORDA_USE_CACHE = "corda-remotes" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } @@ -83,7 +82,6 @@ pipeline { "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + "-Ddocker.buildbase.tag=11latest " + - "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.dockerfile=DockerfileJDK11Azul" + diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 2f0f34914d..ca7250b7b0 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -20,7 +20,6 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') - CORDA_USE_CACHE = "corda-remotes" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } @@ -39,7 +38,6 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index b9a952c1fb..63b78efe70 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -49,7 +49,6 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::") - CORDA_USE_CACHE = "corda-remotes" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } @@ -88,7 +87,6 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + diff --git a/Jenkinsfile b/Jenkinsfile index c96e217de3..cf1a58c32e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,7 +17,6 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') - CORDA_USE_CACHE = "corda-remotes" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } @@ -30,7 +29,6 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + From f2bcc85f30283fd82d56f55bd623b42773f19a68 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Thu, 30 Jul 2020 10:54:24 +0100 Subject: [PATCH 48/48] Fix compilation error --- .../services/statemachine/SingleThreadedStateMachineManager.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index a7512af19d..76a0137eb8 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -474,7 +474,8 @@ internal class SingleThreadedStateMachineManager( val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) ?: return // Resurrect flow - flowCreator.createFlowFromCheckpoint(flowId, checkpoint, currentState.reloadCheckpointAfterSuspendCount) ?: return + flowCreator.createFlowFromCheckpoint(flowId, checkpoint, reloadCheckpointAfterSuspendCount = currentState.reloadCheckpointAfterSuspendCount) + ?: return } else { // Just flow initiation message null