From 0619b4ce901ca57e44b02f70ff649f70426adde5 Mon Sep 17 00:00:00 2001 From: stefano Date: Fri, 24 Jan 2020 12:02:16 +0000 Subject: [PATCH 01/49] add --- .../migration/node-core.changelog-v17.xml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 node/src/main/resources/migration/node-core.changelog-v17.xml diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml new file mode 100644 index 0000000000..4293beef3f --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + \ No newline at end of file From 5750c39348eacfa37ce7a8584e61c615aecb174b Mon Sep 17 00:00:00 2001 From: stefano Date: Fri, 24 Jan 2020 13:24:08 +0000 Subject: [PATCH 02/49] start work on liquibase schema for new checkpoints --- Jenkinsfile | 2 +- .../migration/node-core.changelog-master.xml | 4 + .../node-core.changelog-v17-keys.xml | 34 +++++ .../node-core.changelog-v17-postgres.xml | 139 ++++++++++++++++++ .../migration/node-core.changelog-v17.xml | 128 +++++++++++++++- 5 files changed, 302 insertions(+), 5 deletions(-) create mode 100644 node/src/main/resources/migration/node-core.changelog-v17-keys.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v17-postgres.xml diff --git a/Jenkinsfile b/Jenkinsfile index dcd0522239..4d41a7eb2f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'aks' } options { timestamps() } environment { diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index 28842e0825..8c5a7916e1 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -31,4 +31,8 @@ + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml new file mode 100644 index 0000000000..3253f2ed22 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml new file mode 100644 index 0000000000..d850c57f83 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -0,0 +1,139 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 4293beef3f..77dd65848b 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -4,16 +4,136 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd" logicalFilePath="migration/node-services.changelog-init.xml"> - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 3b0c1b98a1ee551aba9888ff1baa36fd8431d15c Mon Sep 17 00:00:00 2001 From: stefano Date: Sun, 26 Jan 2020 18:44:03 +0000 Subject: [PATCH 03/49] switch to new version of plugin --- .ci/dev/localStorageClass.yml | 5 +++++ build.gradle | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .ci/dev/localStorageClass.yml diff --git a/.ci/dev/localStorageClass.yml b/.ci/dev/localStorageClass.yml new file mode 100644 index 0000000000..f380c51dbe --- /dev/null +++ b/.ci/dev/localStorageClass.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: testing-storage +provisioner: microk8s.io/hostpath \ No newline at end of file diff --git a/build.gradle b/build.gradle index bdb66ed4ec..e5dcc0f818 100644 --- a/build.gradle +++ b/build.gradle @@ -183,7 +183,7 @@ buildscript { // Capsule gradle plugin forked and maintained locally to support Gradle 5.x // See https://github.com/corda/gradle-capsule-plugin classpath "us.kirchmeier:gradle-capsule-plugin:1.0.4_r3" - classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-SNAPSHOT", changing: true + classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-LOCAL-SNAPSHOT", changing: true classpath "com.bmuschko:gradle-docker-plugin:5.0.0" } } From d9f638260ca805881e689f27301bd5e8257dbbe0 Mon Sep 17 00:00:00 2001 From: stefano Date: Sun, 26 Jan 2020 19:01:49 +0000 Subject: [PATCH 04/49] change plugin version to force refresh --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e5dcc0f818..724ea70799 100644 --- a/build.gradle +++ b/build.gradle @@ -183,7 +183,7 @@ buildscript { // Capsule gradle plugin forked and maintained locally to support Gradle 5.x // See https://github.com/corda/gradle-capsule-plugin classpath "us.kirchmeier:gradle-capsule-plugin:1.0.4_r3" - classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-LOCAL-SNAPSHOT", changing: true + classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-LOCAL-K8s-SNAPSHOT", changing: true classpath "com.bmuschko:gradle-docker-plugin:5.0.0" } } From 6435a132e49269679cb63248d5b08dd9cca89ae7 Mon Sep 17 00:00:00 2001 From: stefano Date: Sun, 26 Jan 2020 20:27:19 +0000 Subject: [PATCH 05/49] try using pre-populated cache --- Jenkinsfile | 8 ++++---- build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dcd0522239..6fd3f917a6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -5,7 +5,7 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'local-k8s' } options { timestamps() } environment { @@ -24,7 +24,7 @@ pipeline { "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelUnitTest --stacktrace" + " clean pushBuildImage preAllocateForAllParallelIntegrationTest --stacktrace" } sh "kubectl auth can-i get pods" } @@ -42,7 +42,7 @@ pipeline { "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Dgit.branch=\"\${GIT_BRANCH}\" " + "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelIntegrationTest allParallelIntegrationTest --stacktrace" + " allParallelIntegrationTest --stacktrace" } } stage('Unit Tests') { @@ -55,7 +55,7 @@ pipeline { "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Dgit.branch=\"\${GIT_BRANCH}\" " + "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelUnitTest allParallelUnitTest --stacktrace" + " allParallelUnitTest --stacktrace" } } } diff --git a/build.gradle b/build.gradle index 724ea70799..804f3897b7 100644 --- a/build.gradle +++ b/build.gradle @@ -183,7 +183,7 @@ buildscript { // Capsule gradle plugin forked and maintained locally to support Gradle 5.x // See https://github.com/corda/gradle-capsule-plugin classpath "us.kirchmeier:gradle-capsule-plugin:1.0.4_r3" - classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-LOCAL-K8s-SNAPSHOT", changing: true + classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: "1.2-LOCAL-K8S-SHARED-CACHE-SNAPSHOT", changing: true classpath "com.bmuschko:gradle-docker-plugin:5.0.0" } } From 8c6904e9ef2efc24c852f4076a90ec8bcfb6af3f Mon Sep 17 00:00:00 2001 From: stefano Date: Sun, 26 Jan 2020 21:11:59 +0000 Subject: [PATCH 06/49] do not preallocate --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6fd3f917a6..0b1039cb5d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,7 +24,7 @@ pipeline { "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage preAllocateForAllParallelIntegrationTest --stacktrace" + " clean pushBuildImage --stacktrace" } sh "kubectl auth can-i get pods" } From 4465c15d396d3d1e700e1e3fa38fb61888668cef Mon Sep 17 00:00:00 2001 From: stefano Date: Sun, 26 Jan 2020 21:16:55 +0000 Subject: [PATCH 07/49] no daemon --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0b1039cb5d..4c679de085 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -19,7 +19,7 @@ pipeline { stage('Corda Pull Request - Generate Build Image') { steps { withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + + sh "./gradlew --no-daemon " + "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + @@ -34,7 +34,7 @@ pipeline { parallel { stage('Integration Tests') { steps { - sh "./gradlew " + + sh "./gradlew --no-daemon " + "-DbuildId=\"\${BUILD_ID}\" " + "-Dkubenetize=true " + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + @@ -47,7 +47,7 @@ pipeline { } stage('Unit Tests') { steps { - sh "./gradlew " + + sh "./gradlew --no-daemon " + "-DbuildId=\"\${BUILD_ID}\" " + "-Dkubenetize=true " + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + From 7af363d1a6ace73507c5b57e8c649a160deb188a Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 27 Jan 2020 09:14:27 +0000 Subject: [PATCH 08/49] reduce cores per fork to increase parallelism --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 804f3897b7..f12fb23445 100644 --- a/build.gradle +++ b/build.gradle @@ -634,7 +634,7 @@ task allParallelIntegrationTest(type: ParallelTestGroup) { testGroups "integrationTest" numberOfShards 10 streamOutput false - coresPerFork 5 + coresPerFork 4 memoryInGbPerFork 12 distribute DistributeTestsBy.METHOD nodeTaints "big" From 47c14673c404bb707994c12e77093aee9cb6b8cc Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 27 Jan 2020 11:20:13 +0000 Subject: [PATCH 09/49] address initial review comments --- .../resources/migration/node-core.changelog-v17-postgres.xml | 4 ++-- node/src/main/resources/migration/node-core.changelog-v17.xml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index d850c57f83..13ea6dab2d 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -28,7 +28,7 @@ - + @@ -98,7 +98,7 @@ - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 77dd65848b..8cfad5f8b2 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -28,7 +28,7 @@ - + @@ -98,7 +98,7 @@ - + From a3bc624e16e4af527f177164119e1154b9bd70a3 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 27 Jan 2020 12:19:14 +0000 Subject: [PATCH 10/49] add message to exceptions table --- .../resources/migration/node-core.changelog-v17-postgres.xml | 3 +++ node/src/main/resources/migration/node-core.changelog-v17.xml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 13ea6dab2d..44c9a73583 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -86,6 +86,9 @@ + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 8cfad5f8b2..66c5ff7bfb 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -86,6 +86,9 @@ + + + From 93623d73175e30fbf54d34282e09b606bd6255b8 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 27 Jan 2020 12:56:36 +0000 Subject: [PATCH 11/49] even more aggresive cpu allocation --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index f12fb23445..1228d771bf 100644 --- a/build.gradle +++ b/build.gradle @@ -634,7 +634,7 @@ task allParallelIntegrationTest(type: ParallelTestGroup) { testGroups "integrationTest" numberOfShards 10 streamOutput false - coresPerFork 4 + coresPerFork 2 memoryInGbPerFork 12 distribute DistributeTestsBy.METHOD nodeTaints "big" @@ -644,7 +644,7 @@ task allParallelUnitTest(type: ParallelTestGroup) { testGroups "test" numberOfShards 10 streamOutput false - coresPerFork 3 + coresPerFork 2 memoryInGbPerFork 12 distribute DistributeTestsBy.CLASS nodeTaints "small" From 78d83e9583d307346914277b7f7afc4be173ca90 Mon Sep 17 00:00:00 2001 From: stefano Date: Tue, 28 Jan 2020 16:55:14 +0000 Subject: [PATCH 12/49] start adding hibernate entities --- .../persistence/DBCheckpointStorage.kt | 30 +++++++++++++++++++ .../migration/node-core.changelog-master.xml | 6 ++-- .../node-core.changelog-v17-postgres.xml | 4 +-- .../migration/node-core.changelog-v17.xml | 4 +-- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index b1eec763f6..1ffa860b0d 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -25,6 +25,36 @@ import java.sql.SQLException class DBCheckpointStorage : CheckpointStorage { val log: Logger = LoggerFactory.getLogger(this::class.java) + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") + class DBFlowCheckpoint( + + ) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints_blobs") + class DBFlowCheckpointBlob( + + ) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_results") + class DBFlowResult( + + ) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_exceptions") + class DBFlowException( + + ) + + @Entity + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_metadata") + class DBFlowMetadata( + + ) + @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") class DBCheckpoint( diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index 8c5a7916e1..b320960f02 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -31,8 +31,8 @@ - - - + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 44c9a73583..5c4960b7a1 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -100,13 +100,13 @@ - + - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 66c5ff7bfb..5143e69574 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -100,13 +100,13 @@ - + - + From 7b3da954561021c8fc7c80ef0ae9517ca258ad5c Mon Sep 17 00:00:00 2001 From: stefano Date: Wed, 29 Jan 2020 17:10:34 +0000 Subject: [PATCH 13/49] flesh out entities for new checkpointing --- .../net/corda/core/internal/FlowIORequest.kt | 11 ++- .../persistence/DBCheckpointStorage.kt | 69 +++++++++++++++++-- .../node/services/statemachine/FlowMonitor.kt | 2 +- .../statemachine/FlowStateMachineImpl.kt | 2 +- .../migration/node-core.changelog-v17.xml | 5 +- 5 files changed, 80 insertions(+), 9 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt b/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt index 494c5099aa..0d54a4715a 100644 --- a/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt +++ b/core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt @@ -45,6 +45,7 @@ sealed class FlowIORequest { * @property shouldRetrySend specifies whether the send should be retried. * @return a map from session to received message. */ + //net.corda.core.internal.FlowIORequest.SendAndReceive data class SendAndReceive( val sessionToMessage: Map>, val shouldRetrySend: Boolean @@ -80,7 +81,15 @@ sealed class FlowIORequest { /** * Suspend the flow until all Initiating sessions are confirmed. */ - object WaitForSessionConfirmations : FlowIORequest() + class WaitForSessionConfirmations : FlowIORequest() { + override fun equals(other: Any?): Boolean { + return this === other + } + + override fun hashCode(): Int { + return System.identityHashCode(this) + } + } /** * Execute the specified [operation], suspend the flow until completion. diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 1ffa860b0d..c739a4876e 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -1,7 +1,9 @@ package net.corda.node.services.persistence import net.corda.core.flows.StateMachineRunId +import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes +import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.debug import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.Checkpoint @@ -16,8 +18,13 @@ import javax.persistence.Column import javax.persistence.Entity import javax.persistence.Id import org.hibernate.annotations.Type +import java.math.BigInteger import java.sql.Connection import java.sql.SQLException +import java.time.Instant +import javax.persistence.FetchType +import javax.persistence.JoinColumn +import javax.persistence.OneToOne /** * Simple checkpoint key value storage in DB. @@ -25,16 +32,71 @@ import java.sql.SQLException class DBCheckpointStorage : CheckpointStorage { val log: Logger = LoggerFactory.getLogger(this::class.java) + enum class FlowStatus { + RUNNABLE, + FAILED, + COMPLETED, + HOSPITALIZED, + KILLED, + PAUSED + } + @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") class DBFlowCheckpoint( + @Id + @Column(name = "flow_id", length = 64, nullable = false) + private var id: String? = null, + @OneToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "id") + private var blob: DBFlowCheckpointBlob? = null, + + @OneToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "id") + private var result: DBFlowResult? = null, + + @OneToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "id") + private var exceptionDetails: DBFlowException? = null, + + @OneToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "flow_id") + private var flowMetadata: DBFlowMetadata? = null, + + @Column(name = "status") + private var status: FlowStatus? = null, + + @Column(name = "compatible") + private var compatible: Boolean? = null, + + @Column(name = "progress_step") + private var progressStep: String? = null, + + @Column(name = "flow_io_request") + private val ioRequestType: Class>? = null, + + @Column(name = "timestamp") + private val checkpointInstant: Instant? = null ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints_blobs") class DBFlowCheckpointBlob( + @Id + @Column(name = "id", nullable = false) + private var id: BigInteger? = null, + @Type(type = "corda-blob") + @Column(name = "checkpoint_value", nullable = false) + var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + + @Type(type = "corda-blob") + @Column(name = "flow_state", nullable = false) + var flowStack: ByteArray = EMPTY_BYTE_ARRAY, + + @Column(name = "timestamp") + private val instant: Instant? = null ) @Entity @@ -65,10 +127,10 @@ class DBCheckpointStorage : CheckpointStorage { @Type(type = "corda-blob") @Column(name = "checkpoint_value", nullable = false) - var checkpoint: ByteArray = EMPTY_BYTE_ARRAY + var checkpoint: ByteArray = EMPTY_BYTE_ARRAY ) { - override fun toString() = "DBCheckpoint(checkpointId = ${checkpointId}, checkpointSize = ${checkpoint.size})" - } + override fun toString() = "DBCheckpoint(checkpointId = ${checkpointId}, checkpointSize = ${checkpoint.size})" + } override fun addCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) { currentDBSession().save(DBCheckpoint().apply { @@ -86,7 +148,6 @@ class DBCheckpointStorage : CheckpointStorage { }) } - override fun removeCheckpoint(id: StateMachineRunId): Boolean { val session = currentDBSession() val criteriaBuilder = session.criteriaBuilder diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt index b947f62f2b..9f80005880 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowMonitor.kt @@ -81,7 +81,7 @@ internal class FlowMonitor( is FlowIORequest.WaitForLedgerCommit -> "for the ledger to commit transaction with hash ${request.hash}" is FlowIORequest.GetFlowInfo -> "to get flow information from parties ${request.sessions.partiesInvolved()}" is FlowIORequest.Sleep -> "to wake up from sleep ending at ${LocalDateTime.ofInstant(request.wakeUpAfter, ZoneId.systemDefault())}" - FlowIORequest.WaitForSessionConfirmations -> "for sessions to be confirmed" + is FlowIORequest.WaitForSessionConfirmations -> "for sessions to be confirmed" is FlowIORequest.ExecuteAsyncOperation -> "for asynchronous operation of type ${request.operation::javaClass} to complete" FlowIORequest.ForceCheckpoint -> "for forcing a checkpoint at an arbitrary point in a flow" } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 4a9a407473..2bbdae6ba3 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -269,7 +269,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, Thread.currentThread().contextClassLoader = (serviceHub.cordappProvider as CordappProviderImpl).cordappLoader.appClassLoader val result = logic.call() - suspend(FlowIORequest.WaitForSessionConfirmations, maySkipCheckpoint = true) + suspend(FlowIORequest.WaitForSessionConfirmations(), maySkipCheckpoint = true) Try.Success(result) } catch (t: Throwable) { if(t.isUnrecoverable()) { diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 5143e69574..70dfa0f3f3 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -22,7 +22,7 @@ - + @@ -31,7 +31,8 @@ - + + From 2b079bd92ea61241cbbe05e59742465d7919646c Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 30 Jan 2020 12:17:26 +0000 Subject: [PATCH 14/49] working node startup with new tables and entities --- .../persistence/DBCheckpointStorage.kt | 101 +++++++++++++++--- .../node/services/schema/NodeSchemaService.kt | 8 ++ .../migration/node-core.changelog-master.xml | 6 +- .../node-core.changelog-v17-keys.xml | 10 +- .../node-core.changelog-v17-postgres.xml | 27 +++-- .../migration/node-core.changelog-v17.xml | 14 +-- 6 files changed, 118 insertions(+), 48 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index c739a4876e..e2449568e7 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -18,6 +18,7 @@ import javax.persistence.Column import javax.persistence.Entity import javax.persistence.Id import org.hibernate.annotations.Type +import java.lang.Exception import java.math.BigInteger import java.sql.Connection import java.sql.SQLException @@ -41,43 +42,47 @@ class DBCheckpointStorage : CheckpointStorage { PAUSED } + enum class StartReason { + RPC, FLOW, SERVICE, SCHEDULED, INITIATED + } + @Entity - @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints_new") class DBFlowCheckpoint( @Id @Column(name = "flow_id", length = 64, nullable = false) - private var id: String? = null, + var id: String? = null, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "id") - private var blob: DBFlowCheckpointBlob? = null, + @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") + var blob: DBFlowCheckpointBlob? = null, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "id") - private var result: DBFlowResult? = null, + @JoinColumn(name = "result_id", referencedColumnName = "id") + var result: DBFlowResult? = null, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "id") - private var exceptionDetails: DBFlowException? = null, + @JoinColumn(name = "error_id", referencedColumnName = "id") + var exceptionDetails: DBFlowException? = null, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "flow_id") - private var flowMetadata: DBFlowMetadata? = null, + var flowMetadata: DBFlowMetadata? = null, @Column(name = "status") - private var status: FlowStatus? = null, + var status: FlowStatus? = null, @Column(name = "compatible") - private var compatible: Boolean? = null, + var compatible: Boolean? = null, @Column(name = "progress_step") - private var progressStep: String? = null, + var progressStep: String? = null, @Column(name = "flow_io_request") - private val ioRequestType: Class>? = null, + var ioRequestType: Class>? = null, @Column(name = "timestamp") - private val checkpointInstant: Instant? = null + var checkpointInstant: Instant? = null ) @Entity @@ -85,7 +90,7 @@ class DBCheckpointStorage : CheckpointStorage { class DBFlowCheckpointBlob( @Id @Column(name = "id", nullable = false) - private var id: BigInteger? = null, + var id: BigInteger? = null, @Type(type = "corda-blob") @Column(name = "checkpoint_value", nullable = false) @@ -96,25 +101,89 @@ class DBCheckpointStorage : CheckpointStorage { var flowStack: ByteArray = EMPTY_BYTE_ARRAY, @Column(name = "timestamp") - private val instant: Instant? = null + var persistedInstant: Instant? = null ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_results") class DBFlowResult( + @Id + @Column(name = "id", nullable = false) + var id: BigInteger? = null, + @Type(type = "corda-blob") + @Column(name = "result_value", nullable = false) + var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + + @Column(name = "timestamp") + val persistedInstant: Instant? = null ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_exceptions") class DBFlowException( + @Id + @Column(name = "id", nullable = false) + var id: BigInteger? = null, + @Column(name = "type", nullable = false) + var type: Class? = null, + + @Type(type = "corda-blob") + @Column(name = "exception_value", nullable = false) + var value: ByteArray, + + @Column(name = "exception_message") + var message: String? = null, + + @Column(name = "timestamp") + val persistedInstant: Instant? = null ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_metadata") class DBFlowMetadata( + @Id + @Column(name = "id", nullable = false) + var id: BigInteger? = null, + + @Column(name = "flow_id", length = 64, nullable = false) + var flowId: String? = null, + + @Column(name = "flow_name", nullable = false) + var flowName: String? = null, + + @Column(name = "flow_identifier", nullable = true) + var userSuppliedIdentifier: String? = null, + + @Column(name = "started_type", nullable = true) + var startType: StartReason? = null, + + @Column(name = "flow_parameters", nullable = true) + var initialParameters: ByteArray? = null, + + @Column(name = "cordapp_name", nullable = true) + var launchingCordapp: String? = null, + + @Column(name = "platform_version", nullable = true) + var platformVersion: Int? = null, + + @Column(name = "rpc_user", nullable = true) + var rpcUsername: String? = null, + + @Column(name = "invocation_time", nullable = true) + var invocationInstant: Instant? = null, + + @Column(name = "received_time", nullable = true) + var receivedInstant: Instant? = null, + + @Column(name = "start_time", nullable = true) + var startInstant: Instant? = null, + + @Column(name = "finish_time", nullable = true) + var finishInstant: Instant? = null + ) @Entity diff --git a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt index a834dbb3ab..73d7cbb3d9 100644 --- a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt +++ b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt @@ -34,6 +34,14 @@ class NodeSchemaService(private val extraSchemas: Set = emptySet() object NodeCoreV1 : MappedSchema(schemaFamily = NodeCore.javaClass, version = 1, mappedTypes = listOf(DBCheckpointStorage.DBCheckpoint::class.java, + + //new checkpoints - keeping old around to allow testing easily (for now) + DBCheckpointStorage.DBFlowCheckpoint::class.java, + DBCheckpointStorage.DBFlowCheckpointBlob::class.java, + DBCheckpointStorage.DBFlowResult::class.java, + DBCheckpointStorage.DBFlowException::class.java, + DBCheckpointStorage.DBFlowMetadata::class.java, + DBTransactionStorage.DBTransaction::class.java, BasicHSMKeyManagementService.PersistentKey::class.java, NodeSchedulerService.PersistentScheduledState::class.java, diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index b320960f02..8c5a7916e1 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -31,8 +31,8 @@ - - - + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml index 3253f2ed22..fdc812a02c 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml @@ -5,7 +5,7 @@ logicalFilePath="migration/node-services.changelog-init.xml"> - + @@ -13,21 +13,21 @@ - - - + referencedColumnNames="flow_id" referencedTableName="node_checkpoints_new"/> diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 5c4960b7a1..9602ec51b3 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -4,12 +4,8 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd" logicalFilePath="migration/node-services.changelog-init.xml"> - - - - - - + + @@ -22,7 +18,7 @@ - + @@ -31,7 +27,8 @@ - + + @@ -40,7 +37,7 @@ - + @@ -61,7 +58,7 @@ - + @@ -75,12 +72,12 @@ - + - + @@ -95,7 +92,7 @@ - + @@ -109,7 +106,7 @@ - + @@ -118,7 +115,7 @@ - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 70dfa0f3f3..2c1789fc95 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -4,12 +4,8 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd" logicalFilePath="migration/node-services.changelog-init.xml"> - - - - - + @@ -22,7 +18,7 @@ - + @@ -81,7 +77,7 @@ - + @@ -110,7 +106,7 @@ - + @@ -119,7 +115,7 @@ - + From eec9ada6bab92360e2e27ef4480137417fb29f7d Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 30 Jan 2020 18:18:45 +0000 Subject: [PATCH 15/49] persist some nonsense data to see if it breaks the statemachine --- .../persistence/DBCheckpointStorage.kt | 5 +-- .../SingleThreadedStateMachineManager.kt | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index e2449568e7..cf058dd930 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -66,7 +66,7 @@ class DBCheckpointStorage : CheckpointStorage { var exceptionDetails: DBFlowException? = null, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "flow_id") + @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") var flowMetadata: DBFlowMetadata? = null, @Column(name = "status") @@ -145,9 +145,6 @@ class DBCheckpointStorage : CheckpointStorage { class DBFlowMetadata( @Id - @Column(name = "id", nullable = false) - var id: BigInteger? = null, - @Column(name = "flow_id", length = 64, nullable = false) var flowId: String? = null, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 34c9b77582..3383ef9ba7 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -34,6 +34,7 @@ import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.ServiceHubInternal import net.corda.node.services.config.shouldCheckCheckpoints import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.statemachine.FlowStateMachineImpl.Companion.createSubFlowVersion import net.corda.node.services.statemachine.interceptors.* import net.corda.node.services.statemachine.transitions.StateMachine @@ -42,6 +43,7 @@ import net.corda.node.utilities.errorAndTerminate import net.corda.node.utilities.injectOldProgressTracker import net.corda.node.utilities.isEnabledTimedFlow import net.corda.nodeapi.internal.persistence.CordaPersistence +import net.corda.nodeapi.internal.persistence.currentDBSession import net.corda.nodeapi.internal.persistence.wrapWithDatabaseTransaction import net.corda.serialization.internal.CheckpointSerializeAsTokenContextImpl import net.corda.serialization.internal.withTokenContext @@ -586,6 +588,46 @@ class SingleThreadedStateMachineManager( val flowAlreadyExists = mutex.locked { flows[flowId] != null } val existingCheckpoint = if (flowAlreadyExists) { + + val currentDBSession = currentDBSession() + val dbFlowCheckpoint = currentDBSession.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, flowId.toString()) + ?: DBCheckpointStorage.DBFlowCheckpoint( + flowId.toString(), + null, + null, + null, + null, + DBCheckpointStorage.FlowStatus.RUNNABLE, + true, + flowLogic.progressTracker?.currentStep?.toString(), + null, + null + ) + + var cordappName : String? = null + var initiatingParty: String? = null + var startReason: DBCheckpointStorage.StartReason = DBCheckpointStorage.StartReason.FLOW + when (flowStart) { + is FlowStart.Initiated -> { + cordappName = flowStart.initiatedFlowInfo.appName + initiatingParty = flowStart.peerSession.counterparty.name.toString() + startReason = DBCheckpointStorage.StartReason.INITIATED + } + } + + currentDBSession.get(DBCheckpointStorage.DBFlowMetadata::class.java, flowId.toString()) ?: DBCheckpointStorage.DBFlowMetadata( + flowId = flowId.toString(), + flowName = "thisIsAPlaceholder", + userSuppliedIdentifier = "thisIsAnotherPlaceholder", + startType = startReason, + initialParameters = null, + launchingCordapp = cordappName + ) + + + currentDBSession.persist(dbFlowCheckpoint) + + // Load the flow's checkpoint // The checkpoint will be missing if the flow failed before persisting the original checkpoint // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) From a28c15c2fd5fc4175f8e0e355180586040e9a78d Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 10 Feb 2020 11:52:54 +0000 Subject: [PATCH 16/49] address dan review comments --- .../persistence/DBCheckpointStorage.kt | 46 ++++----- .../SingleThreadedStateMachineManager.kt | 98 ++++++------------- 2 files changed, 51 insertions(+), 93 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index cf058dd930..c2ba70135c 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -51,38 +51,38 @@ class DBCheckpointStorage : CheckpointStorage { class DBFlowCheckpoint( @Id @Column(name = "flow_id", length = 64, nullable = false) - var id: String? = null, + var id: String, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") - var blob: DBFlowCheckpointBlob? = null, + var blob: DBFlowCheckpointBlob, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "result_id", referencedColumnName = "id") - var result: DBFlowResult? = null, + var result: DBFlowResult, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "error_id", referencedColumnName = "id") - var exceptionDetails: DBFlowException? = null, + var exceptionDetails: DBFlowException, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") - var flowMetadata: DBFlowMetadata? = null, + var flowMetadata: DBFlowMetadata, @Column(name = "status") - var status: FlowStatus? = null, + var status: FlowStatus, @Column(name = "compatible") - var compatible: Boolean? = null, + var compatible: Boolean, @Column(name = "progress_step") - var progressStep: String? = null, + var progressStep: String, @Column(name = "flow_io_request") - var ioRequestType: Class>? = null, + var ioRequestType: Class>, @Column(name = "timestamp") - var checkpointInstant: Instant? = null + var checkpointInstant: Instant ) @Entity @@ -131,7 +131,7 @@ class DBCheckpointStorage : CheckpointStorage { @Type(type = "corda-blob") @Column(name = "exception_value", nullable = false) - var value: ByteArray, + var value: ByteArray = EMPTY_BYTE_ARRAY, @Column(name = "exception_message") var message: String? = null, @@ -146,40 +146,40 @@ class DBCheckpointStorage : CheckpointStorage { @Id @Column(name = "flow_id", length = 64, nullable = false) - var flowId: String? = null, + var flowId: String, @Column(name = "flow_name", nullable = false) - var flowName: String? = null, + var flowName: String, @Column(name = "flow_identifier", nullable = true) - var userSuppliedIdentifier: String? = null, + var userSuppliedIdentifier: String, @Column(name = "started_type", nullable = true) - var startType: StartReason? = null, + var startType: StartReason, @Column(name = "flow_parameters", nullable = true) - var initialParameters: ByteArray? = null, + var initialParameters: ByteArray = EMPTY_BYTE_ARRAY, @Column(name = "cordapp_name", nullable = true) - var launchingCordapp: String? = null, + var launchingCordapp: String, @Column(name = "platform_version", nullable = true) - var platformVersion: Int? = null, + var platformVersion: Int, @Column(name = "rpc_user", nullable = true) - var rpcUsername: String? = null, + var rpcUsername: String, @Column(name = "invocation_time", nullable = true) - var invocationInstant: Instant? = null, + var invocationInstant: Instant, @Column(name = "received_time", nullable = true) - var receivedInstant: Instant? = null, + var receivedInstant: Instant, @Column(name = "start_time", nullable = true) - var startInstant: Instant? = null, + var startInstant: Instant, @Column(name = "finish_time", nullable = true) - var finishInstant: Instant? = null + var finishInstant: Instant ) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 3383ef9ba7..371d5ed92a 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -34,7 +34,6 @@ import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.ServiceHubInternal import net.corda.node.services.config.shouldCheckCheckpoints import net.corda.node.services.messaging.DeduplicationHandler -import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.statemachine.FlowStateMachineImpl.Companion.createSubFlowVersion import net.corda.node.services.statemachine.interceptors.* import net.corda.node.services.statemachine.transitions.StateMachine @@ -43,7 +42,6 @@ import net.corda.node.utilities.errorAndTerminate import net.corda.node.utilities.injectOldProgressTracker import net.corda.node.utilities.isEnabledTimedFlow import net.corda.nodeapi.internal.persistence.CordaPersistence -import net.corda.nodeapi.internal.persistence.currentDBSession import net.corda.nodeapi.internal.persistence.wrapWithDatabaseTransaction import net.corda.serialization.internal.CheckpointSerializeAsTokenContextImpl import net.corda.serialization.internal.withTokenContext @@ -372,11 +370,11 @@ class SingleThreadedStateMachineManager( } // Resurrect flow createFlowFromCheckpoint( - id = flowId, - serializedCheckpoint = serializedCheckpoint, - initialDeduplicationHandler = null, - isAnyCheckpointPersisted = true, - isStartIdempotent = false + id = flowId, + serializedCheckpoint = serializedCheckpoint, + initialDeduplicationHandler = null, + isAnyCheckpointPersisted = true, + isStartIdempotent = false ) ?: return } else { // Just flow initiation message @@ -411,9 +409,9 @@ class SingleThreadedStateMachineManager( // Failed to retry - manually put the flow in for observation rather than // relying on the [HospitalisingInterceptor] to do so val exceptions = (currentState.checkpoint.errorState as? ErrorState.Errored) - ?.errors - ?.map { it.exception } - ?.plus(e) ?: emptyList() + ?.errors + ?.map { it.exception } + ?.plus(e) ?: emptyList() logger.info("Failed to retry flow $flowId, keeping in for observation and aborting") flowHospital.forceIntoOvernightObservation(flowId, exceptions) throw e @@ -433,11 +431,11 @@ class SingleThreadedStateMachineManager( private fun onExternalStartFlow(event: ExternalEvent.ExternalStartFlowEvent) { val future = startFlow( - event.flowId, - event.flowLogic, - event.context, - ourIdentity = null, - deduplicationHandler = event.deduplicationHandler + event.flowId, + event.flowLogic, + event.context, + ourIdentity = null, + deduplicationHandler = event.deduplicationHandler ) event.wireUpFuture(future) } @@ -505,14 +503,14 @@ class SingleThreadedStateMachineManager( is InitiatedFlowFactory.CorDapp -> null } startInitiatedFlow( - event.flowId, - flowLogic, - event.deduplicationHandler, - senderSession, - initiatedSessionId, - sessionMessage, - senderCoreFlowVersion, - initiatedFlowInfo + event.flowId, + flowLogic, + event.deduplicationHandler, + senderSession, + initiatedSessionId, + sessionMessage, + senderCoreFlowVersion, + initiatedFlowInfo ) } catch (t: Throwable) { logger.warn("Unable to initiate flow from $sender (appName=${sessionMessage.appName} " + @@ -588,46 +586,6 @@ class SingleThreadedStateMachineManager( val flowAlreadyExists = mutex.locked { flows[flowId] != null } val existingCheckpoint = if (flowAlreadyExists) { - - val currentDBSession = currentDBSession() - val dbFlowCheckpoint = currentDBSession.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, flowId.toString()) - ?: DBCheckpointStorage.DBFlowCheckpoint( - flowId.toString(), - null, - null, - null, - null, - DBCheckpointStorage.FlowStatus.RUNNABLE, - true, - flowLogic.progressTracker?.currentStep?.toString(), - null, - null - ) - - var cordappName : String? = null - var initiatingParty: String? = null - var startReason: DBCheckpointStorage.StartReason = DBCheckpointStorage.StartReason.FLOW - when (flowStart) { - is FlowStart.Initiated -> { - cordappName = flowStart.initiatedFlowInfo.appName - initiatingParty = flowStart.peerSession.counterparty.name.toString() - startReason = DBCheckpointStorage.StartReason.INITIATED - } - } - - currentDBSession.get(DBCheckpointStorage.DBFlowMetadata::class.java, flowId.toString()) ?: DBCheckpointStorage.DBFlowMetadata( - flowId = flowId.toString(), - flowName = "thisIsAPlaceholder", - userSuppliedIdentifier = "thisIsAnotherPlaceholder", - startType = startReason, - initialParameters = null, - launchingCordapp = cordappName - ) - - - currentDBSession.persist(dbFlowCheckpoint) - - // Load the flow's checkpoint // The checkpoint will be missing if the flow failed before persisting the original checkpoint // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) @@ -647,13 +605,13 @@ class SingleThreadedStateMachineManager( null } val checkpoint = existingCheckpoint ?: Checkpoint.create( - invocationContext, - flowStart, - flowLogic.javaClass, - frozenFlowLogic, - ourIdentity, - flowCorDappVersion, - flowLogic.isEnabledTimedFlow() + invocationContext, + flowStart, + flowLogic.javaClass, + frozenFlowLogic, + ourIdentity, + flowCorDappVersion, + flowLogic.isEnabledTimedFlow() ).getOrThrow() val startedFuture = openFuture() From 9e8ce6473d2c12bd16b54fbefdb97d44ac8fcb67 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 10 Feb 2020 13:22:51 +0000 Subject: [PATCH 17/49] address dan review comments pt2 --- .../persistence/DBCheckpointStorage.kt | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index c2ba70135c..c9d883a5b0 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -59,11 +59,11 @@ class DBCheckpointStorage : CheckpointStorage { @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "result_id", referencedColumnName = "id") - var result: DBFlowResult, + var result: DBFlowResult?, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "error_id", referencedColumnName = "id") - var exceptionDetails: DBFlowException, + var exceptionDetails: DBFlowException?, @OneToOne(fetch = FetchType.LAZY) @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") @@ -127,7 +127,7 @@ class DBCheckpointStorage : CheckpointStorage { var id: BigInteger? = null, @Column(name = "type", nullable = false) - var type: Class? = null, + var type: Class, @Type(type = "corda-blob") @Column(name = "exception_value", nullable = false) @@ -152,34 +152,34 @@ class DBCheckpointStorage : CheckpointStorage { var flowName: String, @Column(name = "flow_identifier", nullable = true) - var userSuppliedIdentifier: String, + var userSuppliedIdentifier: String?, - @Column(name = "started_type", nullable = true) + @Column(name = "started_type", nullable = false) var startType: StartReason, - @Column(name = "flow_parameters", nullable = true) + @Column(name = "flow_parameters", nullable = false) var initialParameters: ByteArray = EMPTY_BYTE_ARRAY, - @Column(name = "cordapp_name", nullable = true) + @Column(name = "cordapp_name", nullable = false) var launchingCordapp: String, - @Column(name = "platform_version", nullable = true) + @Column(name = "platform_version", nullable = false) var platformVersion: Int, - @Column(name = "rpc_user", nullable = true) + @Column(name = "rpc_user", nullable = false) var rpcUsername: String, - @Column(name = "invocation_time", nullable = true) + @Column(name = "invocation_time", nullable = false) var invocationInstant: Instant, - @Column(name = "received_time", nullable = true) + @Column(name = "received_time", nullable = false) var receivedInstant: Instant, @Column(name = "start_time", nullable = true) - var startInstant: Instant, + var startInstant: Instant?, @Column(name = "finish_time", nullable = true) - var finishInstant: Instant + var finishInstant: Instant? ) From 8d1b6cf4990dc741662634727a8c26806679db40 Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Tue, 18 Feb 2020 15:31:46 +0000 Subject: [PATCH 18/49] CORDA-3432 update structure of checkpoint class (#5983) * Split StateMachine State into 2 classes The idea is this better reflects the database structure. Added a few helper methods to copy and update state. * Doc + Improve Checkpoint API * Rename methods to be more clear --- .../corda/node/internal/CheckpointVerifier.kt | 2 +- .../node/services/rpc/CheckpointDumperImpl.kt | 8 +-- .../services/statemachine/DeduplicationId.kt | 2 +- .../statemachine/FlowStateMachineImpl.kt | 8 +-- .../SingleThreadedStateMachineManager.kt | 6 +- .../statemachine/StaffedFlowHospital.kt | 5 +- .../statemachine/StateMachineState.kt | 70 ++++++++++++++----- .../DeliverSessionMessageTransition.kt | 27 +++---- .../transitions/ErrorFlowTransition.kt | 9 ++- .../transitions/StartedFlowTransition.kt | 32 ++++----- .../transitions/TopLevelTransition.kt | 39 +++++------ .../transitions/UnstartedFlowTransition.kt | 4 +- 12 files changed, 122 insertions(+), 90 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt index 901d853109..cb7fe99b6f 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt @@ -52,7 +52,7 @@ object CheckpointVerifier { } // For each Subflow, compare the checkpointed version to the current version. - checkpoint.subFlowStack.forEach { checkFlowCompatible(it, cordappsByHash, platformVersion) } + checkpoint.checkpointState.subFlowStack.forEach { checkFlowCompatible(it, cordappsByHash, platformVersion) } } } } diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index c3116f03cb..d27a480a5a 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -211,7 +211,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri // Poke into Quasar's stack and find the object references to the sub-flows so that we can correctly get the current progress // step for each sub-call. val stackObjects = fiber.getQuasarStack() - subFlowStack.map { it.toJson(stackObjects) } + checkpointState.subFlowStack.map { it.toJson(stackObjects) } } else { emptyList() } @@ -226,9 +226,9 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri timestamp, now ), - origin = invocationContext.origin.toOrigin(), - ourIdentity = ourIdentity, - activeSessions = sessions.mapNotNull { it.value.toActiveSession(it.key) }, + origin = checkpointState.invocationContext.origin.toOrigin(), + ourIdentity = checkpointState.ourIdentity, + activeSessions = checkpointState.sessions.mapNotNull { it.value.toActiveSession(it.key) }, errored = errorState as? ErrorState.Errored ) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/DeduplicationId.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/DeduplicationId.kt index 036c2d2846..aa2778ba49 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/DeduplicationId.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/DeduplicationId.kt @@ -27,7 +27,7 @@ data class DeduplicationId(val toString: String) { * message-id map to change, which means deduplication will not happen correctly. */ fun createForNormal(checkpoint: Checkpoint, index: Int, session: SessionState): DeduplicationId { - return DeduplicationId("N-${session.deduplicationSeed}-${checkpoint.numberOfSuspends}-$index") + return DeduplicationId("N-${session.deduplicationSeed}-${checkpoint.checkpointState.numberOfSuspends}-$index") } /** diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 2bbdae6ba3..9b0541f8cc 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -108,8 +108,8 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, */ override val logger = log override val resultFuture: CordaFuture get() = uncheckedCast(getTransientField(TransientValues::resultFuture)) - override val context: InvocationContext get() = transientState!!.value.checkpoint.invocationContext - override val ourIdentity: Party get() = transientState!!.value.checkpoint.ourIdentity + override val context: InvocationContext get() = transientState!!.value.checkpoint.checkpointState.invocationContext + override val ourIdentity: Party get() = transientState!!.value.checkpoint.checkpointState.ourIdentity internal var hasSoftLockedStates: Boolean = false set(value) { if (value) field = value else throw IllegalArgumentException("Can only set to true") @@ -349,7 +349,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, */ @Suspendable private fun checkpointIfSubflowIdempotent(subFlow: Class>) { - val currentFlow = snapshot().checkpoint.subFlowStack.last().flowClass + val currentFlow = snapshot().checkpoint.checkpointState.subFlowStack.last().flowClass if (!currentFlow.isIdempotentFlow() && subFlow.isIdempotentFlow()) { suspend(FlowIORequest.ForceCheckpoint, false) } @@ -453,7 +453,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, } private fun containsIdempotentFlows(): Boolean { - val subFlowStack = snapshot().checkpoint.subFlowStack + val subFlowStack = snapshot().checkpoint.checkpointState.subFlowStack return subFlowStack.any { IdempotentFlow::class.java.isAssignableFrom(it.flowClass) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 371d5ed92a..1352fb6ec6 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -853,9 +853,9 @@ class SingleThreadedStateMachineManager( private fun getFlowSessionIds(checkpoint: Checkpoint): Set { val initiatedFlowStart = (checkpoint.flowState as? FlowState.Unstarted)?.flowStart as? FlowStart.Initiated return if (initiatedFlowStart == null) { - checkpoint.sessions.keys + checkpoint.checkpointState.sessions.keys } else { - checkpoint.sessions.keys + initiatedFlowStart.initiatedSessionId + checkpoint.checkpointState.sessions.keys + initiatedFlowStart.initiatedSessionId } } @@ -901,7 +901,7 @@ class SingleThreadedStateMachineManager( // final sanity checks require(lastState.pendingDeduplicationHandlers.isEmpty()) { "Flow cannot be removed until all pending deduplications have completed" } require(lastState.isRemoved) { "Flow must be in removable state before removal" } - require(lastState.checkpoint.subFlowStack.size == 1) { "Checkpointed stack must be empty" } + require(lastState.checkpoint.checkpointState.subFlowStack.size == 1) { "Checkpointed stack must be empty" } require(flow.fiber.id !in sessionToFlow.values) { "Flow fibre must not be needed by an existing session" } flow.resultFuture.set(removalReason.flowReturnValue) lastState.flowLogic.progressTracker?.currentStep = ProgressTracker.DONE diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt index dbaeb0e586..d78b56b578 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt @@ -232,7 +232,8 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, } } - val record = MedicalRecord.Flow(time, flowFiber.id, currentState.checkpoint.numberOfSuspends, errors, report.by, outcome) + val numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends + val record = MedicalRecord.Flow(time, flowFiber.id, numberOfSuspends, errors, report.by, outcome) medicalHistory.records += record recordsPublisher.onNext(record) Pair(event, backOffForChronicCondition) @@ -314,7 +315,7 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, } fun timesDischargedForTheSameThing(by: Staff, currentState: StateMachineState): Int { - val lastAdmittanceSuspendCount = currentState.checkpoint.numberOfSuspends + val lastAdmittanceSuspendCount = currentState.checkpoint.checkpointState.numberOfSuspends return records.count { it.outcome == Outcome.DISCHARGE && by in it.by && it.suspendCount == lastAdmittanceSuspendCount } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 77e1153181..dbc37090fa 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -46,22 +46,15 @@ data class StateMachineState( ) /** - * @param invocationContext the initiator of the flow. - * @param ourIdentity the identity the flow is run as. - * @param sessions map of source session ID to session state. - * @param subFlowStack the stack of currently executing subflows. + * @param checkpointState the state of the checkpoint * @param flowState the state of the flow itself, including the frozen fiber/FlowLogic. * @param errorState the "dirtiness" state including the involved errors and their propagation status. - * @param numberOfSuspends the number of flow suspends due to IO API calls. */ data class Checkpoint( - val invocationContext: InvocationContext, - val ourIdentity: Party, - val sessions: SessionMap, // This must preserve the insertion order! - val subFlowStack: List, + val checkpointState: CheckpointState, val flowState: FlowState, val errorState: ErrorState, - val numberOfSuspends: Int + val result: Any? = null ) { val timestamp: Instant = Instant.now() // This will get updated every time a Checkpoint object is created/ created by copy. @@ -79,19 +72,62 @@ data class Checkpoint( ): Try { return SubFlow.create(flowLogicClass, subFlowVersion, isEnabledTimedFlow).map { topLevelSubFlow -> Checkpoint( - invocationContext = invocationContext, - ourIdentity = ourIdentity, - sessions = emptyMap(), - subFlowStack = listOf(topLevelSubFlow), + checkpointState = CheckpointState(invocationContext, ourIdentity, emptyMap(), listOf(topLevelSubFlow), numberOfSuspends = 0), flowState = FlowState.Unstarted(flowStart, frozenFlowLogic), - errorState = ErrorState.Clean, - numberOfSuspends = 0 + errorState = ErrorState.Clean ) } } } + + /** + * Returns a copy of the Checkpoint with a new session map. + * @param sessions the new map of session ID to session state. + */ + fun setSessions(sessions: SessionMap) : Checkpoint { + return copy(checkpointState = checkpointState.copy(sessions = sessions)) + } + + /** + * Returns a copy of the Checkpoint with an extra session added to the session map. + * @param session the extra session to add. + */ + fun addSession(session: Pair) : Checkpoint { + return copy(checkpointState = checkpointState.copy(sessions = checkpointState.sessions + session)) + } + + /** + * Returns a copy of the Checkpoint with a new subFlow stack. + * @param subFlows the new List of subFlows. + */ + fun setSubflows(subFlows: List) : Checkpoint { + return copy(checkpointState = checkpointState.copy(subFlowStack = subFlows)) + } + + /** + * Returns a copy of the Checkpoint with an extra subflow added to the subFlow Stack. + * @param subFlow the subFlow to add to the stack of subFlows + */ + fun addSubflow(subFlow: SubFlow) : Checkpoint { + return copy(checkpointState = checkpointState.copy(subFlowStack = checkpointState.subFlowStack + subFlow)) + } } +/** + * @param invocationContext the initiator of the flow. + * @param ourIdentity the identity the flow is run as. + * @param sessions map of source session ID to session state. + * @param subFlowStack the stack of currently executing subflows. + * @param numberOfSuspends the number of flow suspends due to IO API calls. + */ +data class CheckpointState( + val invocationContext: InvocationContext, + val ourIdentity: Party, + val sessions: SessionMap, // This must preserve the insertion order! + val subFlowStack: List, + val numberOfSuspends: Int +) + /** * The state of a session. */ @@ -251,4 +287,4 @@ sealed class SubFlowVersion { abstract val platformVersion: Int data class CoreFlow(override val platformVersion: Int) : SubFlowVersion() data class CorDappFlow(override val platformVersion: Int, val corDappName: String, val corDappHash: SecureHash) : SubFlowVersion() -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt index 127f23f1e2..0aa58241eb 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DeliverSessionMessageTransition.kt @@ -4,7 +4,6 @@ import net.corda.core.flows.FlowException import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField -import net.corda.core.internal.declaredField import net.corda.node.services.statemachine.Action import net.corda.node.services.statemachine.ConfirmSessionMessage import net.corda.node.services.statemachine.DataSessionMessage @@ -48,7 +47,7 @@ class DeliverSessionMessageTransition( pendingDeduplicationHandlers = currentState.pendingDeduplicationHandlers + event.deduplicationHandler ) // Check whether we have a session corresponding to the message. - val existingSession = startingState.checkpoint.sessions[event.sessionMessage.recipientSessionId] + val existingSession = startingState.checkpoint.checkpointState.sessions[event.sessionMessage.recipientSessionId] if (existingSession == null) { freshErrorTransition(CannotFindSessionException(event.sessionMessage.recipientSessionId)) } else { @@ -81,8 +80,8 @@ class DeliverSessionMessageTransition( errors = emptyList(), deduplicationSeed = sessionState.deduplicationSeed ) - val newCheckpoint = currentState.checkpoint.copy( - sessions = currentState.checkpoint.sessions + (event.sessionMessage.recipientSessionId to initiatedSession) + val newCheckpoint = currentState.checkpoint.addSession( + event.sessionMessage.recipientSessionId to initiatedSession ) // Send messages that were buffered pending confirmation of session. val sendActions = sessionState.bufferedMessages.map { (deduplicationId, bufferedMessage) -> @@ -104,9 +103,10 @@ class DeliverSessionMessageTransition( val newSessionState = sessionState.copy( receivedMessages = sessionState.receivedMessages + message ) + currentState = currentState.copy( - checkpoint = currentState.checkpoint.copy( - sessions = startingState.checkpoint.sessions + (event.sessionMessage.recipientSessionId to newSessionState) + checkpoint = currentState.checkpoint.addSession( + event.sessionMessage.recipientSessionId to newSessionState ) ) } @@ -138,9 +138,7 @@ class DeliverSessionMessageTransition( val flowError = FlowError(payload.errorId, exception) val newSessionState = sessionState.copy(errors = sessionState.errors + flowError) currentState = currentState.copy( - checkpoint = checkpoint.copy( - sessions = checkpoint.sessions + (sessionId to newSessionState) - ) + checkpoint = checkpoint.addSession(sessionId to newSessionState) ) } else -> freshErrorTransition(UnexpectedEventInState()) @@ -159,9 +157,7 @@ class DeliverSessionMessageTransition( val sessionId = event.sessionMessage.recipientSessionId val flowError = FlowError(payload.errorId, exception) currentState = currentState.copy( - checkpoint = checkpoint.copy( - sessions = checkpoint.sessions + (sessionId to sessionState.copy(rejectionError = flowError)) - ) + checkpoint = checkpoint.addSession(sessionId to sessionState.copy(rejectionError = flowError)) ) } } @@ -171,7 +167,7 @@ class DeliverSessionMessageTransition( private fun TransitionBuilder.endMessageTransition() { val sessionId = event.sessionMessage.recipientSessionId - val sessions = currentState.checkpoint.sessions + val sessions = currentState.checkpoint.checkpointState.sessions val sessionState = sessions[sessionId] if (sessionState == null) { return freshErrorTransition(CannotFindSessionException(sessionId)) @@ -180,9 +176,8 @@ class DeliverSessionMessageTransition( is SessionState.Initiated -> { val newSessionState = sessionState.copy(initiatedState = InitiatedSessionState.Ended) currentState = currentState.copy( - checkpoint = currentState.checkpoint.copy( - sessions = sessions + (sessionId to newSessionState) - ) + checkpoint = currentState.checkpoint.addSession(sessionId to newSessionState) + ) } else -> { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt index 89b1b00a29..9692a374ba 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt @@ -40,10 +40,13 @@ class ErrorFlowTransition( return builder { // If we're errored and propagating do the actual propagation and update the index. if (remainingErrorsToPropagate.isNotEmpty() && errorState.propagating) { - val (initiatedSessions, newSessions) = bufferErrorMessagesInInitiatingSessions(startingState.checkpoint.sessions, errorMessages) + val (initiatedSessions, newSessions) = bufferErrorMessagesInInitiatingSessions( + startingState.checkpoint.checkpointState.sessions, + errorMessages + ) val newCheckpoint = startingState.checkpoint.copy( errorState = errorState.copy(propagatedIndex = allErrors.size), - sessions = newSessions + checkpointState = startingState.checkpoint.checkpointState.copy(sessions = newSessions) ) currentState = currentState.copy(checkpoint = newCheckpoint) actions.add(Action.PropagateErrors(errorMessages, initiatedSessions, startingState.senderUUID)) @@ -65,7 +68,7 @@ class ErrorFlowTransition( Action.ReleaseSoftLocks(context.id.uuid), Action.CommitTransaction, Action.AcknowledgeMessages(currentState.pendingDeduplicationHandlers), - Action.RemoveSessionBindings(currentState.checkpoint.sessions.keys) + Action.RemoveSessionBindings(currentState.checkpoint.checkpointState.sessions.keys) )) currentState = currentState.copy( diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt index 3269a87a2f..2efd9f6328 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt @@ -46,7 +46,7 @@ class StartedFlowTransition( private fun waitForSessionConfirmationsTransition(): TransitionResult { return builder { - if (currentState.checkpoint.sessions.values.any { it is SessionState.Initiating }) { + if (currentState.checkpoint.checkpointState.sessions.values.any { it is SessionState.Initiating }) { FlowContinuation.ProcessEvents } else { resumeFlowLogic(Unit) @@ -76,7 +76,7 @@ class StartedFlowTransition( val checkpoint = currentState.checkpoint val resultMap = LinkedHashMap() for ((sessionId, session) in sessionIdToSession) { - val sessionState = checkpoint.sessions[sessionId] + val sessionState = checkpoint.checkpointState.sessions[sessionId] if (sessionState is SessionState.Initiated) { resultMap[session] = sessionState.peerFlowInfo } else { @@ -159,14 +159,14 @@ class StartedFlowTransition( sourceSessionIdToSessionMap: Map ): Map>? { val checkpoint = currentState.checkpoint - val pollResult = pollSessionMessages(checkpoint.sessions, sourceSessionIdToSessionMap.keys) ?: return null + val pollResult = pollSessionMessages(checkpoint.checkpointState.sessions, sourceSessionIdToSessionMap.keys) ?: return null val resultMap = LinkedHashMap>() for ((sessionId, message) in pollResult.messages) { val session = sourceSessionIdToSessionMap[sessionId]!! resultMap[session] = message } currentState = currentState.copy( - checkpoint = checkpoint.copy(sessions = pollResult.newSessionMap) + checkpoint = checkpoint.setSessions(sessions = pollResult.newSessionMap) ) return resultMap } @@ -205,10 +205,10 @@ class StartedFlowTransition( private fun TransitionBuilder.sendInitialSessionMessagesIfNeeded(sourceSessions: Set) { val checkpoint = startingState.checkpoint - val newSessions = LinkedHashMap(checkpoint.sessions) + val newSessions = LinkedHashMap(checkpoint.checkpointState.sessions) var index = 0 for (sourceSessionId in sourceSessions) { - val sessionState = checkpoint.sessions[sourceSessionId] + val sessionState = checkpoint.checkpointState.sessions[sourceSessionId] if (sessionState == null) { return freshErrorTransition(CannotFindSessionException(sourceSessionId)) } @@ -225,7 +225,7 @@ class StartedFlowTransition( actions.add(Action.SendInitial(sessionState.destination, initialMessage, SenderDeduplicationId(deduplicationId, startingState.senderUUID))) newSessions[sourceSessionId] = newSessionState } - currentState = currentState.copy(checkpoint = checkpoint.copy(sessions = newSessions)) + currentState = currentState.copy(checkpoint = checkpoint.setSessions(sessions = newSessions)) } private fun sendTransition(flowIORequest: FlowIORequest.Send): TransitionResult { @@ -244,10 +244,10 @@ class StartedFlowTransition( private fun TransitionBuilder.sendToSessionsTransition(sourceSessionIdToMessage: Map>) { val checkpoint = startingState.checkpoint - val newSessions = LinkedHashMap(checkpoint.sessions) + val newSessions = LinkedHashMap(checkpoint.checkpointState.sessions) var index = 0 for ((sourceSessionId, message) in sourceSessionIdToMessage) { - val existingSessionState = checkpoint.sessions[sourceSessionId] + val existingSessionState = checkpoint.checkpointState.sessions[sourceSessionId] if (existingSessionState == null) { return freshErrorTransition(CannotFindSessionException(sourceSessionId)) } else { @@ -286,7 +286,7 @@ class StartedFlowTransition( } } - currentState = currentState.copy(checkpoint = checkpoint.copy(sessions = newSessions)) + currentState = currentState.copy(checkpoint = checkpoint.setSessions(newSessions)) } private fun sessionToSessionId(session: FlowSession): SessionId { @@ -295,7 +295,7 @@ class StartedFlowTransition( private fun collectErroredSessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { return sessionIds.flatMap { sessionId -> - val sessionState = checkpoint.sessions[sessionId]!! + val sessionState = checkpoint.checkpointState.sessions[sessionId]!! when (sessionState) { is SessionState.Uninitiated -> emptyList() is SessionState.Initiating -> { @@ -311,14 +311,14 @@ class StartedFlowTransition( } private fun collectErroredInitiatingSessionErrors(checkpoint: Checkpoint): List { - return checkpoint.sessions.values.mapNotNull { sessionState -> + return checkpoint.checkpointState.sessions.values.mapNotNull { sessionState -> (sessionState as? SessionState.Initiating)?.rejectionError?.exception } } private fun collectEndedSessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { return sessionIds.mapNotNull { sessionId -> - val sessionState = checkpoint.sessions[sessionId]!! + val sessionState = checkpoint.checkpointState.sessions[sessionId]!! when (sessionState) { is SessionState.Initiated -> { if (sessionState.initiatedState === InitiatedSessionState.Ended) { @@ -338,7 +338,7 @@ class StartedFlowTransition( private fun collectEndedEmptySessionErrors(sessionIds: Collection, checkpoint: Checkpoint): List { return sessionIds.mapNotNull { sessionId -> - val sessionState = checkpoint.sessions[sessionId]!! + val sessionState = checkpoint.checkpointState.sessions[sessionId]!! when (sessionState) { is SessionState.Initiated -> { if (sessionState.initiatedState === InitiatedSessionState.Ended && @@ -372,7 +372,7 @@ class StartedFlowTransition( collectErroredSessionErrors(sessionIds, checkpoint) + collectEndedSessionErrors(sessionIds, checkpoint) } is FlowIORequest.WaitForLedgerCommit -> { - collectErroredSessionErrors(checkpoint.sessions.keys, checkpoint) + collectErroredSessionErrors(checkpoint.checkpointState.sessions.keys, checkpoint) } is FlowIORequest.GetFlowInfo -> { collectErroredSessionErrors(flowIORequest.sessions.map(this::sessionToSessionId), checkpoint) @@ -413,7 +413,7 @@ class StartedFlowTransition( return builder { // The `numberOfSuspends` is added to the deduplication ID in case an async // operation is executed multiple times within the same flow. - val deduplicationId = context.id.toString() + ":" + currentState.checkpoint.numberOfSuspends.toString() + val deduplicationId = context.id.toString() + ":" + currentState.checkpoint.checkpointState.numberOfSuspends.toString() actions.add(Action.ExecuteAsyncOperation(deduplicationId, flowIORequest.operation)) FlowContinuation.ProcessEvents } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 23b4126a95..eaad3d99b5 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -81,7 +81,7 @@ class TopLevelTransition( return TransitionResult( newState = lastState, actions = listOf( - Action.RemoveSessionBindings(startingState.checkpoint.sessions.keys), + Action.RemoveSessionBindings(startingState.checkpoint.checkpointState.sessions.keys), Action.RemoveFlow(context.id, FlowRemovalReason.SoftShutdown, lastState) ), continuation = FlowContinuation.Abort @@ -111,11 +111,9 @@ class TopLevelTransition( val subFlow = SubFlow.create(event.subFlowClass, event.subFlowVersion, event.isEnabledTimedFlow) when (subFlow) { is Try.Success -> { - val containsTimedSubflow = containsTimedFlows(currentState.checkpoint.subFlowStack) + val containsTimedSubflow = containsTimedFlows(currentState.checkpoint.checkpointState.subFlowStack) currentState = currentState.copy( - checkpoint = currentState.checkpoint.copy( - subFlowStack = currentState.checkpoint.subFlowStack + subFlow.value - ) + checkpoint = currentState.checkpoint.addSubflow(subFlow.value) ) // We don't schedule a timeout if there already is a timed subflow on the stack - a timeout had // been scheduled already. @@ -134,17 +132,15 @@ class TopLevelTransition( private fun leaveSubFlowTransition(): TransitionResult { return builder { val checkpoint = currentState.checkpoint - if (checkpoint.subFlowStack.isEmpty()) { + if (checkpoint.checkpointState.subFlowStack.isEmpty()) { freshErrorTransition(UnexpectedEventInState()) } else { - val isLastSubFlowTimed = checkpoint.subFlowStack.last().isEnabledTimedFlow - val newSubFlowStack = checkpoint.subFlowStack.dropLast(1) + val isLastSubFlowTimed = checkpoint.checkpointState.subFlowStack.last().isEnabledTimedFlow + val newSubFlowStack = checkpoint.checkpointState.subFlowStack.dropLast(1) currentState = currentState.copy( - checkpoint = checkpoint.copy( - subFlowStack = newSubFlowStack - ) + checkpoint = checkpoint.setSubflows(newSubFlowStack) ) - if (isLastSubFlowTimed && !containsTimedFlows(currentState.checkpoint.subFlowStack)) { + if (isLastSubFlowTimed && !containsTimedFlows(currentState.checkpoint.checkpointState.subFlowStack)) { actions.add(Action.CancelFlowTimeout(currentState.flowLogic.runId)) } } @@ -160,7 +156,9 @@ class TopLevelTransition( return builder { val newCheckpoint = currentState.checkpoint.copy( flowState = FlowState.Started(event.ioRequest, event.fiber), - numberOfSuspends = currentState.checkpoint.numberOfSuspends + 1 + checkpointState = currentState.checkpoint.checkpointState.copy( + numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends + 1 + ) ) if (event.maySkipCheckpoint) { actions.addAll(arrayOf( @@ -198,13 +196,14 @@ class TopLevelTransition( val pendingDeduplicationHandlers = currentState.pendingDeduplicationHandlers currentState = currentState.copy( checkpoint = checkpoint.copy( - numberOfSuspends = checkpoint.numberOfSuspends + 1 - ), + checkpointState = checkpoint.checkpointState.copy( + numberOfSuspends = checkpoint.checkpointState.numberOfSuspends + 1 + )), pendingDeduplicationHandlers = emptyList(), isFlowResumed = false, isRemoved = true ) - val allSourceSessionIds = checkpoint.sessions.keys + val allSourceSessionIds = checkpoint.checkpointState.sessions.keys if (currentState.isAnyCheckpointPersisted) { actions.add(Action.RemoveCheckpoint(context.id)) } @@ -230,7 +229,7 @@ class TopLevelTransition( } private fun TransitionBuilder.sendEndMessages() { - val sendEndMessageActions = currentState.checkpoint.sessions.values.mapIndexed { index, state -> + val sendEndMessageActions = currentState.checkpoint.checkpointState.sessions.values.mapIndexed { index, state -> if (state is SessionState.Initiated && state.initiatedState is InitiatedSessionState.Live) { val message = ExistingSessionMessage(state.initiatedState.peerSinkSessionId, EndSessionMessage) val deduplicationId = DeduplicationId.createForNormal(currentState.checkpoint, index, state) @@ -252,15 +251,15 @@ class TopLevelTransition( } val sourceSessionId = SessionId.createRandom(context.secureRandom) val sessionImpl = FlowSessionImpl(event.destination, event.wellKnownParty, sourceSessionId) - val newSessions = checkpoint.sessions + (sourceSessionId to SessionState.Uninitiated(event.destination, initiatingSubFlow, sourceSessionId, context.secureRandom.nextLong())) - currentState = currentState.copy(checkpoint = checkpoint.copy(sessions = newSessions)) + val newSessions = checkpoint.checkpointState.sessions + (sourceSessionId to SessionState.Uninitiated(event.destination, initiatingSubFlow, sourceSessionId, context.secureRandom.nextLong())) + currentState = currentState.copy(checkpoint = checkpoint.setSessions(newSessions)) actions.add(Action.AddSessionBinding(context.id, sourceSessionId)) FlowContinuation.Resume(sessionImpl) } } private fun getClosestAncestorInitiatingSubFlow(checkpoint: Checkpoint): SubFlow.Initiating? { - for (subFlow in checkpoint.subFlowStack.asReversed()) { + for (subFlow in checkpoint.checkpointState.subFlowStack.asReversed()) { if (subFlow is SubFlow.Initiating) { return subFlow } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt index ac3c232377..c85830fb03 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/UnstartedFlowTransition.kt @@ -61,9 +61,7 @@ class UnstartedFlowTransition( val confirmationMessage = ConfirmSessionMessage(flowStart.initiatedSessionId, flowStart.initiatedFlowInfo) val sessionMessage = ExistingSessionMessage(initiatingMessage.initiatorSessionId, confirmationMessage) currentState = currentState.copy( - checkpoint = currentState.checkpoint.copy( - sessions = mapOf(flowStart.initiatedSessionId to initiatedState) - ) + checkpoint = currentState.checkpoint.setSessions(mapOf(flowStart.initiatedSessionId to initiatedState)) ) actions.add( Action.SendExisting( From 9d4d128f4e6f04dd0ba5f59c98ce81c6e0ecbbbe Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Wed, 26 Feb 2020 10:48:52 +0000 Subject: [PATCH 19/49] CORDA-3597 add missed data to checkpoint class (#5995) * Add extra fields to StateMachineState * Move structures into checkpoint as this is a more natural place. --- .../services/persistence/DBCheckpointStorage.kt | 10 +--------- .../services/statemachine/StateMachineState.kt | 14 +++++++++++++- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index c9d883a5b0..13b5b2ace7 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -7,6 +7,7 @@ import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.debug import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.Checkpoint +import net.corda.node.services.statemachine.Checkpoint.FlowStatus import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.nodeapi.internal.persistence.currentDBSession import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY @@ -33,15 +34,6 @@ import javax.persistence.OneToOne class DBCheckpointStorage : CheckpointStorage { val log: Logger = LoggerFactory.getLogger(this::class.java) - enum class FlowStatus { - RUNNABLE, - FAILED, - COMPLETED, - HOSPITALIZED, - KILLED, - PAUSED - } - enum class StartReason { RPC, FLOW, SERVICE, SCHEDULED, INITIATED } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index dbc37090fa..f4c9e790b6 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -54,8 +54,20 @@ data class Checkpoint( val checkpointState: CheckpointState, val flowState: FlowState, val errorState: ErrorState, - val result: Any? = null + val result: Any? = null, + val status: FlowStatus = FlowStatus.RUNNABLE, + val progressStep: String? = null, + val flowIoRequest: FlowIORequest<*>? = null, + val compatible: Boolean = true ) { + enum class FlowStatus { + RUNNABLE, + FAILED, + COMPLETED, + HOSPITALIZED, + KILLED, + PAUSED + } val timestamp: Instant = Instant.now() // This will get updated every time a Checkpoint object is created/ created by copy. From ab000e05338b35b90923842b9ec18fe361b6718f Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Mon, 2 Mar 2020 10:04:48 +0000 Subject: [PATCH 20/49] CORDA-3597 Replace old Checkpoint table with new one. (#5992) * Replace old Checkpoint table with new one. Adds some of the new fields into the table where needed (I have guessed this stuff but we can update it as we go along). * Fix database constraints + name table correctly opps. * Fixed typos in Liquidbase script Also corrected constraints and added missed fields in hibernate checkpoint class and liquibase scripts. * Update CheckpointStorage to pass in serialization context. This is cleaner than passing both the checkpoint and the serialized checkpoint into the methods. Also fixed CordaPersistanceServiceTests which I accidentally broke. * Fix detekt problem * Revert "Update CheckpointStorage to pass in serialization context." This reverts commit b71e78f20274ab0f5b3cf3fda1451ae2bd7a6797. * Fix test broken by reverting commit * CORDA-3597 Update metadata join, timestamp columns and serialization - Change the metadata join to the checkpoints table to use `invocation_id` instead of `flow_id`. There were issues joining between the tables because `flow_id` was not the primary key of the metadata table. Switching over to `invocation_id` has at least allowed us to bypass this issue. The information about the `invocation_id` is stored in the `Checkpoint` class which makes it simple to save at runtime. - Some of timestamp columns were nullable when they should always be populated, the nullable flags have now been removed. - Previously the whole checkpoint was being serialized and stored into the `checkpoints_blob.checkpoint` column. This meant duplicated saving as the `flow_state` was contained in this object. Only the `CheckpointState` property of `Checkpoint` is now being serialized and saved to this field. Furthermore, it now uses the default `STORAGE_CONTEXT` serialization (AMQP) instead of Kryo (which is only used for serializing the `flow_state` / flow stack). - The checkpoint database performance metrics recording has been abstracted to its own class. * CORDA-3597 Make metadata join non optional Remove the nullable declaration on the metadata field of `DBFlowCheckpoint` * CORDA-3597 Rename `node_checkpoints_blobs` to `node_checkpoint_blobs` * CORDA-3597 Update some kdocs Co-authored-by: Dan Newton --- .../persistence/MissingSchemaMigrationTest.kt | 6 +- .../CordaPersistenceServiceTests.kt | 54 ++- .../net/corda/node/internal/AbstractNode.kt | 5 +- .../corda/node/internal/CheckpointVerifier.kt | 2 +- .../node/services/api/CheckpointStorage.kt | 24 +- .../DBCheckpointPerformanceRecorder.kt | 60 +++ .../persistence/DBCheckpointStorage.kt | 433 ++++++++++++------ .../node/services/rpc/CheckpointDumperImpl.kt | 4 +- .../node/services/schema/NodeSchemaService.kt | 5 +- .../statemachine/ActionExecutorImpl.kt | 30 +- .../SingleThreadedStateMachineManager.kt | 18 +- .../statemachine/StateMachineState.kt | 70 ++- .../node-core.changelog-v17-keys.xml | 33 +- .../node-core.changelog-v17-postgres.xml | 12 +- .../migration/node-core.changelog-v17.xml | 12 +- .../persistence/DBCheckpointStorageTests.kt | 270 +++++++++-- .../services/rpc/CheckpointDumperImplTest.kt | 24 +- 17 files changed, 785 insertions(+), 277 deletions(-) create mode 100644 node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/persistence/MissingSchemaMigrationTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/persistence/MissingSchemaMigrationTest.kt index 1c975cd93a..d5d803d9c1 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/persistence/MissingSchemaMigrationTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/persistence/MissingSchemaMigrationTest.kt @@ -47,7 +47,7 @@ class MissingSchemaMigrationTest { fun `test that an error is thrown when forceThrowOnMissingMigration is set and a mapped schema is missing a migration`() { assertThatThrownBy { createSchemaMigration(setOf(GoodSchema), true) - .nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + .nodeStartup(dataSource.connection.use { DBCheckpointStorage.getCheckpointCount(it) != 0L }) }.isInstanceOf(MissingMigrationException::class.java) } @@ -55,7 +55,7 @@ class MissingSchemaMigrationTest { fun `test that an error is not thrown when forceThrowOnMissingMigration is not set and a mapped schema is missing a migration`() { assertDoesNotThrow { createSchemaMigration(setOf(GoodSchema), false) - .nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + .nodeStartup(dataSource.connection.use { DBCheckpointStorage.getCheckpointCount(it) != 0L }) } } @@ -64,7 +64,7 @@ class MissingSchemaMigrationTest { assertDoesNotThrow("This test failure indicates " + "a new table has been added to the node without the appropriate migration scripts being present") { createSchemaMigration(NodeSchemaService().internalSchemas(), false) - .nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + .nodeStartup(dataSource.connection.use { DBCheckpointStorage.getCheckpointCount(it) != 0L }) } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index 7d6aa1edf8..ff36d5e7a2 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -3,11 +3,17 @@ package net.corda.node.services.persistence import co.paralleluniverse.fibers.Suspendable import net.corda.core.flows.FlowLogic import net.corda.core.flows.StartableByRPC +import net.corda.core.internal.FlowIORequest +import net.corda.core.internal.PLATFORM_VERSION import net.corda.core.messaging.startFlow import net.corda.core.node.AppServiceHub import net.corda.core.node.services.CordaService +import net.corda.core.node.services.vault.SessionScope import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.utilities.getOrThrow +import net.corda.node.services.statemachine.Checkpoint +import net.corda.node.services.statemachine.Checkpoint.FlowStatus +import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.driver @@ -15,6 +21,8 @@ import net.corda.testing.driver.internal.incrementalPortAllocation import net.corda.testing.node.internal.enclosedCordapp import org.junit.Test import java.sql.DriverManager +import java.time.Instant +import java.util.UUID import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -50,16 +58,52 @@ class CordaPersistenceServiceTests { @CordaService class MultiThreadedDbLoader(private val services: AppServiceHub) : SingletonSerializeAsToken() { - fun createObjects(count: Int) : Int { + fun createObjects(count: Int): Int { (1..count).toList().parallelStream().forEach { + val now = Instant.now() services.database.transaction { - session.save(DBCheckpointStorage.DBCheckpoint().apply { - checkpointId = it.toString() - }) + session.save( + DBCheckpointStorage.DBFlowCheckpoint( + id = it.toString(), + blob = DBCheckpointStorage.DBFlowCheckpointBlob( + checkpoint = ByteArray(8192), + flowStack = ByteArray(8192), + hmac = ByteArray(16), + persistedInstant = now + ), + result = DBCheckpointStorage.DBFlowResult(value = ByteArray(16), persistedInstant = now), + exceptionDetails = null, + status = FlowStatus.RUNNABLE, + compatible = false, + progressStep = "", + ioRequestType = FlowIORequest.ForceCheckpoint.javaClass, + checkpointInstant = Instant.now(), + flowMetadata = createMetadataRecord(UUID.randomUUID(), now) + ) + ) } } return count } + + private fun SessionScope.createMetadataRecord(invocationId: UUID, timestamp: Instant): DBCheckpointStorage.DBFlowMetadata { + val metadata = DBCheckpointStorage.DBFlowMetadata( + invocationId = invocationId.toString(), + flowId = null, + flowName = "random.flow", + userSuppliedIdentifier = null, + startType = DBCheckpointStorage.StartReason.RPC, + launchingCordapp = "this cordapp", + platformVersion = PLATFORM_VERSION, + rpcUsername = "Batman", + invocationInstant = Instant.now(), + receivedInstant = Instant.now(), + startInstant = timestamp, + finishInstant = null + ) + session.save(metadata) + return metadata + } } -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 38675eb979..6164c92ccd 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -109,6 +109,7 @@ import net.corda.node.services.network.PersistentNetworkMapCache import net.corda.node.services.persistence.AbstractPartyDescriptor import net.corda.node.services.persistence.AbstractPartyToX500NameAsStringConverter import net.corda.node.services.persistence.AttachmentStorageInternal +import net.corda.node.services.persistence.DBCheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.DBTransactionMappingStorage import net.corda.node.services.persistence.DBTransactionStorage @@ -251,7 +252,6 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } val networkMapCache = PersistentNetworkMapCache(cacheFactory, database, identityService).tokenize() - val checkpointStorage = DBCheckpointStorage() @Suppress("LeakingThis") val transactionStorage = makeTransactionStorage(configuration.transactionCacheSizeBytes).tokenize() val networkMapClient: NetworkMapClient? = configuration.networkServices?.let { NetworkMapClient(it.networkMapURL, versionInfo) } @@ -318,6 +318,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, @Suppress("LeakingThis") protected val network: MessagingService = makeMessagingService().tokenize() val services = ServiceHubInternalImpl().tokenize() + val checkpointStorage = DBCheckpointStorage(DBCheckpointPerformanceRecorder(services.monitoringService.metrics)) @Suppress("LeakingThis") val smm = makeStateMachineManager() val flowStarter = FlowStarterImpl(smm, flowLogicRefFactory) @@ -1276,7 +1277,7 @@ fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfi try { val dataSource = DataSourceFactory.createDataSource(hikariProperties, metricRegistry = metricRegistry) val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig, cordappLoader, currentDir, ourName) - schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage.getCheckpointCount(it) != 0L }) start(dataSource) } catch (ex: Exception) { when { diff --git a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt index cb7fe99b6f..340226492e 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt @@ -39,7 +39,7 @@ object CheckpointVerifier { checkpointStorage.getAllCheckpoints().use { it.forEach { (_, serializedCheckpoint) -> val checkpoint = try { - serializedCheckpoint.checkpointDeserialize(context = checkpointSerializationContext) + serializedCheckpoint.deserialize(checkpointSerializationContext) } catch (e: ClassNotFoundException) { val message = e.message if (message != null) { diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index b463372909..8268322233 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -3,7 +3,7 @@ package net.corda.node.services.api import net.corda.core.flows.StateMachineRunId import net.corda.core.serialization.SerializedBytes import net.corda.node.services.statemachine.Checkpoint -import java.sql.Connection +import net.corda.node.services.statemachine.FlowState import java.util.stream.Stream /** @@ -13,12 +13,12 @@ interface CheckpointStorage { /** * Add a checkpoint for a new id to the store. Will throw if there is already a checkpoint for this id */ - fun addCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) + fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) /** * Update an existing checkpoint. Will throw if there is not checkpoint for this id. */ - fun updateCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) + fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) /** * Remove existing checkpoint from the store. @@ -28,21 +28,17 @@ interface CheckpointStorage { /** * Load an existing checkpoint from the store. - * @return the checkpoint, still in serialized form, or null if not found. + * + * The checkpoint returned from this function will be a _clean_ checkpoint. No error information is loaded into the checkpoint + * even if the previous status of the checkpoint was [Checkpoint.FlowStatus.FAILED] or [Checkpoint.FlowStatus.HOSPITALIZED]. + * + * @return The checkpoint, in a partially serialized form, or null if not found. */ - fun getCheckpoint(id: StateMachineRunId): SerializedBytes? + fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? /** * Stream all checkpoints from the store. If this is backed by a database the stream will be valid until the * underlying database connection is closed, so any processing should happen before it is closed. */ - fun getAllCheckpoints(): Stream>> - - /** - * This needs to run before Hibernate is initialised. - * - * @param connection The SQL Connection. - * @return the number of checkpoints stored in the database. - */ - fun getCheckpointCount(connection: Connection): Long + fun getAllCheckpoints(): Stream> } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt new file mode 100644 index 0000000000..7bc30ec804 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt @@ -0,0 +1,60 @@ +package net.corda.node.services.persistence + +import com.codahale.metrics.Gauge +import com.codahale.metrics.Histogram +import com.codahale.metrics.MetricRegistry +import com.codahale.metrics.Reservoir +import com.codahale.metrics.SlidingTimeWindowArrayReservoir +import com.codahale.metrics.SlidingTimeWindowReservoir +import net.corda.core.serialization.SerializedBytes +import net.corda.node.services.statemachine.CheckpointState +import net.corda.node.services.statemachine.FlowState +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong + +interface CheckpointPerformanceRecorder { + + /** + * Record performance metrics regarding the serialized size of [CheckpointState] and [FlowState] + */ + fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes) +} + +class DBCheckpointPerformanceRecorder(metrics: MetricRegistry) : CheckpointPerformanceRecorder { + + private val checkpointingMeter = metrics.meter("Flows.Checkpointing Rate") + private val checkpointSizesThisSecond = SlidingTimeWindowReservoir(1, TimeUnit.SECONDS) + private val lastBandwidthUpdate = AtomicLong(0) + private val checkpointBandwidthHist = metrics.register( + "Flows.CheckpointVolumeBytesPerSecondHist", Histogram( + SlidingTimeWindowArrayReservoir(1, TimeUnit.DAYS) + ) + ) + private val checkpointBandwidth = metrics.register( + "Flows.CheckpointVolumeBytesPerSecondCurrent", + LatchedGauge(checkpointSizesThisSecond) + ) + + /** + * This [Gauge] just reports the sum of the bytes checkpointed during the last second. + */ + private class LatchedGauge(private val reservoir: Reservoir) : Gauge { + override fun getValue(): Long { + return reservoir.snapshot.values.sum() + } + } + + override fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes) { + val totalSize = serializedCheckpointState.size.toLong() + serializedFlowState.size.toLong() + checkpointingMeter.mark() + checkpointSizesThisSecond.update(totalSize) + var lastUpdateTime = lastBandwidthUpdate.get() + while (System.nanoTime() - lastUpdateTime > TimeUnit.SECONDS.toNanos(1)) { + if (lastBandwidthUpdate.compareAndSet(lastUpdateTime, System.nanoTime())) { + val checkpointVolume = checkpointSizesThisSecond.snapshot.values.sum() + checkpointBandwidthHist.update(checkpointVolume) + } + lastUpdateTime = lastBandwidthUpdate.get() + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 13b5b2ace7..45a6739b2a 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -2,245 +2,392 @@ package net.corda.node.services.persistence import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.FlowIORequest +import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes -import net.corda.core.utilities.ProgressTracker -import net.corda.core.utilities.debug +import net.corda.core.serialization.serialize +import net.corda.core.utilities.contextLogger import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.Checkpoint.FlowStatus +import net.corda.node.services.statemachine.CheckpointState +import net.corda.node.services.statemachine.ErrorState +import net.corda.node.services.statemachine.FlowState import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.nodeapi.internal.persistence.currentDBSession import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY +import org.hibernate.annotations.Type import org.slf4j.Logger import org.slf4j.LoggerFactory -import java.util.* -import java.util.stream.Stream -import javax.persistence.Column -import javax.persistence.Entity -import javax.persistence.Id -import org.hibernate.annotations.Type -import java.lang.Exception -import java.math.BigInteger import java.sql.Connection import java.sql.SQLException import java.time.Instant +import java.util.UUID +import java.util.stream.Stream +import javax.persistence.CascadeType +import javax.persistence.Column +import javax.persistence.Entity import javax.persistence.FetchType +import javax.persistence.GeneratedValue +import javax.persistence.GenerationType +import javax.persistence.Id import javax.persistence.JoinColumn import javax.persistence.OneToOne /** * Simple checkpoint key value storage in DB. */ -class DBCheckpointStorage : CheckpointStorage { - val log: Logger = LoggerFactory.getLogger(this::class.java) +class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointPerformanceRecorder) : CheckpointStorage { + + companion object { + val log = contextLogger() + + private const val HMAC_SIZE_BYTES = 16 + + /** + * This needs to run before Hibernate is initialised. + * + * No need to set up [DBCheckpointStorage] fully for this function + * + * @param connection The SQL Connection. + * @return the number of checkpoints stored in the database. + */ + fun getCheckpointCount(connection: Connection): Long { + // No need to set up [DBCheckpointStorage] fully for this function + return try { + connection.prepareStatement("select count(*) from node_checkpoints").use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + } + } catch (e: SQLException) { + // Happens when the table was not created yet. + 0L + } + } + } enum class StartReason { RPC, FLOW, SERVICE, SCHEDULED, INITIATED } @Entity - @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints_new") + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") class DBFlowCheckpoint( - @Id - @Column(name = "flow_id", length = 64, nullable = false) - var id: String, + @Id + @Column(name = "flow_id", length = 64, nullable = false) + var id: String, - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") - var blob: DBFlowCheckpointBlob, + @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) + @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") + var blob: DBFlowCheckpointBlob, - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "result_id", referencedColumnName = "id") - var result: DBFlowResult?, + @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) + @JoinColumn(name = "result_id", referencedColumnName = "id") + var result: DBFlowResult?, - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "error_id", referencedColumnName = "id") - var exceptionDetails: DBFlowException?, + @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) + @JoinColumn(name = "error_id", referencedColumnName = "id") + var exceptionDetails: DBFlowException?, - @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") - var flowMetadata: DBFlowMetadata, + @OneToOne(fetch = FetchType.LAZY) + @JoinColumn(name = "invocation_id", referencedColumnName = "invocation_id") + var flowMetadata: DBFlowMetadata, - @Column(name = "status") - var status: FlowStatus, + @Column(name = "status", nullable = false) + var status: FlowStatus, - @Column(name = "compatible") - var compatible: Boolean, + @Column(name = "compatible", nullable = false) + var compatible: Boolean, - @Column(name = "progress_step") - var progressStep: String, + @Column(name = "progress_step") + var progressStep: String?, - @Column(name = "flow_io_request") - var ioRequestType: Class>, + @Column(name = "flow_io_request") + var ioRequestType: Class>?, - @Column(name = "timestamp") - var checkpointInstant: Instant + @Column(name = "timestamp", nullable = false) + var checkpointInstant: Instant ) @Entity - @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints_blobs") + @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoint_blobs") class DBFlowCheckpointBlob( - @Id - @Column(name = "id", nullable = false) - var id: BigInteger? = null, + @Id + @GeneratedValue(strategy = GenerationType.SEQUENCE) + @Column(name = "id", nullable = false) + private var id: Long = 0, - @Type(type = "corda-blob") - @Column(name = "checkpoint_value", nullable = false) - var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + @Type(type = "corda-blob") + @Column(name = "checkpoint_value", nullable = false) + var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, - @Type(type = "corda-blob") - @Column(name = "flow_state", nullable = false) - var flowStack: ByteArray = EMPTY_BYTE_ARRAY, + // A future task will make this nullable + @Type(type = "corda-blob") + @Column(name = "flow_state", nullable = false) + var flowStack: ByteArray = EMPTY_BYTE_ARRAY, - @Column(name = "timestamp") - var persistedInstant: Instant? = null + @Column(name = "hmac") + var hmac: ByteArray, + + @Column(name = "timestamp") + var persistedInstant: Instant ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_results") class DBFlowResult( - @Id - @Column(name = "id", nullable = false) - var id: BigInteger? = null, + @Id + @Column(name = "id", nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private var id: Long = 0, - @Type(type = "corda-blob") - @Column(name = "result_value", nullable = false) - var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + @Type(type = "corda-blob") + @Column(name = "result_value", nullable = false) + var value: ByteArray = EMPTY_BYTE_ARRAY, - @Column(name = "timestamp") - val persistedInstant: Instant? = null + @Column(name = "timestamp") + val persistedInstant: Instant ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_exceptions") class DBFlowException( - @Id - @Column(name = "id", nullable = false) - var id: BigInteger? = null, + @Id + @Column(name = "id", nullable = false) + @GeneratedValue(strategy = GenerationType.SEQUENCE) + private var id: Long = 0, - @Column(name = "type", nullable = false) - var type: Class, + @Column(name = "type", nullable = false) + var type: Class, - @Type(type = "corda-blob") - @Column(name = "exception_value", nullable = false) - var value: ByteArray = EMPTY_BYTE_ARRAY, + @Type(type = "corda-blob") + @Column(name = "exception_value", nullable = false) + var value: ByteArray = EMPTY_BYTE_ARRAY, - @Column(name = "exception_message") - var message: String? = null, + @Column(name = "exception_message") + var message: String? = null, - @Column(name = "timestamp") - val persistedInstant: Instant? = null + @Column(name = "timestamp") + val persistedInstant: Instant ) @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_metadata") class DBFlowMetadata( - @Id - @Column(name = "flow_id", length = 64, nullable = false) - var flowId: String, + @Id + @Column(name = "invocation_id", nullable = false) + var invocationId: String, - @Column(name = "flow_name", nullable = false) - var flowName: String, + @Column(name = "flow_id", nullable = true) + var flowId: String?, - @Column(name = "flow_identifier", nullable = true) - var userSuppliedIdentifier: String?, + @Column(name = "flow_name", nullable = false) + var flowName: String, - @Column(name = "started_type", nullable = false) - var startType: StartReason, + @Column(name = "flow_identifier", nullable = true) + var userSuppliedIdentifier: String?, - @Column(name = "flow_parameters", nullable = false) - var initialParameters: ByteArray = EMPTY_BYTE_ARRAY, + @Column(name = "started_type", nullable = false) + var startType: StartReason, - @Column(name = "cordapp_name", nullable = false) - var launchingCordapp: String, + @Column(name = "flow_parameters", nullable = false) + var initialParameters: ByteArray = EMPTY_BYTE_ARRAY, - @Column(name = "platform_version", nullable = false) - var platformVersion: Int, + @Column(name = "cordapp_name", nullable = false) + var launchingCordapp: String, - @Column(name = "rpc_user", nullable = false) - var rpcUsername: String, + @Column(name = "platform_version", nullable = false) + var platformVersion: Int, - @Column(name = "invocation_time", nullable = false) - var invocationInstant: Instant, + @Column(name = "rpc_user", nullable = false) + var rpcUsername: String, - @Column(name = "received_time", nullable = false) - var receivedInstant: Instant, + @Column(name = "invocation_time", nullable = false) + var invocationInstant: Instant, - @Column(name = "start_time", nullable = true) - var startInstant: Instant?, + @Column(name = "received_time", nullable = false) + var receivedInstant: Instant, - @Column(name = "finish_time", nullable = true) - var finishInstant: Instant? + @Column(name = "start_time", nullable = true) + var startInstant: Instant?, + + @Column(name = "finish_time", nullable = true) + var finishInstant: Instant? ) - @Entity - @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") - class DBCheckpoint( - @Id - @Suppress("MagicNumber") // database column width - @Column(name = "checkpoint_id", length = 64, nullable = false) - var checkpointId: String = "", - - @Type(type = "corda-blob") - @Column(name = "checkpoint_value", nullable = false) - var checkpoint: ByteArray = EMPTY_BYTE_ARRAY - ) { - override fun toString() = "DBCheckpoint(checkpointId = ${checkpointId}, checkpointSize = ${checkpoint.size})" + override fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) { + currentDBSession().save(createDBCheckpoint(id, checkpoint, serializedFlowState)) } - override fun addCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) { - currentDBSession().save(DBCheckpoint().apply { - checkpointId = id.uuid.toString() - this.checkpoint = checkpoint.bytes - log.debug { "Checkpoint $checkpointId, size=${this.checkpoint.size}" } - }) - } - - override fun updateCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) { - currentDBSession().update(DBCheckpoint().apply { - checkpointId = id.uuid.toString() - this.checkpoint = checkpoint.bytes - log.debug { "Checkpoint $checkpointId, size=${this.checkpoint.size}" } - }) + override fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) { + currentDBSession().update(updateDBCheckpoint(id, checkpoint, serializedFlowState)) } override fun removeCheckpoint(id: StateMachineRunId): Boolean { val session = currentDBSession() val criteriaBuilder = session.criteriaBuilder - val delete = criteriaBuilder.createCriteriaDelete(DBCheckpoint::class.java) - val root = delete.from(DBCheckpoint::class.java) - delete.where(criteriaBuilder.equal(root.get(DBCheckpoint::checkpointId.name), id.uuid.toString())) + val delete = criteriaBuilder.createCriteriaDelete(DBFlowCheckpoint::class.java) + val root = delete.from(DBFlowCheckpoint::class.java) + delete.where(criteriaBuilder.equal(root.get(DBFlowCheckpoint::id.name), id.uuid.toString())) return session.createQuery(delete).executeUpdate() > 0 } - override fun getCheckpoint(id: StateMachineRunId): SerializedBytes? { - val bytes = currentDBSession().get(DBCheckpoint::class.java, id.uuid.toString())?.checkpoint ?: return null - return SerializedBytes(bytes) + override fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? { + return currentDBSession().get(DBFlowCheckpoint::class.java, id.uuid.toString())?.toSerializedCheckpoint() } - override fun getAllCheckpoints(): Stream>> { + override fun getAllCheckpoints(): Stream> { val session = currentDBSession() - val criteriaQuery = session.criteriaBuilder.createQuery(DBCheckpoint::class.java) - val root = criteriaQuery.from(DBCheckpoint::class.java) + val criteriaQuery = session.criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) + val root = criteriaQuery.from(DBFlowCheckpoint::class.java) criteriaQuery.select(root) return session.createQuery(criteriaQuery).stream().map { - StateMachineRunId(UUID.fromString(it.checkpointId)) to SerializedBytes(it.checkpoint) + StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() } } - override fun getCheckpointCount(connection: Connection): Long { - return try { - connection.prepareStatement("select count(*) from node_checkpoints").use { ps -> - ps.executeQuery().use { rs -> - rs.next() - rs.getLong(1) - } - } - } catch (e: SQLException) { - // Happens when the table was not created yet. - 0L + private fun createDBCheckpoint( + id: StateMachineRunId, + checkpoint: Checkpoint, + serializedFlowState: SerializedBytes + ): DBFlowCheckpoint { + val flowId = id.uuid.toString() + val now = Instant.now() + val invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value + + val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() + checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + + val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) + // Need to update the metadata record to join it to the main checkpoint record + + // This code needs to be added back in once the metadata record is properly created (remove the code below it) + // val metadata = requireNotNull(currentDBSession().find( + // DBFlowMetadata::class.java, + // invocationId + // )) { "The flow metadata record for flow [$flowId] with invocation id [$invocationId] does not exist"} + val metadata = (currentDBSession().find( + DBFlowMetadata::class.java, + invocationId + )) ?: createTemporaryMetadata(checkpoint) + metadata.flowId = flowId + currentDBSession().update(metadata) + // Most fields are null as they cannot have been set when creating the initial checkpoint + return DBFlowCheckpoint( + id = flowId, + blob = blob, + result = null, + exceptionDetails = null, + flowMetadata = metadata, + status = checkpoint.status, + compatible = checkpoint.compatible, + progressStep = null, + ioRequestType = null, + checkpointInstant = Instant.now() + ) + } + + // Remove this when saving of metadata is properly handled + private fun createTemporaryMetadata(checkpoint: Checkpoint): DBFlowMetadata { + return DBFlowMetadata( + invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value, + flowId = null, + flowName = "random.flow", + userSuppliedIdentifier = null, + startType = DBCheckpointStorage.StartReason.RPC, + launchingCordapp = "this cordapp", + platformVersion = PLATFORM_VERSION, + rpcUsername = "Batman", + invocationInstant = checkpoint.checkpointState.invocationContext.trace.invocationId.timestamp, + receivedInstant = Instant.now(), + startInstant = null, + finishInstant = null + ).apply { + currentDBSession().save(this) } } + + private fun updateDBCheckpoint( + id: StateMachineRunId, + checkpoint: Checkpoint, + serializedFlowState: SerializedBytes + ): DBFlowCheckpoint { + val flowId = id.uuid.toString() + val now = Instant.now() + + val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() + checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + + val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) + val result = checkpoint.result?.let { createDBFlowResult(it, now) } + val exceptionDetails = (checkpoint.errorState as? ErrorState.Errored)?.let { createDBFlowException(it, now) } + // Load the previous entity from the hibernate cache so the meta data join does not get updated + val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) + return entity.apply { + this.blob = blob + this.result = result + this.exceptionDetails = exceptionDetails + // Do not update the meta data relationship on updates + this.flowMetadata = entity.flowMetadata + this.status = checkpoint.status + this.compatible = checkpoint.compatible + this.progressStep = checkpoint.progressStep + this.ioRequestType = checkpoint.flowIoRequest + this.checkpointInstant = now + } + } + + private fun createDBCheckpointBlob( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes, + now: Instant + ): DBFlowCheckpointBlob { + return DBFlowCheckpointBlob( + checkpoint = serializedCheckpointState.bytes, + flowStack = serializedFlowState.bytes, + hmac = ByteArray(HMAC_SIZE_BYTES), + persistedInstant = now + ) + } + + private fun createDBFlowResult(result: Any, now: Instant): DBFlowResult { + return DBFlowResult( + value = result.storageSerialize().bytes, + persistedInstant = now + ) + } + + private fun createDBFlowException(errorState: ErrorState.Errored, now: Instant): DBFlowException { + return errorState.errors.last().exception.let { + DBFlowException( + type = it::class.java, + message = it.message, + value = it.storageSerialize().bytes, + persistedInstant = now + ) + } + } + + private fun DBFlowCheckpoint.toSerializedCheckpoint(): Checkpoint.Serialized { + return Checkpoint.Serialized( + serializedCheckpointState = SerializedBytes(blob.checkpoint), + serializedFlowState = SerializedBytes(blob.flowStack), + // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint + errorState = ErrorState.Clean, + // A checkpoint with a result should not normally be loaded (it should be [null] most of the time) + result = result?.let { SerializedBytes(it.value) }, + status = status, + progressStep = progressStep, + flowIoRequest = ioRequestType, + compatible = compatible + ) + } + + private fun T.storageSerialize(): SerializedBytes { + return serialize(context = SerializationDefaults.STORAGE_CONTEXT) + } } diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index d27a480a5a..c3979612ab 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -149,8 +149,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri instrumentCheckpointAgent(runId) val (bytes, fileName) = try { - val checkpoint = - serialisedCheckpoint.checkpointDeserialize(context = checkpointSerializationContext) + val checkpoint = serialisedCheckpoint.deserialize(checkpointSerializationContext) val json = checkpoint.toJson(runId.uuid, now) val jsonBytes = writer.writeValueAsBytes(json) jsonBytes to "${json.topLevelFlowClass.simpleName}-${runId.uuid}.json" @@ -229,6 +228,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri origin = checkpointState.invocationContext.origin.toOrigin(), ourIdentity = checkpointState.ourIdentity, activeSessions = checkpointState.sessions.mapNotNull { it.value.toActiveSession(it.key) }, + // This can only ever return as [ErrorState.Clean] which causes it to become [null] errored = errorState as? ErrorState.Errored ) } diff --git a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt index 73d7cbb3d9..d38c6371ef 100644 --- a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt +++ b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt @@ -33,10 +33,7 @@ class NodeSchemaService(private val extraSchemas: Set = emptySet() object NodeCore object NodeCoreV1 : MappedSchema(schemaFamily = NodeCore.javaClass, version = 1, - mappedTypes = listOf(DBCheckpointStorage.DBCheckpoint::class.java, - - //new checkpoints - keeping old around to allow testing easily (for now) - DBCheckpointStorage.DBFlowCheckpoint::class.java, + mappedTypes = listOf(DBCheckpointStorage.DBFlowCheckpoint::class.java, DBCheckpointStorage.DBFlowCheckpointBlob::class.java, DBCheckpointStorage.DBFlowResult::class.java, DBCheckpointStorage.DBFlowException::class.java, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 3ffdd4b709..e7a45b145b 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -31,8 +31,7 @@ class ActionExecutorImpl( private val checkpointStorage: CheckpointStorage, private val flowMessaging: FlowMessaging, private val stateMachineManager: StateMachineManagerInternal, - private val checkpointSerializationContext: CheckpointSerializationContext, - metrics: MetricRegistry + private val checkpointSerializationContext: CheckpointSerializationContext ) : ActionExecutor { private companion object { @@ -48,12 +47,6 @@ class ActionExecutorImpl( } } - private val checkpointingMeter = metrics.meter("Flows.Checkpointing Rate") - private val checkpointSizesThisSecond = SlidingTimeWindowReservoir(1, TimeUnit.SECONDS) - private val lastBandwidthUpdate = AtomicLong(0) - private val checkpointBandwidthHist = metrics.register("Flows.CheckpointVolumeBytesPerSecondHist", Histogram(SlidingTimeWindowArrayReservoir(1, TimeUnit.DAYS))) - private val checkpointBandwidth = metrics.register("Flows.CheckpointVolumeBytesPerSecondCurrent", LatchedGauge(checkpointSizesThisSecond)) - @Suspendable override fun executeAction(fiber: FlowFiber, action: Action) { log.trace { "Flow ${fiber.id} executing $action" } @@ -100,21 +93,12 @@ class ActionExecutorImpl( @Suspendable private fun executePersistCheckpoint(action: Action.PersistCheckpoint) { - val checkpointBytes = serializeCheckpoint(action.checkpoint) + val checkpoint = action.checkpoint + val serializedFlowState = checkpoint.flowState.checkpointSerialize(checkpointSerializationContext) if (action.isCheckpointUpdate) { - checkpointStorage.updateCheckpoint(action.id, checkpointBytes) + checkpointStorage.updateCheckpoint(action.id, checkpoint, serializedFlowState) } else { - checkpointStorage.addCheckpoint(action.id, checkpointBytes) - } - checkpointingMeter.mark() - checkpointSizesThisSecond.update(checkpointBytes.size.toLong()) - var lastUpdateTime = lastBandwidthUpdate.get() - while (System.nanoTime() - lastUpdateTime > TimeUnit.SECONDS.toNanos(1)) { - if (lastBandwidthUpdate.compareAndSet(lastUpdateTime, System.nanoTime())) { - val checkpointVolume = checkpointSizesThisSecond.snapshot.values.sum() - checkpointBandwidthHist.update(checkpointVolume) - } - lastUpdateTime = lastBandwidthUpdate.get() + checkpointStorage.addCheckpoint(action.id, checkpoint, serializedFlowState) } } @@ -258,10 +242,6 @@ class ActionExecutorImpl( stateMachineManager.retryFlowFromSafePoint(action.currentState) } - private fun serializeCheckpoint(checkpoint: Checkpoint): SerializedBytes { - return checkpoint.checkpointSerialize(context = checkpointSerializationContext) - } - private fun cancelFlowTimeout(action: Action.CancelFlowTimeout) { stateMachineManager.cancelFlowTimeout(action.flowId) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 1352fb6ec6..d722ad9fdb 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -590,7 +590,7 @@ class SingleThreadedStateMachineManager( // The checkpoint will be missing if the flow failed before persisting the original checkpoint // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) checkpointStorage.getCheckpoint(flowId)?.let { serializedCheckpoint -> - val checkpoint = tryCheckpointDeserialize(serializedCheckpoint, flowId) + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) if (checkpoint == null) { return openFuture>().mapError { IllegalStateException("Unable to deserialize database checkpoint for flow $flowId. " + @@ -758,14 +758,23 @@ class SingleThreadedStateMachineManager( } } + private fun tryDeserializeCheckpoint(serializedCheckpoint: Checkpoint.Serialized, flowId: StateMachineRunId): Checkpoint? { + return try { + serializedCheckpoint.deserialize(checkpointSerializationContext!!) + } catch (e: Exception) { + logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) + null + } + } + private fun createFlowFromCheckpoint( id: StateMachineRunId, - serializedCheckpoint: SerializedBytes, + serializedCheckpoint: Checkpoint.Serialized, isAnyCheckpointPersisted: Boolean, isStartIdempotent: Boolean, initialDeduplicationHandler: DeduplicationHandler? ): Flow? { - val checkpoint = tryCheckpointDeserialize(serializedCheckpoint, id) ?: return null + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id) ?: return null val flowState = checkpoint.flowState val resultFuture = openFuture() val fiber = when (flowState) { @@ -865,8 +874,7 @@ class SingleThreadedStateMachineManager( checkpointStorage, flowMessaging, this, - checkpointSerializationContext, - metrics + checkpointSerializationContext ) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index f4c9e790b6..99882aa022 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -7,7 +7,12 @@ import net.corda.core.flows.FlowInfo import net.corda.core.flows.FlowLogic import net.corda.core.identity.Party import net.corda.core.internal.FlowIORequest +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.deserialize +import net.corda.core.serialization.internal.CheckpointSerializationContext +import net.corda.core.serialization.internal.checkpointDeserialize import net.corda.core.utilities.Try import net.corda.node.services.messaging.DeduplicationHandler import java.time.Instant @@ -57,9 +62,10 @@ data class Checkpoint( val result: Any? = null, val status: FlowStatus = FlowStatus.RUNNABLE, val progressStep: String? = null, - val flowIoRequest: FlowIORequest<*>? = null, + val flowIoRequest: Class>? = null, val compatible: Boolean = true ) { + @CordaSerializable enum class FlowStatus { RUNNABLE, FAILED, @@ -84,9 +90,15 @@ data class Checkpoint( ): Try { return SubFlow.create(flowLogicClass, subFlowVersion, isEnabledTimedFlow).map { topLevelSubFlow -> Checkpoint( - checkpointState = CheckpointState(invocationContext, ourIdentity, emptyMap(), listOf(topLevelSubFlow), numberOfSuspends = 0), - flowState = FlowState.Unstarted(flowStart, frozenFlowLogic), - errorState = ErrorState.Clean + checkpointState = CheckpointState( + invocationContext, + ourIdentity, + emptyMap(), + listOf(topLevelSubFlow), + numberOfSuspends = 0 + ), + errorState = ErrorState.Clean, + flowState = FlowState.Unstarted(flowStart, frozenFlowLogic) ) } } @@ -123,6 +135,41 @@ data class Checkpoint( fun addSubflow(subFlow: SubFlow) : Checkpoint { return copy(checkpointState = checkpointState.copy(subFlowStack = checkpointState.subFlowStack + subFlow)) } + + /** + * A partially serialized form of [Checkpoint]. + * + * [Checkpoint.Serialized] contains the same fields as [Checkpoint] except that some of its fields are still serialized. The checkpoint + * can then be deserialized as needed. + */ + data class Serialized( + val serializedCheckpointState: SerializedBytes, + val serializedFlowState: SerializedBytes, + val errorState: ErrorState, + val result: SerializedBytes?, + val status: FlowStatus, + val progressStep: String?, + val flowIoRequest: Class>?, + val compatible: Boolean + ) { + /** + * Deserializes the serialized fields contained in [Checkpoint.Serialized]. + * + * @return A [Checkpoint] with all its fields filled in from [Checkpoint.Serialized] + */ + fun deserialize(checkpointSerializationContext: CheckpointSerializationContext): Checkpoint { + return Checkpoint( + checkpointState = serializedCheckpointState.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), + flowState = serializedFlowState.checkpointDeserialize(checkpointSerializationContext), + errorState = errorState, + result = result?.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), + status = status, + progressStep = progressStep, + flowIoRequest = flowIoRequest, + compatible = compatible + ) + } + } } /** @@ -132,12 +179,13 @@ data class Checkpoint( * @param subFlowStack the stack of currently executing subflows. * @param numberOfSuspends the number of flow suspends due to IO API calls. */ +@CordaSerializable data class CheckpointState( - val invocationContext: InvocationContext, - val ourIdentity: Party, - val sessions: SessionMap, // This must preserve the insertion order! - val subFlowStack: List, - val numberOfSuspends: Int + val invocationContext: InvocationContext, + val ourIdentity: Party, + val sessions: SessionMap, // This must preserve the insertion order! + val subFlowStack: List, + val numberOfSuspends: Int ) /** @@ -254,17 +302,20 @@ sealed class FlowState { * @param exception the exception itself. Note that this may not contain information about the source error depending * on whether the source error was a FlowException or otherwise. */ +@CordaSerializable data class FlowError(val errorId: Long, val exception: Throwable) /** * The flow's error state. */ +@CordaSerializable sealed class ErrorState { abstract fun addErrors(newErrors: List): ErrorState /** * The flow is in a clean state. */ + @CordaSerializable object Clean : ErrorState() { override fun addErrors(newErrors: List): ErrorState { return Errored(newErrors, 0, false) @@ -281,6 +332,7 @@ sealed class ErrorState { * @param propagating true if error propagation was triggered. If this is set the dirtiness is permanent as the * sessions associated with the flow have been (or about to be) dirtied in counter-flows. */ + @CordaSerializable data class Errored( val errors: List, val propagatedIndex: Int, diff --git a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml index fdc812a02c..482c4d6418 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml @@ -5,30 +5,31 @@ logicalFilePath="migration/node-services.changelog-init.xml"> - + - - - + + - + + + - + + - - \ No newline at end of file + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 9602ec51b3..acb3557060 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -5,7 +5,8 @@ logicalFilePath="migration/node-services.changelog-init.xml"> - + + @@ -18,6 +19,9 @@ + + + @@ -84,7 +88,7 @@ - + @@ -94,7 +98,7 @@ - + @@ -136,4 +140,4 @@ - \ No newline at end of file + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 2c1789fc95..266361124e 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -5,7 +5,8 @@ logicalFilePath="migration/node-services.changelog-init.xml"> - + + @@ -18,6 +19,9 @@ + + + @@ -84,7 +88,7 @@ - + @@ -94,7 +98,7 @@ - + @@ -136,4 +140,4 @@ - \ No newline at end of file + diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index c7ceb785da..c30044cb02 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -3,6 +3,7 @@ package net.corda.node.services.persistence import net.corda.core.context.InvocationContext import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId +import net.corda.core.internal.PLATFORM_VERSION import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize @@ -10,11 +11,16 @@ import net.corda.node.internal.CheckpointIncompatibleException import net.corda.node.internal.CheckpointVerifier import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.Checkpoint +import net.corda.node.services.statemachine.CheckpointState +import net.corda.node.services.statemachine.ErrorState +import net.corda.node.services.statemachine.FlowError import net.corda.node.services.statemachine.FlowStart +import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.SubFlowVersion import net.corda.node.services.transactions.PersistentUniquenessProvider import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig +import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity @@ -28,9 +34,15 @@ import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test +import java.lang.IllegalStateException +import java.time.Instant import kotlin.streams.toList +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue -internal fun CheckpointStorage.checkpoints(): List> { +internal fun CheckpointStorage.checkpoints(): List { return getAllCheckpoints().use { it.map { it.second }.toList() } @@ -61,26 +73,42 @@ class DBCheckpointStorageTests { LogHelper.reset(PersistentUniquenessProvider::class) } - @Test(timeout=300_000) - fun `add new checkpoint`() { + @Test(timeout = 300_000) + fun `add new checkpoint`() { val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint) + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(checkpoint) + assertEquals( + checkpoint, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) } newCheckpointStorage() database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(checkpoint) + assertEquals( + checkpoint, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) + session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).also { + assertNotNull(it) + assertNotNull(it.blob) + } } } - @Test(timeout=300_000) - fun `remove checkpoint`() { + @Test(timeout = 300_000) + fun `remove checkpoint`() { val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint) + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { checkpointStorage.removeCheckpoint(id) @@ -94,58 +122,87 @@ class DBCheckpointStorageTests { } } - @Test(timeout=300_000) - fun `add and remove checkpoint in single commit operate`() { + @Test(timeout = 300_000) + fun `add and remove checkpoint in single commit operation`() { val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) val (id2, checkpoint2) = newCheckpoint() + val serializedFlowState2 = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint) - checkpointStorage.addCheckpoint(id2, checkpoint2) + createMetadataRecord(checkpoint) + createMetadataRecord(checkpoint2) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id2, checkpoint2, serializedFlowState2) checkpointStorage.removeCheckpoint(id) } database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(checkpoint2) + assertEquals( + checkpoint2, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) } newCheckpointStorage() database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(checkpoint2) + assertEquals( + checkpoint2, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) } } - @Test(timeout=300_000) - fun `add two checkpoints then remove first one`() { + @Test(timeout = 300_000) + fun `add two checkpoints then remove first one`() { val (id, firstCheckpoint) = newCheckpoint() + val serializedFirstFlowState = + firstCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { - checkpointStorage.addCheckpoint(id, firstCheckpoint) + createMetadataRecord(firstCheckpoint) + checkpointStorage.addCheckpoint(id, firstCheckpoint, serializedFirstFlowState) } val (id2, secondCheckpoint) = newCheckpoint() + val serializedSecondFlowState = + secondCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage.addCheckpoint(id2, secondCheckpoint) + createMetadataRecord(secondCheckpoint) + checkpointStorage.addCheckpoint(id2, secondCheckpoint, serializedSecondFlowState) } database.transaction { checkpointStorage.removeCheckpoint(id) } database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(secondCheckpoint) + assertEquals( + secondCheckpoint, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) } newCheckpointStorage() database.transaction { - assertThat(checkpointStorage.checkpoints()).containsExactly(secondCheckpoint) + assertEquals( + secondCheckpoint, + checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ) } } - @Test(timeout=300_000) - fun `add checkpoint and then remove after 'restart'`() { + @Test(timeout = 300_000) + fun `add checkpoint and then remove after 'restart'`() { val (id, originalCheckpoint) = newCheckpoint() + val serializedOriginalFlowState = + originalCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage.addCheckpoint(id, originalCheckpoint) + createMetadataRecord(originalCheckpoint) + checkpointStorage.addCheckpoint(id, originalCheckpoint, serializedOriginalFlowState) } newCheckpointStorage() val reconstructedCheckpoint = database.transaction { checkpointStorage.checkpoints().single() } database.transaction { - assertThat(reconstructedCheckpoint).isEqualTo(originalCheckpoint).isNotSameAs(originalCheckpoint) + assertEquals(originalCheckpoint, reconstructedCheckpoint.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT)) + assertThat(reconstructedCheckpoint.serializedFlowState).isEqualTo(serializedOriginalFlowState) + .isNotSameAs(serializedOriginalFlowState) } database.transaction { checkpointStorage.removeCheckpoint(id) @@ -155,12 +212,15 @@ class DBCheckpointStorageTests { } } - @Test(timeout=300_000) - fun `verify checkpoints compatible`() { + @Test(timeout = 300_000) + fun `verify checkpoints compatible`() { val mockServices = MockServices(emptyList(), ALICE.name) database.transaction { val (id, checkpoint) = newCheckpoint(1) - checkpointStorage.addCheckpoint(id, checkpoint) + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { @@ -169,7 +229,10 @@ class DBCheckpointStorageTests { database.transaction { val (id1, checkpoint1) = newCheckpoint(2) - checkpointStorage.addCheckpoint(id1, checkpoint1) + val serializedFlowState1 = + checkpoint1.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + createMetadataRecord(checkpoint1) + checkpointStorage.addCheckpoint(id1, checkpoint1, serializedFlowState1) } assertThatThrownBy { @@ -179,21 +242,158 @@ class DBCheckpointStorageTests { }.isInstanceOf(CheckpointIncompatibleException::class.java) } - private fun newCheckpointStorage() { + @Test(timeout = 300_000) + fun `checkpoint can be recreated from database record`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) database.transaction { - checkpointStorage = DBCheckpointStorage() + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + database.transaction { + assertEquals(serializedFlowState, checkpointStorage.checkpoints().single().serializedFlowState) + } + database.transaction { + assertEquals(checkpoint, checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT)) } } - private fun newCheckpoint(version: Int = 1): Pair> { + @Test(timeout = 300_000) + fun `update checkpoint with result information`() { + val result = "This is the result" + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.copy(result = result) + val updatedSerializedFlowState = + updatedCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + } + database.transaction { + assertEquals( + result, + checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).result + ) + assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) + } + } + + @Test(timeout = 300_000) + fun `update checkpoint with error information`() { + val exception = IllegalStateException("I am a naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.copy( + errorState = ErrorState.Errored( + listOf( + FlowError( + 0, + exception + ) + ), 0, false + ) + ) + val updatedSerializedFlowState = updatedCheckpoint.flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { + // Checkpoint always returns clean error state when retrieved via [getCheckpoint] + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) + assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) + } + } + + @Test(timeout = 300_000) + fun `clean checkpoints clear out error information from the database`() { + val exception = IllegalStateException("I am a naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.copy( + errorState = ErrorState.Errored( + listOf( + FlowError( + 0, + exception + ) + ), 0, false + ) + ) + val updatedSerializedFlowState = updatedCheckpoint.flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { + // Checkpoint always returns clean error state when retrieved via [getCheckpoint] + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) + } + // Set back to clean + database.transaction { checkpointStorage.updateCheckpoint(id, checkpoint, serializedFlowState) } + database.transaction { + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) + assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) + } + } + + private fun newCheckpointStorage() { + database.transaction { + checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }) + } + } + + private fun newCheckpoint(version: Int = 1): Pair { val id = StateMachineRunId.createRandom() val logic: FlowLogic<*> = object : FlowLogic() { override fun call() {} } val frozenLogic = logic.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - val checkpoint = Checkpoint.create(InvocationContext.shell(), FlowStart.Explicit, logic.javaClass, frozenLogic, ALICE, SubFlowVersion.CoreFlow(version), false) - .getOrThrow() - return id to checkpoint.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val checkpoint = Checkpoint.create( + InvocationContext.shell(), + FlowStart.Explicit, + logic.javaClass, + frozenLogic, + ALICE, + SubFlowVersion.CoreFlow(version), + false + ) + .getOrThrow() + return id to checkpoint } + private fun DatabaseTransaction.createMetadataRecord(checkpoint: Checkpoint) { + val metadata = DBCheckpointStorage.DBFlowMetadata( + invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value, + flowId = null, + flowName = "random.flow", + userSuppliedIdentifier = null, + startType = DBCheckpointStorage.StartReason.RPC, + launchingCordapp = "this cordapp", + platformVersion = PLATFORM_VERSION, + rpcUsername = "Batman", + invocationInstant = checkpoint.checkpointState.invocationContext.trace.invocationId.timestamp, + receivedInstant = Instant.now(), + startInstant = null, + finishInstant = null + ) + session.save(metadata) + } } diff --git a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt index bf9a482e66..14c4586ee6 100644 --- a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt @@ -23,9 +23,12 @@ import net.corda.core.serialization.internal.checkpointSerialize import net.corda.nodeapi.internal.lifecycle.NodeServicesContext import net.corda.nodeapi.internal.lifecycle.NodeLifecycleEvent import net.corda.node.internal.NodeStartup +import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.statemachine.Checkpoint +import net.corda.node.services.statemachine.CheckpointState import net.corda.node.services.statemachine.FlowStart +import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.SubFlowVersion import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.testing.core.SerializationEnvironmentRule @@ -104,7 +107,7 @@ class CheckpointDumperImplTest { // add a checkpoint val (id, checkpoint) = newCheckpoint() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) } dumper.dumpCheckpoints() @@ -130,7 +133,7 @@ class CheckpointDumperImplTest { // add a checkpoint val (id, checkpoint) = newCheckpoint() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) } dumper.dumpCheckpoints() @@ -140,11 +143,18 @@ class CheckpointDumperImplTest { private fun newCheckpointStorage() { database.transaction { - checkpointStorage = DBCheckpointStorage() + checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }) } } - private fun newCheckpoint(version: Int = 1): Pair> { + private fun newCheckpoint(version: Int = 1): Pair { val id = StateMachineRunId.createRandom() val logic: FlowLogic<*> = object : FlowLogic() { override fun call() {} @@ -152,6 +162,10 @@ class CheckpointDumperImplTest { val frozenLogic = logic.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) val checkpoint = Checkpoint.create(InvocationContext.shell(), FlowStart.Explicit, logic.javaClass, frozenLogic, myself.identity.party, SubFlowVersion.CoreFlow(version), false) .getOrThrow() - return id to checkpoint.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + return id to checkpoint + } + + private fun serializeFlowState(checkpoint: Checkpoint): SerializedBytes { + return checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) } } \ No newline at end of file From d5e84a4f936aa81015e24dea14b1575ba73b3150 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Mon, 9 Mar 2020 10:01:30 +0000 Subject: [PATCH 21/49] NOTICK Do not cascade checkpoint tables (#6040) Do not cascade updates to checkpoint error and result tables to hopefully improve database performance moving forward. Because the joined tables are no longer being updated by updating the main `DBFlowCheckpoint` entity, they must be created/updated/deleted manually. The checkpoint blobs still cascade as they pretty much always evolve in tandem with the main checkpoint table. --- .../CordaPersistenceServiceTests.kt | 4 +- .../persistence/DBCheckpointStorage.kt | 71 ++++- .../persistence/DBCheckpointStorageTests.kt | 275 ++++++++++++------ 3 files changed, 257 insertions(+), 93 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index ff36d5e7a2..a983b55798 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -11,9 +11,7 @@ import net.corda.core.node.services.CordaService import net.corda.core.node.services.vault.SessionScope import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.utilities.getOrThrow -import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.Checkpoint.FlowStatus -import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.driver @@ -71,7 +69,7 @@ class CordaPersistenceServiceTests { hmac = ByteArray(16), persistedInstant = now ), - result = DBCheckpointStorage.DBFlowResult(value = ByteArray(16), persistedInstant = now), + result = null, exceptionDetails = null, status = FlowStatus.RUNNABLE, compatible = false, diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 45a6739b2a..adde4e489f 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -37,6 +37,7 @@ import javax.persistence.OneToOne /** * Simple checkpoint key value storage in DB. */ +@Suppress("TooManyFunctions") class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointPerformanceRecorder) : CheckpointStorage { companion object { @@ -83,11 +84,11 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") var blob: DBFlowCheckpointBlob, - @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) + @OneToOne(fetch = FetchType.LAZY, optional = true) @JoinColumn(name = "result_id", referencedColumnName = "id") var result: DBFlowResult?, - @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) + @OneToOne(fetch = FetchType.LAZY, optional = true) @JoinColumn(name = "error_id", referencedColumnName = "id") var exceptionDetails: DBFlowException?, @@ -117,7 +118,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @Id @GeneratedValue(strategy = GenerationType.SEQUENCE) @Column(name = "id", nullable = false) - private var id: Long = 0, + var id: Long = 0, @Type(type = "corda-blob") @Column(name = "checkpoint_value", nullable = false) @@ -141,7 +142,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @Id @Column(name = "id", nullable = false) @GeneratedValue(strategy = GenerationType.SEQUENCE) - private var id: Long = 0, + var id: Long = 0, @Type(type = "corda-blob") @Column(name = "result_value", nullable = false) @@ -157,7 +158,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @Id @Column(name = "id", nullable = false) @GeneratedValue(strategy = GenerationType.SEQUENCE) - private var id: Long = 0, + var id: Long = 0, @Column(name = "type", nullable = false) var type: Class, @@ -319,14 +320,16 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP val flowId = id.uuid.toString() val now = Instant.now() + // Load the previous entity from the hibernate cache so the meta data join does not get updated + val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) + val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) - val result = checkpoint.result?.let { createDBFlowResult(it, now) } - val exceptionDetails = (checkpoint.errorState as? ErrorState.Errored)?.let { createDBFlowException(it, now) } - // Load the previous entity from the hibernate cache so the meta data join does not get updated - val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) + val result = updateDBFlowResult(entity, checkpoint, now) + val exceptionDetails = updateDBFlowException(entity, checkpoint, now) + return entity.apply { this.blob = blob this.result = result @@ -354,6 +357,31 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP ) } + /** + * Creates, updates or deletes the result related to the current flow/checkpoint. + * + * This is needed because updates are not cascading via Hibernate, therefore operations must be handled manually. + * + * A [DBFlowResult] is created if [DBFlowCheckpoint.result] does not exist and the [Checkpoint] has a result.. + * The existing [DBFlowResult] is updated if [DBFlowCheckpoint.result] exists and the [Checkpoint] has a result. + * The existing [DBFlowResult] is deleted if [DBFlowCheckpoint.result] exists and the [Checkpoint] has no result. + * Nothing happens if both [DBFlowCheckpoint] and [Checkpoint] do not have a result. + */ + private fun updateDBFlowResult(entity: DBFlowCheckpoint, checkpoint: Checkpoint, now: Instant): DBFlowResult? { + val result = checkpoint.result?.let { createDBFlowResult(it, now) } + if (entity.result != null) { + if (result != null) { + result.id = entity.result!!.id + currentDBSession().update(result) + } else { + currentDBSession().delete(entity.result) + } + } else if (result != null) { + currentDBSession().save(result) + } + return result + } + private fun createDBFlowResult(result: Any, now: Instant): DBFlowResult { return DBFlowResult( value = result.storageSerialize().bytes, @@ -361,6 +389,31 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP ) } + /** + * Creates, updates or deletes the error related to the current flow/checkpoint. + * + * This is needed because updates are not cascading via Hibernate, therefore operations must be handled manually. + * + * A [DBFlowException] is created if [DBFlowCheckpoint.exceptionDetails] does not exist and the [Checkpoint] has an error attached to it. + * The existing [DBFlowException] is updated if [DBFlowCheckpoint.exceptionDetails] exists and the [Checkpoint] has an error. + * The existing [DBFlowException] is deleted if [DBFlowCheckpoint.exceptionDetails] exists and the [Checkpoint] has no error. + * Nothing happens if both [DBFlowCheckpoint] and [Checkpoint] are related to no errors. + */ + private fun updateDBFlowException(entity: DBFlowCheckpoint, checkpoint: Checkpoint, now: Instant): DBFlowException? { + val exceptionDetails = (checkpoint.errorState as? ErrorState.Errored)?.let { createDBFlowException(it, now) } + if (entity.exceptionDetails != null) { + if (exceptionDetails != null) { + exceptionDetails.id = entity.exceptionDetails!!.id + currentDBSession().update(exceptionDetails) + } else { + currentDBSession().delete(entity.exceptionDetails) + } + } else if (exceptionDetails != null) { + currentDBSession().save(exceptionDetails) + } + return exceptionDetails + } + private fun createDBFlowException(errorState: ErrorState.Errored, now: Instant): DBFlowException { return errorState.errors.last().exception.let { DBFlowException( diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index c30044cb02..08be3bbf3e 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -3,6 +3,7 @@ package net.corda.node.services.persistence import net.corda.core.context.InvocationContext import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId +import net.corda.core.internal.FlowIORequest import net.corda.core.internal.PLATFORM_VERSION import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults @@ -34,7 +35,6 @@ import org.junit.After import org.junit.Before import org.junit.Rule import org.junit.Test -import java.lang.IllegalStateException import java.time.Instant import kotlin.streams.toList import kotlin.test.assertEquals @@ -76,23 +76,23 @@ class DBCheckpointStorageTests { @Test(timeout = 300_000) fun `add new checkpoint`() { val (id, checkpoint) = newCheckpoint() - val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState = checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { + assertEquals(serializedFlowState, checkpointStorage.checkpoints().single().serializedFlowState) assertEquals( checkpoint, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) } newCheckpointStorage() database.transaction { assertEquals( checkpoint, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).also { assertNotNull(it) @@ -101,11 +101,44 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `update a checkpoint`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val logic: FlowLogic<*> = object : FlowLogic() { + override fun call(): String { + return "Updated flow logic" + } + } + val updatedCheckpoint = checkpoint.copy( + checkpointState = checkpoint.checkpointState.copy(numberOfSuspends = 20), + flowState = FlowState.Unstarted( + FlowStart.Explicit, + logic.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + ), + progressStep = "I have made progress", + flowIoRequest = FlowIORequest.SendAndReceive::class.java + ) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + } + database.transaction { + assertEquals( + updatedCheckpoint, + checkpointStorage.checkpoints().single().deserialize() + ) + } + } + @Test(timeout = 300_000) fun `remove checkpoint`() { val (id, checkpoint) = newCheckpoint() - val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState = checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) @@ -125,10 +158,9 @@ class DBCheckpointStorageTests { @Test(timeout = 300_000) fun `add and remove checkpoint in single commit operation`() { val (id, checkpoint) = newCheckpoint() - val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState = checkpoint.serializeFlowState() val (id2, checkpoint2) = newCheckpoint() - val serializedFlowState2 = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState2 = checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) createMetadataRecord(checkpoint2) @@ -139,14 +171,14 @@ class DBCheckpointStorageTests { database.transaction { assertEquals( checkpoint2, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) } newCheckpointStorage() database.transaction { assertEquals( checkpoint2, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) } } @@ -154,16 +186,14 @@ class DBCheckpointStorageTests { @Test(timeout = 300_000) fun `add two checkpoints then remove first one`() { val (id, firstCheckpoint) = newCheckpoint() - val serializedFirstFlowState = - firstCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFirstFlowState = firstCheckpoint.serializeFlowState() database.transaction { createMetadataRecord(firstCheckpoint) checkpointStorage.addCheckpoint(id, firstCheckpoint, serializedFirstFlowState) } val (id2, secondCheckpoint) = newCheckpoint() - val serializedSecondFlowState = - secondCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedSecondFlowState = secondCheckpoint.serializeFlowState() database.transaction { createMetadataRecord(secondCheckpoint) checkpointStorage.addCheckpoint(id2, secondCheckpoint, serializedSecondFlowState) @@ -174,14 +204,14 @@ class DBCheckpointStorageTests { database.transaction { assertEquals( secondCheckpoint, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) } newCheckpointStorage() database.transaction { assertEquals( secondCheckpoint, - checkpointStorage.checkpoints().single().deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.checkpoints().single().deserialize() ) } } @@ -189,8 +219,7 @@ class DBCheckpointStorageTests { @Test(timeout = 300_000) fun `add checkpoint and then remove after 'restart'`() { val (id, originalCheckpoint) = newCheckpoint() - val serializedOriginalFlowState = - originalCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedOriginalFlowState = originalCheckpoint.serializeFlowState() database.transaction { createMetadataRecord(originalCheckpoint) checkpointStorage.addCheckpoint(id, originalCheckpoint, serializedOriginalFlowState) @@ -200,7 +229,7 @@ class DBCheckpointStorageTests { checkpointStorage.checkpoints().single() } database.transaction { - assertEquals(originalCheckpoint, reconstructedCheckpoint.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT)) + assertEquals(originalCheckpoint, reconstructedCheckpoint.deserialize()) assertThat(reconstructedCheckpoint.serializedFlowState).isEqualTo(serializedOriginalFlowState) .isNotSameAs(serializedOriginalFlowState) } @@ -217,8 +246,7 @@ class DBCheckpointStorageTests { val mockServices = MockServices(emptyList(), ALICE.name) database.transaction { val (id, checkpoint) = newCheckpoint(1) - val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState = checkpoint.serializeFlowState() createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } @@ -229,8 +257,7 @@ class DBCheckpointStorageTests { database.transaction { val (id1, checkpoint1) = newCheckpoint(2) - val serializedFlowState1 = - checkpoint1.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState1 = checkpoint1.serializeFlowState() createMetadataRecord(checkpoint1) checkpointStorage.addCheckpoint(id1, checkpoint1, serializedFlowState1) } @@ -243,107 +270,172 @@ class DBCheckpointStorageTests { } @Test(timeout = 300_000) - fun `checkpoint can be recreated from database record`() { - val (id, checkpoint) = newCheckpoint() - val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - database.transaction { - createMetadataRecord(checkpoint) - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) - } - database.transaction { - assertEquals(serializedFlowState, checkpointStorage.checkpoints().single().serializedFlowState) - } - database.transaction { - assertEquals(checkpoint, checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT)) - } - } - - @Test(timeout = 300_000) - fun `update checkpoint with result information`() { + fun `update checkpoint with result information creates new result database record`() { val result = "This is the result" val (id, checkpoint) = newCheckpoint() val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.copy(result = result) - val updatedSerializedFlowState = - updatedCheckpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } database.transaction { assertEquals( result, - checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).result + checkpointStorage.getCheckpoint(id)!!.deserialize().result ) assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) + assertEquals(1, session.createQuery(criteria).resultList.size) } } @Test(timeout = 300_000) - fun `update checkpoint with error information`() { - val exception = IllegalStateException("I am a naughty exception") + fun `update checkpoint with result information updates existing result database record`() { + val result = "This is the result" + val somehowThereIsANewResult = "Another result (which should not be possible!)" val (id, checkpoint) = newCheckpoint() val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } - val updatedCheckpoint = checkpoint.copy( - errorState = ErrorState.Errored( - listOf( - FlowError( - 0, - exception - ) - ), 0, false - ) - ) - val updatedSerializedFlowState = updatedCheckpoint.flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + val updatedCheckpoint = checkpoint.copy(result = result) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { - // Checkpoint always returns clean error state when retrieved via [getCheckpoint] - assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) - assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + } + val updatedCheckpoint2 = checkpoint.copy(result = somehowThereIsANewResult) + val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) + } + database.transaction { + assertEquals( + somehowThereIsANewResult, + checkpointStorage.getCheckpoint(id)!!.deserialize().result + ) + assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) + assertEquals(1, session.createQuery(criteria).resultList.size) } } @Test(timeout = 300_000) - fun `clean checkpoints clear out error information from the database`() { - val exception = IllegalStateException("I am a naughty exception") + fun `removing result information from checkpoint deletes existing result database record`() { + val result = "This is the result" val (id, checkpoint) = newCheckpoint() val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpoint.serializeFlowState() database.transaction { createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } - val updatedCheckpoint = checkpoint.copy( - errorState = ErrorState.Errored( - listOf( - FlowError( - 0, - exception - ) - ), 0, false - ) - ) - val updatedSerializedFlowState = updatedCheckpoint.flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val updatedCheckpoint = checkpoint.copy(result = result) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + } + val updatedCheckpoint2 = checkpoint.copy(result = null) + val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) + } + database.transaction { + assertNull(checkpointStorage.getCheckpoint(id)!!.deserialize().result) + assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) + assertEquals(0, session.createQuery(criteria).resultList.size) + } + } + + @Test(timeout = 300_000) + fun `update checkpoint with error information creates a new error database record`() { + val exception = IllegalStateException("I am a naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.addError(exception) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } database.transaction { // Checkpoint always returns clean error state when retrieved via [getCheckpoint] - assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) + val exceptionDetails = session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails + assertNotNull(exceptionDetails) + assertEquals(exception::class.java, exceptionDetails!!.type) + assertEquals(exception.message, exceptionDetails.message) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) + assertEquals(1, session.createQuery(criteria).resultList.size) + } + } + + @Test(timeout = 300_000) + fun `update checkpoint with new error information updates the existing error database record`() { + val illegalStateException = IllegalStateException("I am a naughty exception") + val illegalArgumentException = IllegalArgumentException("I am a very naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint1 = checkpoint.addError(illegalStateException) + val updatedSerializedFlowState1 = updatedCheckpoint1.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint1, updatedSerializedFlowState1) } + // Set back to clean + val updatedCheckpoint2 = checkpoint.addError(illegalArgumentException) + val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) } + database.transaction { + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) + val exceptionDetails = session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails + assertNotNull(exceptionDetails) + assertEquals(illegalArgumentException::class.java, exceptionDetails!!.type) + assertEquals(illegalArgumentException.message, exceptionDetails.message) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) + assertEquals(1, session.createQuery(criteria).resultList.size) + } + } + + @Test(timeout = 300_000) + fun `clean checkpoints delete the error record from the database`() { + val exception = IllegalStateException("I am a naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + createMetadataRecord(checkpoint) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.addError(exception) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { + // Checkpoint always returns clean error state when retrieved via [getCheckpoint] + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) } // Set back to clean database.transaction { checkpointStorage.updateCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { - assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT).errorState is ErrorState.Clean) + assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) + val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) + criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) + assertEquals(0, session.createQuery(criteria).resultList.size) } } @@ -379,6 +471,27 @@ class DBCheckpointStorageTests { return id to checkpoint } + private fun Checkpoint.serializeFlowState(): SerializedBytes { + return flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + } + + private fun Checkpoint.Serialized.deserialize(): Checkpoint { + return deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + } + + private fun Checkpoint.addError(exception: Exception): Checkpoint { + return copy( + errorState = ErrorState.Errored( + listOf( + FlowError( + 0, + exception + ) + ), 0, false + ) + ) + } + private fun DatabaseTransaction.createMetadataRecord(checkpoint: Checkpoint) { val metadata = DBCheckpointStorage.DBFlowMetadata( invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value, From 51db2c2d7f5b1213a647b094158c020baf91f012 Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Tue, 10 Mar 2020 15:46:37 +0000 Subject: [PATCH 22/49] CORDA-3600 Add flowIO request to checkpoint (#6017) * Update Checkpoint DB to update flow io request * Modify flow monitor to update Checkpoint DB with waiting flows This happens periodically. * Refactored code to avoid looping twice and updated tests * Fix tests after rebasing * Fix MR comments (non-functional refactor of tests + FlowMonitor). * Made visible for testing method private in DBCheckpointStorage This is not needed anymore. * Explicity check if ioRequestType has changed in update method * Fix shadowing warning * Import non deprecated Assert into test * Use AssertEquals not assert in test * Address more comments (minor refactor) of DBCheckpointStorage * Minor fix use it instead of referencing object explicitly * Add null check to DBCheckpointStorage * Revert changes to Flow Monitor. We will instead store the information in the main thread of the state machine. * Remove now uneeded API and make statemachine update ioRequest * Add Integration Test to check statemachine updates DB on Recieve * Use simpleName in checkpoint storage instead of class. Hibernate was previously resetting the class field this is now set to null (when getting checkpoint form DB) and a new method for getting back the simple name as a string. * Update StateMachineState to store simple name. * Fix after rebase broke stuff + renamed test * Fix Detekt issue * Remove uneeded null assertion --- .../CordaPersistenceServiceTests.kt | 2 +- .../node/services/api/CheckpointStorage.kt | 3 +- .../persistence/DBCheckpointStorage.kt | 13 ++++---- .../statemachine/StateMachineState.kt | 4 +-- .../transitions/TopLevelTransition.kt | 3 +- .../persistence/DBCheckpointStorageTests.kt | 30 +++++++++++++++++-- .../statemachine/FlowFrameworkTests.kt | 27 +++++++++++++++++ 7 files changed, 68 insertions(+), 14 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index a983b55798..0d5572c3dc 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -74,7 +74,7 @@ class CordaPersistenceServiceTests { status = FlowStatus.RUNNABLE, compatible = false, progressStep = "", - ioRequestType = FlowIORequest.ForceCheckpoint.javaClass, + ioRequestType = FlowIORequest.ForceCheckpoint::class.java.simpleName, checkpointInstant = Instant.now(), flowMetadata = createMetadataRecord(UUID.randomUUID(), now) ) diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index 8268322233..953c2fc562 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -1,6 +1,7 @@ package net.corda.node.services.api import net.corda.core.flows.StateMachineRunId +import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.FlowState @@ -41,4 +42,4 @@ interface CheckpointStorage { * underlying database connection is closed, so any processing should happen before it is closed. */ fun getAllCheckpoints(): Stream> -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index adde4e489f..ae082b928a 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -1,7 +1,6 @@ package net.corda.node.services.persistence import net.corda.core.flows.StateMachineRunId -import net.corda.core.internal.FlowIORequest import net.corda.core.internal.PLATFORM_VERSION import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes @@ -17,12 +16,10 @@ import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.nodeapi.internal.persistence.currentDBSession import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY import org.hibernate.annotations.Type -import org.slf4j.Logger -import org.slf4j.LoggerFactory import java.sql.Connection import java.sql.SQLException import java.time.Instant -import java.util.UUID +import java.util.* import java.util.stream.Stream import javax.persistence.CascadeType import javax.persistence.Column @@ -106,7 +103,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP var progressStep: String?, @Column(name = "flow_io_request") - var ioRequestType: Class>?, + var ioRequestType: String?, @Column(name = "timestamp", nullable = false) var checkpointInstant: Instant @@ -238,7 +235,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP } override fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? { - return currentDBSession().get(DBFlowCheckpoint::class.java, id.uuid.toString())?.toSerializedCheckpoint() + return getDBCheckpoint(id)?.toSerializedCheckpoint() } override fun getAllCheckpoints(): Stream> { @@ -251,6 +248,10 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP } } + private fun getDBCheckpoint(id: StateMachineRunId): DBFlowCheckpoint? { + return currentDBSession().find(DBFlowCheckpoint::class.java, id.uuid.toString()) + } + private fun createDBCheckpoint( id: StateMachineRunId, checkpoint: Checkpoint, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 99882aa022..d1f75dbcf3 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -62,7 +62,7 @@ data class Checkpoint( val result: Any? = null, val status: FlowStatus = FlowStatus.RUNNABLE, val progressStep: String? = null, - val flowIoRequest: Class>? = null, + val flowIoRequest: String? = null, val compatible: Boolean = true ) { @CordaSerializable @@ -149,7 +149,7 @@ data class Checkpoint( val result: SerializedBytes?, val status: FlowStatus, val progressStep: String?, - val flowIoRequest: Class>?, + val flowIoRequest: String?, val compatible: Boolean ) { /** diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index eaad3d99b5..00ff850d16 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -158,7 +158,8 @@ class TopLevelTransition( flowState = FlowState.Started(event.ioRequest, event.fiber), checkpointState = currentState.checkpoint.checkpointState.copy( numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends + 1 - ) + ), + flowIoRequest = event.ioRequest::class.java.simpleName ) if (event.maySkipCheckpoint) { actions.addAll(arrayOf( diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 08be3bbf3e..5b9c4d4c34 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -32,14 +32,14 @@ import net.corda.testing.node.MockServices.Companion.makeTestDataSourcePropertie import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.After +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull import org.junit.Before import org.junit.Rule import org.junit.Test import java.time.Instant import kotlin.streams.toList import kotlin.test.assertEquals -import kotlin.test.assertNotNull -import kotlin.test.assertNull import kotlin.test.assertTrue internal fun CheckpointStorage.checkpoints(): List { @@ -121,7 +121,7 @@ class DBCheckpointStorageTests { logic.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) ), progressStep = "I have made progress", - flowIoRequest = FlowIORequest.SendAndReceive::class.java + flowIoRequest = FlowIORequest.SendAndReceive::class.java.simpleName ) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { @@ -439,6 +439,30 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `Checkpoint can be updated with flow io request information`() { + val (id, checkpoint) = newCheckpoint(1) + database.transaction { + val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + val checkpointFromStorage = checkpointStorage.getCheckpoint(id) + assertNull(checkpointFromStorage!!.flowIoRequest) + } + database.transaction { + val newCheckpoint = checkpoint.copy(flowIoRequest = FlowIORequest.Sleep::class.java.simpleName) + val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( + context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT + ) + checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) + } + database.transaction { + val checkpointFromStorage = checkpointStorage.getCheckpoint(id) + assertNotNull(checkpointFromStorage!!.flowIoRequest) + val flowIORequest = checkpointFromStorage.flowIoRequest + assertEquals(FlowIORequest.Sleep::class.java.simpleName, flowIORequest) + } + } + private fun newCheckpointStorage() { database.transaction { checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index a4f77c5390..8477702db2 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -12,10 +12,12 @@ import net.corda.core.crypto.random63BitValue import net.corda.core.flows.* import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField +import net.corda.core.internal.FlowIORequest import net.corda.core.internal.concurrent.flatMap import net.corda.core.messaging.MessageRecipients import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy +import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.toFuture @@ -25,6 +27,8 @@ import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.ProgressTracker.Change import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap +import net.corda.node.services.persistence.CheckpointPerformanceRecorder +import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState @@ -69,6 +73,16 @@ class FlowFrameworkTests { private lateinit var notaryIdentity: Party private val receivedSessionMessages = ArrayList() + private val dbCheckpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }) + + @Before fun setUpMockNet() { mockNet = InternalMockNetwork( @@ -208,6 +222,19 @@ class FlowFrameworkTests { script(FlowMonitor(aliceNode.smm, Duration.ZERO, Duration.ZERO), FlowMonitor(bobNode.smm, Duration.ZERO, Duration.ZERO)) } + @Test(timeout = 300_000) + fun `flow status is updated in database when flow suspends on ioRequest`() { + val terminationSignal = Semaphore(0) + bobNode.registerCordappFlowFactory(ReceiveFlow::class) { NoOpFlow( terminateUponSignal = terminationSignal) } + val flowId = aliceNode.services.startFlow(ReceiveFlow(bob)).id + mockNet.runNetwork() + aliceNode.database.transaction { + val checkpoint = dbCheckpointStorage.getCheckpoint(flowId) + assertEquals(FlowIORequest.Receive::class.java.simpleName, checkpoint?.flowIoRequest) + } + terminationSignal.release() + } + @Test(timeout=300_000) fun `receiving unexpected session end before entering sendAndReceive`() { bobNode.registerCordappFlowFactory(WaitForOtherSideEndBeforeSendAndReceive::class) { NoOpFlow() } From 499b6cf17e220701f5255b340b0585e7a2ba968b Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Thu, 12 Mar 2020 11:45:30 +0000 Subject: [PATCH 23/49] CORDA-3603 Save completed flow information (#6034) When a flow is finished do not delete the checkpoint from the DB. Instead, the FlowStatus is marked as Completed in the DB. Updated numerous tests which relied on the flow being removed when finished. --- ...owCheckpointVersionNodeStartupCheckTest.kt | 4 +- .../StatemachineErrorHandlingTest.kt | 5 +- .../StatemachineFinalityErrorHandlingTest.kt | 16 ++--- .../StatemachineGeneralErrorHandlingTest.kt | 34 +++++------ .../StatemachineKillFlowErrorHandlingTest.kt | 6 +- .../StatemachineSubflowErrorHandlingTest.kt | 8 +-- .../net/corda/node/flows/FlowRetryTest.kt | 16 +++-- .../CordaPersistenceServiceTests.kt | 5 +- .../persistence/DBCheckpointStorage.kt | 7 ++- .../transitions/TopLevelTransition.kt | 11 ++-- .../node/messaging/TwoPartyTradeFlowTests.kt | 24 +++++--- .../persistence/DBCheckpointStorageTests.kt | 4 ++ .../statemachine/FlowFrameworkTests.kt | 60 +++++++++++++++---- .../net/corda/traderdemo/TraderDemoTest.kt | 4 +- .../node/internal/InternalTestUtils.kt | 6 +- 15 files changed, 132 insertions(+), 78 deletions(-) diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt index 1e60ce62fb..d6abe718f1 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt @@ -17,7 +17,7 @@ import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.NodeParameters import net.corda.testing.driver.driver import net.corda.testing.node.internal.ListenProcessDeathException -import net.corda.testing.node.internal.assertCheckpoints +import net.corda.testing.node.internal.assertUncompletedCheckpoints import net.corda.testing.node.internal.enclosedCordapp import org.assertj.core.api.Assertions.assertThat import org.junit.Test @@ -75,7 +75,7 @@ class FlowCheckpointVersionNodeStartupCheckTest { } private fun DriverDSL.assertBobFailsToStartWithLogMessage(logMessage: String) { - assertCheckpoints(BOB_NAME, 1) + assertUncompletedCheckpoints(BOB_NAME, 1) assertFailsWith(ListenProcessDeathException::class) { startNode(NodeParameters( diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt index d933625407..e7bda45134 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt @@ -110,9 +110,10 @@ abstract class StatemachineErrorHandlingTest { } @StartableByRPC - class GetNumberOfCheckpointsFlow : FlowLogic() { + class GetNumberOfUncompletedCheckpointsFlow : FlowLogic() { override fun call(): Long { - return serviceHub.jdbcSession().prepareStatement("select count(*) from node_checkpoints").use { ps -> + val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})" + return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> ps.executeQuery().use { rs -> rs.next() rs.getLong(1) diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt index 1855aa11c3..98e199afe2 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt @@ -89,9 +89,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, aliceClient.stateMachinesSnapshot().size) assertEquals(1, charlieClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -160,9 +160,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, aliceClient.stateMachinesSnapshot().size) assertEquals(1, charlieClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -252,9 +252,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, aliceClient.stateMachinesSnapshot().size) assertEquals(0, charlieClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -349,9 +349,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, aliceClient.stateMachinesSnapshot().size) assertEquals(1, charlieClient.stateMachinesSnapshot().size) // 1 for CashIssueAndPaymentFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } } \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt index 8308328827..8965be1db0 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt @@ -94,7 +94,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(1, aliceClient.stateMachinesSnapshot().size) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -172,7 +172,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -252,7 +252,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -337,7 +337,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -426,7 +426,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(1, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -527,7 +527,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -616,7 +616,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -714,7 +714,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(1, aliceClient.stateMachinesSnapshot().size) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -812,7 +812,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -898,7 +898,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(1, aliceClient.stateMachinesSnapshot().size) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -990,7 +990,7 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(1, aliceClient.stateMachinesSnapshot().size) // 1 for errored flow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -1079,9 +1079,9 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, aliceClient.stateMachinesSnapshot().size) assertEquals(0, charlieClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -1176,11 +1176,11 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, aliceClient.stateMachinesSnapshot().size) assertEquals(1, charlieClient.stateMachinesSnapshot().size) // 1 for the flow that is waiting for the errored counterparty flow to finish and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for GetNumberOfCheckpointsFlow // the checkpoint is not persisted since it kept failing the original checkpoint commit // the flow will recover since artemis will keep the events and replay them on node restart - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -1273,9 +1273,9 @@ class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, aliceClient.stateMachinesSnapshot().size) assertEquals(0, charlieClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } } \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt index 6e4f7bf2d8..1a2fe5e7e1 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt @@ -99,7 +99,7 @@ class StatemachineKillFlowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -186,7 +186,7 @@ class StatemachineKillFlowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -278,7 +278,7 @@ class StatemachineKillFlowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(1, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt index fd491eab97..161f3c4b39 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt @@ -128,7 +128,7 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -230,7 +230,7 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -324,7 +324,7 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } @@ -426,7 +426,7 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { assertEquals(0, observation) assertEquals(0, aliceClient.stateMachinesSnapshot().size) // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt index bfd95eb498..2a2eece9ec 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt @@ -4,19 +4,16 @@ import co.paralleluniverse.fibers.Suspendable import net.corda.client.rpc.CordaRPCClient import net.corda.client.rpc.CordaRPCClientConfiguration import net.corda.core.CordaRuntimeException -import net.corda.core.concurrent.CordaFuture import net.corda.core.flows.* import net.corda.core.identity.Party -import net.corda.core.internal.FlowAsyncOperation import net.corda.core.internal.IdempotentFlow -import net.corda.core.internal.concurrent.doneFuture -import net.corda.core.internal.executeAsync import net.corda.core.messaging.startFlow import net.corda.core.serialization.CordaSerializable import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap import net.corda.node.services.Permissions +import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.FlowTimeoutException import net.corda.node.services.statemachine.StaffedFlowHospital import net.corda.testing.core.ALICE_NAME @@ -146,7 +143,7 @@ class FlowRetryTest { } assertEquals(3, TransientConnectionFailureFlow.retryCount) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, it.proxy.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } } @@ -165,7 +162,7 @@ class FlowRetryTest { } assertEquals(3, WrappedTransientConnectionFailureFlow.retryCount) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, it.proxy.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(2, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } } @@ -184,7 +181,7 @@ class FlowRetryTest { } assertEquals(0, GeneralExternalFailureFlow.retryCount) // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(1, it.proxy.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + assertEquals(1, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) } } } @@ -461,9 +458,10 @@ class GeneralExternalFailureResponder(private val session: FlowSession) : FlowLo } @StartableByRPC -class GetNumberOfCheckpointsFlow : FlowLogic() { +class GetNumberOfUncompletedCheckpointsFlow : FlowLogic() { override fun call(): Long { - return serviceHub.jdbcSession().prepareStatement("select count(*) from node_checkpoints").use { ps -> + val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})" + return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> ps.executeQuery().use { rs -> rs.next() rs.getLong(1) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index 0d5572c3dc..475e4f1959 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -11,6 +11,7 @@ import net.corda.core.node.services.CordaService import net.corda.core.node.services.vault.SessionScope import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.utilities.getOrThrow +import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.Checkpoint.FlowStatus import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.testing.driver.DriverParameters @@ -37,7 +38,9 @@ class CordaPersistenceServiceTests { assertEquals(sampleSize, count) DriverManager.getConnection("jdbc:h2:tcp://localhost:$port/node", "sa", "").use { - val resultSet = it.createStatement().executeQuery("SELECT count(*) from ${NODE_DATABASE_PREFIX}checkpoints") + val resultSet = it.createStatement().executeQuery( + "SELECT count(*) from ${NODE_DATABASE_PREFIX}checkpoints where status not in (${FlowStatus.COMPLETED.ordinal})" + ) assertTrue(resultSet.next()) val resultSize = resultSet.getInt(1) assertEquals(sampleSize, resultSize) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index ae082b928a..fabbeb2d81 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -2,6 +2,7 @@ package net.corda.node.services.persistence import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.serialize @@ -328,12 +329,14 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) - val result = updateDBFlowResult(entity, checkpoint, now) + //This code needs to be added back in when we want to persist the result. For now this requires the result to be @CordaSerializable. + //val result = updateDBFlowResult(entity, checkpoint, now) val exceptionDetails = updateDBFlowException(entity, checkpoint, now) return entity.apply { this.blob = blob - this.result = result + //Set the result to null for now. + this.result = null this.exceptionDetails = exceptionDetails // Do not update the meta data relationship on updates this.flowMetadata = entity.flowMetadata diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 00ff850d16..85d7afb302 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -199,16 +199,17 @@ class TopLevelTransition( checkpoint = checkpoint.copy( checkpointState = checkpoint.checkpointState.copy( numberOfSuspends = checkpoint.checkpointState.numberOfSuspends + 1 - )), + ), + result = event.returnValue, + status = Checkpoint.FlowStatus.COMPLETED + ), pendingDeduplicationHandlers = emptyList(), isFlowResumed = false, isRemoved = true ) val allSourceSessionIds = checkpoint.checkpointState.sessions.keys - if (currentState.isAnyCheckpointPersisted) { - actions.add(Action.RemoveCheckpoint(context.id)) - } actions.addAll(arrayOf( + Action.PersistCheckpoint(context.id, currentState.checkpoint, currentState.isAnyCheckpointPersisted), Action.PersistDeduplicationFacts(pendingDeduplicationHandlers), Action.ReleaseSoftLocks(event.softLocksId), Action.CommitTransaction, @@ -289,4 +290,4 @@ class TopLevelTransition( FlowContinuation.Abort } } -} \ No newline at end of file +} diff --git a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt index e25308e5ec..a8016ed17a 100644 --- a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt +++ b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt @@ -32,9 +32,10 @@ import net.corda.finance.contracts.asset.CASH import net.corda.finance.contracts.asset.Cash import net.corda.finance.flows.TwoPartyTradeFlow.Buyer import net.corda.finance.flows.TwoPartyTradeFlow.Seller +import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.WritableTransactionStorage import net.corda.node.services.persistence.DBTransactionStorage -import net.corda.node.services.persistence.checkpoints +import net.corda.node.services.statemachine.Checkpoint import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.testing.core.* import net.corda.testing.dsl.LedgerDSL @@ -56,10 +57,17 @@ import java.io.ByteArrayOutputStream import java.util.* import java.util.jar.JarOutputStream import java.util.zip.ZipEntry +import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertTrue +internal fun CheckpointStorage.getAllIncompleteCheckpoints(): List { + return getAllCheckpoints().use { + it.map { it.second }.toList() + }.filter { it.status != Checkpoint.FlowStatus.COMPLETED } +} + /** * In this example, Alice wishes to sell her commercial paper to Bob in return for $1,000,000 and they wish to do * it on the ledger atomically. Therefore they must work together to build a transaction. @@ -135,11 +143,11 @@ class TwoPartyTradeFlowTests(private val anonymous: Boolean) { bobNode.dispose() aliceNode.database.transaction { - assertThat(aliceNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(aliceNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } aliceNode.internals.manuallyCloseDB() bobNode.database.transaction { - assertThat(bobNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(bobNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } bobNode.internals.manuallyCloseDB() } @@ -191,11 +199,11 @@ class TwoPartyTradeFlowTests(private val anonymous: Boolean) { bobNode.dispose() aliceNode.database.transaction { - assertThat(aliceNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(aliceNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } aliceNode.internals.manuallyCloseDB() bobNode.database.transaction { - assertThat(bobNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(bobNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } bobNode.internals.manuallyCloseDB() } @@ -245,7 +253,7 @@ class TwoPartyTradeFlowTests(private val anonymous: Boolean) { // OK, now Bob has sent the partial transaction back to Alice and is waiting for Alice's signature. bobNode.database.transaction { - assertThat(bobNode.internals.checkpointStorage.checkpoints()).hasSize(1) + assertThat(bobNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).hasSize(1) } val storage = bobNode.services.validatedTransactions @@ -278,10 +286,10 @@ class TwoPartyTradeFlowTests(private val anonymous: Boolean) { assertThat(bobNode.smm.findStateMachines(Buyer::class.java)).isEmpty() bobNode.database.transaction { - assertThat(bobNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(bobNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } aliceNode.database.transaction { - assertThat(aliceNode.internals.checkpointStorage.checkpoints()).isEmpty() + assertThat(aliceNode.internals.checkpointStorage.getAllIncompleteCheckpoints()).isEmpty() } bobNode.database.transaction { diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 5b9c4d4c34..83128be220 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -23,6 +23,7 @@ import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity import net.corda.testing.internal.LogHelper @@ -35,6 +36,7 @@ import org.junit.After import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Before +import org.junit.Ignore import org.junit.Rule import org.junit.Test import java.time.Instant @@ -270,6 +272,7 @@ class DBCheckpointStorageTests { } @Test(timeout = 300_000) + @Ignore fun `update checkpoint with result information creates new result database record`() { val result = "This is the result" val (id, checkpoint) = newCheckpoint() @@ -297,6 +300,7 @@ class DBCheckpointStorageTests { } @Test(timeout = 300_000) + @Ignore fun `update checkpoint with result information updates existing result database record`() { val result = "This is the result" val somehowThereIsANewResult = "Another result (which should not be possible!)" diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 8477702db2..54e863d76f 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -9,7 +9,15 @@ import net.corda.core.concurrent.CordaFuture import net.corda.core.contracts.ContractState import net.corda.core.crypto.SecureHash import net.corda.core.crypto.random63BitValue -import net.corda.core.flows.* +import net.corda.core.flows.Destination +import net.corda.core.flows.FinalityFlow +import net.corda.core.flows.FlowException +import net.corda.core.flows.FlowInfo +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.ReceiveFinalityFlow +import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField import net.corda.core.internal.FlowIORequest @@ -40,12 +48,20 @@ import net.corda.testing.flows.registerCordappFlowFactory import net.corda.testing.internal.LogHelper import net.corda.testing.node.InMemoryMessagingNetwork.MessageTransfer import net.corda.testing.node.InMemoryMessagingNetwork.ServicePeerAllocationStrategy.RoundRobin -import net.corda.testing.node.internal.* +import net.corda.testing.node.internal.DUMMY_CONTRACTS_CORDAPP +import net.corda.testing.node.internal.InternalMockNetwork +import net.corda.testing.node.internal.InternalMockNodeParameters +import net.corda.testing.node.internal.TestStartedNode +import net.corda.testing.node.internal.getMessage +import net.corda.testing.node.internal.startFlow import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatIllegalArgumentException import org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType import org.assertj.core.api.Condition import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNull import org.junit.Before import org.junit.Test import rx.Notification @@ -55,7 +71,6 @@ import java.time.Instant import java.util.* import java.util.function.Predicate import kotlin.reflect.KClass -import kotlin.test.assertEquals import kotlin.test.assertFailsWith class FlowFrameworkTests { @@ -73,15 +88,14 @@ class FlowFrameworkTests { private lateinit var notaryIdentity: Party private val receivedSessionMessages = ArrayList() - private val dbCheckpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { - override fun record( - serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes - ) { - // do nothing - } - }) - + private val dbCheckpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }) @Before fun setUpMockNet() { @@ -312,6 +326,26 @@ class FlowFrameworkTests { }, "FlowException's private peer field has value set")) } + //We should update this test when we do the work to persists the flow result. + @Test(timeout = 300_000) + fun `Flow status is set to completed in database when the flow finishes`() { + val terminationSignal = Semaphore(0) + val flow = aliceNode.services.startFlow(NoOpFlow( terminateUponSignal = terminationSignal)) + mockNet.waitQuiescent() // current thread needs to wait fiber running on a different thread, has reached the blocking point + aliceNode.database.transaction { + val checkpoint = dbCheckpointStorage.getCheckpoint(flow.id) + assertNull(checkpoint!!.result) + assertNotEquals(Checkpoint.FlowStatus.COMPLETED, checkpoint.status) + } + terminationSignal.release() + mockNet.waitQuiescent() + aliceNode.database.transaction { + val checkpoint = dbCheckpointStorage.getCheckpoint(flow.id) + assertNull(checkpoint!!.result) + assertEquals(Checkpoint.FlowStatus.COMPLETED, checkpoint.status) + } + } + private class ConditionalExceptionFlow(val otherPartySession: FlowSession, val sendPayload: Any) : FlowLogic() { @Suspendable override fun call() { @@ -915,4 +949,4 @@ internal class ExceptionFlow(val exception: () -> E) : FlowLogic< exceptionThrown = exception() throw exceptionThrown } -} +} \ No newline at end of file diff --git a/samples/trader-demo/src/integration-test/kotlin/net/corda/traderdemo/TraderDemoTest.kt b/samples/trader-demo/src/integration-test/kotlin/net/corda/traderdemo/TraderDemoTest.kt index e9f91f7cbe..46e954ca06 100644 --- a/samples/trader-demo/src/integration-test/kotlin/net/corda/traderdemo/TraderDemoTest.kt +++ b/samples/trader-demo/src/integration-test/kotlin/net/corda/traderdemo/TraderDemoTest.kt @@ -19,7 +19,7 @@ import net.corda.testing.driver.* import net.corda.testing.node.TestCordapp import net.corda.testing.node.User import net.corda.testing.node.internal.FINANCE_CORDAPPS -import net.corda.testing.node.internal.assertCheckpoints +import net.corda.testing.node.internal.assertUncompletedCheckpoints import net.corda.testing.node.internal.poll import net.corda.traderdemo.flow.CommercialPaperIssueFlow import net.corda.traderdemo.flow.SellerFlow @@ -100,7 +100,7 @@ class TraderDemoTest { val saleFuture = seller.rpc.startFlow(::SellerFlow, buyer.nodeInfo.singleIdentity(), 5.DOLLARS).returnValue buyer.rpc.stateMachinesFeed().updates.toBlocking().first() // wait until initiated flow starts buyer.stop() - assertCheckpoints(DUMMY_BANK_A_NAME, 1) + assertUncompletedCheckpoints(DUMMY_BANK_A_NAME, 1) val buyer2 = startNode(providedName = DUMMY_BANK_A_NAME, customOverrides = mapOf("p2pAddress" to buyer.p2pAddress.toString())).getOrThrow() saleFuture.getOrThrow() assertThat(buyer2.rpc.getCashBalance(USD)).isEqualTo(95.DOLLARS) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt index 633aa98b83..526ee339c2 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt @@ -23,6 +23,7 @@ import net.corda.core.utilities.millis import net.corda.core.utilities.seconds import net.corda.node.services.api.StartedNodeServices import net.corda.node.services.messaging.Message +import net.corda.node.services.statemachine.Checkpoint import net.corda.testing.driver.DriverDSL import net.corda.testing.driver.NodeHandle import net.corda.testing.internal.chooseIdentity @@ -273,9 +274,10 @@ fun CordaRPCOps.waitForShutdown(): Observable { return completable } -fun DriverDSL.assertCheckpoints(name: CordaX500Name, expected: Long) { +fun DriverDSL.assertUncompletedCheckpoints(name: CordaX500Name, expected: Long) { + val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})" DriverManager.getConnection("jdbc:h2:file:${baseDirectory(name) / "persistence"}", "sa", "").use { connection -> - connection.createStatement().executeQuery("select count(*) from NODE_CHECKPOINTS").use { rs -> + connection.createStatement().executeQuery(sqlStatement).use { rs -> rs.next() assertThat(rs.getLong(1)).isEqualTo(expected) } From 0174d996bd6ab86f0a1d06767e21d0bdf21e1e6c Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Thu, 12 Mar 2020 17:57:35 +0000 Subject: [PATCH 24/49] CORDA-3598 Set Checkpoint.status to RUNNABLE (#6019) * Set/ Reset Checkpoint.status to RUNNABLE after when suspending * Removing/ Moving comment as it makes no longer sense to be there since, we now always create a new Checkpoint object in SingleThreadedStateMachineManager.createFlowFromCheckpoint through tryDeserializeCheckpoint * Set -in memory- Checkpoint.status to RUNNABLE when a flow is retrying from Checkpoint --- .../SingleThreadedStateMachineManager.kt | 11 +- .../statemachine/StateMachineState.kt | 7 +- .../statemachine/FlowFrameworkTests.kt | 107 ++++++++++++++++++ 3 files changed, 116 insertions(+), 9 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index d722ad9fdb..c7f750fd31 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -604,7 +604,7 @@ class SingleThreadedStateMachineManager( // This is a brand new flow null } - val checkpoint = existingCheckpoint ?: Checkpoint.create( + val checkpoint = existingCheckpoint?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: Checkpoint.create( invocationContext, flowStart, flowLogic.javaClass, @@ -774,7 +774,7 @@ class SingleThreadedStateMachineManager( isStartIdempotent: Boolean, initialDeduplicationHandler: DeduplicationHandler? ): Flow? { - val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id) ?: return null + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id)?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: return null val flowState = checkpoint.flowState val resultFuture = openFuture() val fiber = when (flowState) { @@ -800,12 +800,7 @@ class SingleThreadedStateMachineManager( is FlowState.Started -> { val fiber = tryCheckpointDeserialize(flowState.frozenFiber, id) ?: return null val state = StateMachineState( - // Do a trivial checkpoint copy below, to update the Checkpoint#timestamp value. - // The Checkpoint#timestamp is being used by FlowMonitor as the starting time point of a potential suspension. - // We need to refresh the Checkpoint#timestamp here, in case of an e.g. node start up after a long period. - // If not then, there is a time window (until the next checkpoint update) in which the FlowMonitor - // could log this flow as a waiting flow, from the last checkpoint update i.e. before the node's start up. - checkpoint = checkpoint.copy(), + checkpoint = checkpoint, pendingDeduplicationHandlers = initialDeduplicationHandler?.let { listOf(it) } ?: emptyList(), isFlowResumed = false, isTransactionTracked = false, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index d1f75dbcf3..4e4a1414a4 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -75,7 +75,12 @@ data class Checkpoint( PAUSED } - val timestamp: Instant = Instant.now() // This will get updated every time a Checkpoint object is created/ created by copy. + /** + * [timestamp] will get updated every time a [Checkpoint] object is created/ created by copy. + * It will be updated, therefore, for example when a flow is being suspended or whenever a flow + * is being loaded from [Checkpoint] through [Serialized.deserialize]. + */ + val timestamp: Instant = Instant.now() companion object { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 54e863d76f..01bd6e76e6 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -20,6 +20,7 @@ import net.corda.core.flows.ReceiveFinalityFlow import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField +import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.FlowIORequest import net.corda.core.internal.concurrent.flatMap import net.corda.core.messaging.MessageRecipients @@ -27,6 +28,7 @@ import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize +import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.serialize import net.corda.core.toFuture import net.corda.core.transactions.SignedTransaction @@ -34,10 +36,14 @@ import net.corda.core.transactions.TransactionBuilder import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.ProgressTracker.Change import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds import net.corda.core.utilities.unwrap import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints +import net.corda.nodeapi.internal.persistence.contextDatabase +import net.corda.nodeapi.internal.persistence.contextTransaction +import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState import net.corda.testing.core.ALICE_NAME @@ -66,12 +72,16 @@ import org.junit.Before import org.junit.Test import rx.Notification import rx.Observable +import java.sql.SQLException import java.time.Duration import java.time.Instant import java.util.* import java.util.function.Predicate import kotlin.reflect.KClass +import kotlin.streams.toList +import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertTrue class FlowFrameworkTests { companion object { @@ -597,6 +607,88 @@ class FlowFrameworkTests { assertThat(result.getOrThrow()).isEqualTo("HelloHello") } + @Test(timeout=300_000) + fun `Checkpoint status changes to RUNNABLE when flow is loaded from checkpoint - FlowState Unstarted`() { + var firstExecution = true + var checkpointStatusInDBBeforeSuspension: Checkpoint.FlowStatus? = null + var checkpointStatusInDBAfterSuspension: Checkpoint.FlowStatus? = null + var checkpointStatusInMemoryBeforeSuspension: Checkpoint.FlowStatus? = null + + SuspendingFlow.hookBeforeCheckpoint = { + val flowFiber = this as? FlowStateMachineImpl<*> + assertTrue(flowFiber!!.transientState!!.value.checkpoint.flowState is FlowState.Unstarted) + + if (firstExecution) { + // the following manual persisting Checkpoint.status to FAILED should be removed when implementing CORDA-3604. + manuallyFailCheckpointInDB(aliceNode) + + firstExecution = false + throw SQLException("deadlock") // will cause flow to retry + } else { + // The persisted Checkpoint should be still failed here -> it should change to RUNNABLE after suspension + checkpointStatusInDBBeforeSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + checkpointStatusInMemoryBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status + } + } + + SuspendingFlow.hookAfterCheckpoint = { + checkpointStatusInDBAfterSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + } + + aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow() + + assertEquals(Checkpoint.FlowStatus.FAILED, checkpointStatusInDBBeforeSuspension) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInMemoryBeforeSuspension) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInDBAfterSuspension) + + SuspendingFlow.hookBeforeCheckpoint = {} + SuspendingFlow.hookAfterCheckpoint = {} + } + + @Test(timeout=300_000) + fun `Checkpoint status changes to RUNNABLE when flow is loaded from checkpoint - FlowState Started`() { + var firstExecution = true + var checkpointStatusInDB: Checkpoint.FlowStatus? = null + var checkpointStatusInMemory: Checkpoint.FlowStatus? = null + + SuspendingFlow.hookAfterCheckpoint = { + val flowFiber = this as? FlowStateMachineImpl<*> + assertTrue(flowFiber!!.transientState!!.value.checkpoint.flowState is FlowState.Started) + + if (firstExecution) { + // the following manual persisting Checkpoint.status to FAILED should be removed when implementing CORDA-3604. + manuallyFailCheckpointInDB(aliceNode) + + firstExecution = false + throw SQLException("deadlock") // will cause flow to retry + } else { + checkpointStatusInDB = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + checkpointStatusInMemory = flowFiber.transientState!!.value.checkpoint.status + } + } + + aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow() + + assertEquals(Checkpoint.FlowStatus.FAILED, checkpointStatusInDB) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInMemory) + + SuspendingFlow.hookAfterCheckpoint = {} + } + + // the following method should be removed when implementing CORDA-3604. + private fun manuallyFailCheckpointInDB(node: TestStartedNode) { + val idCheckpoint = node.internals.checkpointStorage.getAllCheckpoints().toList().single() + val checkpoint = idCheckpoint.second + val updatedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) + node.internals.checkpointStorage.updateCheckpoint(idCheckpoint.first, + updatedCheckpoint.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT), + updatedCheckpoint.serializedFlowState) + contextTransaction.commit() + contextTransaction.close() + contextTransactionOrNull = null + contextDatabase.newTransaction() + } + //region Helpers private val normalEnd = ExistingSessionMessage(SessionId(0), EndSessionMessage) // NormalSessionEnd(0) @@ -949,4 +1041,19 @@ internal class ExceptionFlow(val exception: () -> E) : FlowLogic< exceptionThrown = exception() throw exceptionThrown } +} + +internal class SuspendingFlow : FlowLogic() { + + companion object { + var hookBeforeCheckpoint: FlowStateMachine<*>.() -> Unit = {} + var hookAfterCheckpoint: FlowStateMachine<*>.() -> Unit = {} + } + + @Suspendable + override fun call() { + stateMachine.hookBeforeCheckpoint() + sleep(1.seconds) // flow checkpoints => checkpoint is in DB + stateMachine.hookAfterCheckpoint() + } } \ No newline at end of file From 1025ee1dee2ee642c2bf0527516a0172543fcf1c Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Mon, 16 Mar 2020 09:30:23 +0000 Subject: [PATCH 25/49] CORDA-3599 Add progress tracker information to checkpoint (#6063) * Add progress tracker information to checkpoint The checkpoint Datebase is updated when the statemachine suspends with the progress trackers current step name. This is truncated if it is longer than the Database column. * Minor rename in statemachine for clarity --- .../persistence/DBCheckpointStorage.kt | 4 ++- .../corda/node/services/statemachine/Event.kt | 6 +++- .../statemachine/FlowStateMachineImpl.kt | 3 +- .../transitions/TopLevelTransition.kt | 3 +- .../persistence/DBCheckpointStorageTests.kt | 28 ++++++++++++++++++- .../statemachine/FlowFrameworkTests.kt | 18 ++++++++++-- 6 files changed, 55 insertions(+), 7 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index fabbeb2d81..7f554edbf6 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -43,6 +43,8 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP private const val HMAC_SIZE_BYTES = 16 + private const val MAX_PROGRESS_STEP_LENGTH = 256 + /** * This needs to run before Hibernate is initialised. * @@ -342,7 +344,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP this.flowMetadata = entity.flowMetadata this.status = checkpoint.status this.compatible = checkpoint.compatible - this.progressStep = checkpoint.progressStep + this.progressStep = checkpoint.progressStep?.take(MAX_PROGRESS_STEP_LENGTH) this.ioRequestType = checkpoint.flowIoRequest this.checkpointInstant = now } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt index a7d15c87c8..d0f96925d2 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt @@ -6,6 +6,7 @@ import net.corda.core.identity.Party import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes import net.corda.core.transactions.SignedTransaction +import net.corda.core.utilities.ProgressTracker import net.corda.node.services.messaging.DeduplicationHandler import java.util.* @@ -101,17 +102,20 @@ sealed class Event { * @param ioRequest the request triggering the suspension. * @param maySkipCheckpoint indicates whether the persistence may be skipped. * @param fiber the serialised stack of the flow. + * @param progressStep the current progress tracker step. */ data class Suspend( val ioRequest: FlowIORequest<*>, val maySkipCheckpoint: Boolean, - val fiber: SerializedBytes> + val fiber: SerializedBytes>, + var progressStep: ProgressTracker.Step? ) : Event() { override fun toString() = "Suspend(" + "ioRequest=$ioRequest, " + "maySkipCheckpoint=$maySkipCheckpoint, " + "fiber=${fiber.hash}, " + + "currentStep=${progressStep?.label}" + ")" } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 9b0541f8cc..12b578ec37 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -430,7 +430,8 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, Event.Suspend( ioRequest = ioRequest, maySkipCheckpoint = skipPersistingCheckpoint, - fiber = this.checkpointSerialize(context = serializationContext.value) + fiber = this.checkpointSerialize(context = serializationContext.value), + progressStep = logic.progressTracker?.currentStep ) } catch (exception: Exception) { Event.Error(exception) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 85d7afb302..e20f3fc290 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -159,7 +159,8 @@ class TopLevelTransition( checkpointState = currentState.checkpoint.checkpointState.copy( numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends + 1 ), - flowIoRequest = event.ioRequest::class.java.simpleName + flowIoRequest = event.ioRequest::class.java.simpleName, + progressStep = event.progressStep?.label ) if (event.maySkipCheckpoint) { actions.addAll(arrayOf( diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 83128be220..d1da5967b0 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -23,7 +23,6 @@ import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.ALICE_NAME -import net.corda.testing.core.DUMMY_NOTARY_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity import net.corda.testing.internal.LogHelper @@ -467,6 +466,33 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `Checkpoint truncates long progressTracker step name`() { + val maxProgressStepLength = 256 + val (id, checkpoint) = newCheckpoint(1) + database.transaction { + val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + val checkpointFromStorage = checkpointStorage.getCheckpoint(id) + assertNull(checkpointFromStorage!!.progressStep) + } + val longString = """Long string Long string Long string Long string Long string Long string Long string Long string Long string + Long string Long string Long string Long string Long string Long string Long string Long string Long string Long string + Long string Long string Long string Long string Long string Long string Long string Long string Long string Long string + """.trimIndent() + database.transaction { + val newCheckpoint = checkpoint.copy(progressStep = longString) + val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( + context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT + ) + checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) + } + database.transaction { + val checkpointFromStorage = checkpointStorage.getCheckpoint(id) + assertEquals(longString.take(maxProgressStepLength), checkpointFromStorage!!.progressStep) + } + } + private fun newCheckpointStorage() { database.transaction { checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 01bd6e76e6..242eaa0dba 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -20,8 +20,8 @@ import net.corda.core.flows.ReceiveFinalityFlow import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField -import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.FlowIORequest +import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.concurrent.flatMap import net.corda.core.messaging.MessageRecipients import net.corda.core.node.services.PartyInfo @@ -79,7 +79,6 @@ import java.util.* import java.util.function.Predicate import kotlin.reflect.KClass import kotlin.streams.toList -import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertTrue @@ -356,6 +355,21 @@ class FlowFrameworkTests { } } + @Test(timeout = 300_000) + fun `Flow persists progress tracker in the database when the flow suspends`() { + bobNode.registerCordappFlowFactory(ReceiveFlow::class) { InitiatedReceiveFlow(it) } + val aliceFlowId = aliceNode.services.startFlow(ReceiveFlow(bob)).id + mockNet.runNetwork() + aliceNode.database.transaction { + val checkpoint = aliceNode.internals.checkpointStorage.getCheckpoint(aliceFlowId) + assertEquals(ReceiveFlow.START_STEP.label, checkpoint!!.progressStep) + } + bobNode.database.transaction { + val checkpoints = bobNode.internals.checkpointStorage.checkpoints().single() + assertEquals(InitiatedReceiveFlow.START_STEP.label, checkpoints.progressStep) + } + } + private class ConditionalExceptionFlow(val otherPartySession: FlowSession, val sendPayload: Any) : FlowLogic() { @Suspendable override fun call() { From 121c789c59bdc4310ec717e984225f2ac9e54654 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Mon, 16 Mar 2020 15:25:36 +0000 Subject: [PATCH 26/49] CORDA-3655 Do not run COMPLETED, FAILED or KILLED flows on node startup (#6070) * CheckpointStorage.getAllCheckpoints will not fetch COMPLETED, FAILED and KILLED flows by default * Rename getAllCheckpoints to getAllRunnableCheckpoints for clarity * Fix Detekt issue * Rename getAllRunnableCheckpoints to getRunnableCheckpoints * Minor kdoc update * Bring back in CheckpointStorage.getAllCheckpoints to co-exist with getRunnableCheckpoints --- .../corda/node/internal/CheckpointVerifier.kt | 3 +- .../node/services/api/CheckpointStorage.kt | 7 ++++- .../persistence/DBCheckpointStorage.kt | 15 ++++++++- .../node/services/rpc/CheckpointDumperImpl.kt | 2 +- .../SingleThreadedStateMachineManager.kt | 2 +- .../node/messaging/TwoPartyTradeFlowTests.kt | 2 +- .../persistence/DBCheckpointStorageTests.kt | 31 ++++++++++++++++++- .../statemachine/FlowFrameworkTests.kt | 4 +-- 8 files changed, 56 insertions(+), 10 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt index 340226492e..bebb451d74 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt @@ -5,7 +5,6 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.FlowLogic import net.corda.core.node.ServiceHub import net.corda.core.serialization.internal.CheckpointSerializationDefaults -import net.corda.core.serialization.internal.checkpointDeserialize import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.statemachine.SubFlow import net.corda.node.services.statemachine.SubFlowVersion @@ -36,7 +35,7 @@ object CheckpointVerifier { val cordappsByHash = currentCordapps.associateBy { it.jarHash } - checkpointStorage.getAllCheckpoints().use { + checkpointStorage.getRunnableCheckpoints().use { it.forEach { (_, serializedCheckpoint) -> val checkpoint = try { serializedCheckpoint.deserialize(checkpointSerializationContext) diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index 953c2fc562..c9d3ee0eb4 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -1,7 +1,6 @@ package net.corda.node.services.api import net.corda.core.flows.StateMachineRunId -import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.FlowState @@ -42,4 +41,10 @@ interface CheckpointStorage { * underlying database connection is closed, so any processing should happen before it is closed. */ fun getAllCheckpoints(): Stream> + + /** + * Stream runnable checkpoints from the store. If this is backed by a database the stream will be valid + * until the underlying database connection is closed, so any processing should happen before it is closed. + */ + fun getRunnableCheckpoints(): Stream> } diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 7f554edbf6..cd0a465706 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -2,7 +2,6 @@ package net.corda.node.services.persistence import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.PLATFORM_VERSION -import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.serialize @@ -45,6 +44,8 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP private const val MAX_PROGRESS_STEP_LENGTH = 256 + private val NOT_RUNNABLE_CHECKPOINTS = listOf(FlowStatus.COMPLETED, FlowStatus.FAILED, FlowStatus.KILLED) + /** * This needs to run before Hibernate is initialised. * @@ -251,6 +252,18 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP } } + override fun getRunnableCheckpoints(): Stream> { + val session = currentDBSession() + val criteriaBuilder = session.criteriaBuilder + val criteriaQuery = criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) + val root = criteriaQuery.from(DBFlowCheckpoint::class.java) + criteriaQuery.select(root) + .where(criteriaBuilder.not(root.get(DBFlowCheckpoint::status.name).`in`(NOT_RUNNABLE_CHECKPOINTS))) + return session.createQuery(criteriaQuery).stream().map { + StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() + } + } + private fun getDBCheckpoint(id: StateMachineRunId): DBFlowCheckpoint? { return currentDBSession().find(DBFlowCheckpoint::class.java, id.uuid.toString()) } diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index c3979612ab..200947538b 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -141,7 +141,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri try { if (lock.getAndIncrement() == 0 && !file.exists()) { database.transaction { - checkpointStorage.getAllCheckpoints().use { stream -> + checkpointStorage.getRunnableCheckpoints().use { stream -> ZipOutputStream(file.outputStream()).use { zip -> stream.forEach { (runId, serialisedCheckpoint) -> diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index c7f750fd31..910815a9d3 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -329,7 +329,7 @@ class SingleThreadedStateMachineManager( } private fun restoreFlowsFromCheckpoints(): List { - return checkpointStorage.getAllCheckpoints().use { + return checkpointStorage.getRunnableCheckpoints().use { it.mapNotNull { (id, serializedCheckpoint) -> // If a flow is added before start() then don't attempt to restore it mutex.locked { if (id in flows) return@mapNotNull null } diff --git a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt index a8016ed17a..789909f3ab 100644 --- a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt +++ b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt @@ -63,7 +63,7 @@ import kotlin.test.assertFailsWith import kotlin.test.assertTrue internal fun CheckpointStorage.getAllIncompleteCheckpoints(): List { - return getAllCheckpoints().use { + return getRunnableCheckpoints().use { it.map { it.second }.toList() }.filter { it.status != Checkpoint.FlowStatus.COMPLETED } } diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index d1da5967b0..726541775f 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -44,7 +44,7 @@ import kotlin.test.assertEquals import kotlin.test.assertTrue internal fun CheckpointStorage.checkpoints(): List { - return getAllCheckpoints().use { + return getRunnableCheckpoints().use { it.map { it.second }.toList() } } @@ -493,6 +493,35 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `fetch runnable checkpoints`() { + val (_, checkpoint) = newCheckpoint(1) + // runnables + val runnable = checkpoint.copy(status = Checkpoint.FlowStatus.RUNNABLE) + val hospitalized = checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) + // not runnables + val completed = checkpoint.copy(status = Checkpoint.FlowStatus.COMPLETED) + val failed = checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) + val killed = checkpoint.copy(status = Checkpoint.FlowStatus.KILLED) + // tentative + val paused = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) // is considered runnable + + database.transaction { + val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), runnable, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), hospitalized, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), completed, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), failed, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), killed, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), paused, serializedFlowState) + } + + database.transaction { + assertEquals(3, checkpointStorage.getRunnableCheckpoints().count()) + } + } + private fun newCheckpointStorage() { database.transaction { checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 242eaa0dba..8bb06ae51d 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -646,7 +646,7 @@ class FlowFrameworkTests { } SuspendingFlow.hookAfterCheckpoint = { - checkpointStatusInDBAfterSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + checkpointStatusInDBAfterSuspension = aliceNode.internals.checkpointStorage.getRunnableCheckpoints().toList().single().second.status } aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow() @@ -691,7 +691,7 @@ class FlowFrameworkTests { // the following method should be removed when implementing CORDA-3604. private fun manuallyFailCheckpointInDB(node: TestStartedNode) { - val idCheckpoint = node.internals.checkpointStorage.getAllCheckpoints().toList().single() + val idCheckpoint = node.internals.checkpointStorage.getRunnableCheckpoints().toList().single() val checkpoint = idCheckpoint.second val updatedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) node.internals.checkpointStorage.updateCheckpoint(idCheckpoint.first, From ca23612fe1c81bece20d3ceb54360d9b1b0c75bc Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Tue, 17 Mar 2020 17:28:32 +0000 Subject: [PATCH 27/49] CORDA-3596 Record flow metadata (#6067) * CORDA-3596 Record flow metadata Record flow metadata during the zero'th checkpoint that occurs before calling the flow's `call` function. This required adding an RPC call's arguments to the `InvocationContext` that gets created. These arguments are then accessible within the statemachine and from the `Checkpoint` class. The arguments are then extracted when recording a flow's metadata inside of `DBCheckpointStorage`. Updated the size of the started by column to 128 since it was not long enough to hold the fully qualified class of a service that started a flow. * CORDA-3596 Remove arguments from in-memory checkpoint When executing a flows first real suspend (from flow code) the arguments contained in the `InvocationContext` are removed. This saves holding these arguments for the whole lifecyle of a flow. * CORDA-3596 Increase `cordapp_name` column to 128 * CORDA-3596 Join metadata by `flow_id` Due to changes in where metadata is recorded, there is no need for having `invocation_id` as the metadata table's primary key. The `flow_id` is now the primary key of the table and is used to join to the main checkpoints table. The `invocation_id` has been removed from the checkpoints table since it is not needed for the join anymore. * CORDA-3596 Remove `received_time` from metadata table * CORDA-3596 Remove unused `StartReason` enum * CORDA-3596 Simple `DBCheckpointStorageTests` for metadata * CORDA-3596 Truncate really long flow names --- .../corda/core/context/InvocationContext.kt | 55 +- .../CordaPersistenceServiceTests.kt | 17 +- .../persistence/DBCheckpointStorage.kt | 118 ++-- .../net/corda/node/services/rpc/RPCServer.kt | 11 +- .../transitions/TopLevelTransition.kt | 18 +- .../node-core.changelog-v17-keys.xml | 6 +- .../node-core.changelog-v17-postgres.xml | 18 +- .../migration/node-core.changelog-v17.xml | 18 +- .../persistence/DBCheckpointStorageTests.kt | 80 +-- .../statemachine/FlowMetadataRecordingTest.kt | 512 ++++++++++++++++++ 10 files changed, 733 insertions(+), 120 deletions(-) create mode 100644 node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt diff --git a/core/src/main/kotlin/net/corda/core/context/InvocationContext.kt b/core/src/main/kotlin/net/corda/core/context/InvocationContext.kt index b9f66ca423..ef90810b05 100644 --- a/core/src/main/kotlin/net/corda/core/context/InvocationContext.kt +++ b/core/src/main/kotlin/net/corda/core/context/InvocationContext.kt @@ -18,21 +18,53 @@ import java.security.Principal * @property impersonatedActor Optional impersonated actor, used for logging but not for authorisation. */ @CordaSerializable -data class InvocationContext(val origin: InvocationOrigin, val trace: Trace, val actor: Actor?, val externalTrace: Trace? = null, val impersonatedActor: Actor? = null) { +data class InvocationContext( + val origin: InvocationOrigin, + val trace: Trace, + val actor: Actor?, + val externalTrace: Trace? = null, + val impersonatedActor: Actor? = null, + val arguments: List = emptyList() +) { + + constructor( + origin: InvocationOrigin, + trace: Trace, + actor: Actor?, + externalTrace: Trace? = null, + impersonatedActor: Actor? = null + ) : this(origin, trace, actor, externalTrace, impersonatedActor, emptyList()) + companion object { /** * Creates an [InvocationContext] with a [Trace] that defaults to a [java.util.UUID] as value and [java.time.Instant.now] timestamp. */ @DeleteForDJVM @JvmStatic - fun newInstance(origin: InvocationOrigin, trace: Trace = Trace.newInstance(), actor: Actor? = null, externalTrace: Trace? = null, impersonatedActor: Actor? = null) = InvocationContext(origin, trace, actor, externalTrace, impersonatedActor) + @JvmOverloads + @Suppress("LongParameterList") + fun newInstance( + origin: InvocationOrigin, + trace: Trace = Trace.newInstance(), + actor: Actor? = null, + externalTrace: Trace? = null, + impersonatedActor: Actor? = null, + arguments: List = emptyList() + ) = InvocationContext(origin, trace, actor, externalTrace, impersonatedActor, arguments) /** * Creates an [InvocationContext] with [InvocationOrigin.RPC] origin. */ @DeleteForDJVM @JvmStatic - fun rpc(actor: Actor, trace: Trace = Trace.newInstance(), externalTrace: Trace? = null, impersonatedActor: Actor? = null): InvocationContext = newInstance(InvocationOrigin.RPC(actor), trace, actor, externalTrace, impersonatedActor) + @JvmOverloads + fun rpc( + actor: Actor, + trace: Trace = Trace.newInstance(), + externalTrace: Trace? = null, + impersonatedActor: Actor? = null, + arguments: List = emptyList() + ): InvocationContext = newInstance(InvocationOrigin.RPC(actor), trace, actor, externalTrace, impersonatedActor, arguments) /** * Creates an [InvocationContext] with [InvocationOrigin.Peer] origin. @@ -67,6 +99,23 @@ data class InvocationContext(val origin: InvocationOrigin, val trace: Trace, val * Associated security principal. */ fun principal(): Principal = origin.principal() + + fun copy( + origin: InvocationOrigin = this.origin, + trace: Trace = this.trace, + actor: Actor? = this.actor, + externalTrace: Trace? = this.externalTrace, + impersonatedActor: Actor? = this.impersonatedActor + ): InvocationContext { + return copy( + origin = origin, + trace = trace, + actor = actor, + externalTrace = externalTrace, + impersonatedActor = impersonatedActor, + arguments = arguments + ) + } } /** diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index 475e4f1959..b5c455ce8e 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -11,7 +11,6 @@ import net.corda.core.node.services.CordaService import net.corda.core.node.services.vault.SessionScope import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.utilities.getOrThrow -import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.Checkpoint.FlowStatus import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.testing.driver.DriverParameters @@ -63,9 +62,10 @@ class CordaPersistenceServiceTests { (1..count).toList().parallelStream().forEach { val now = Instant.now() services.database.transaction { + val flowId = it.toString() session.save( DBCheckpointStorage.DBFlowCheckpoint( - id = it.toString(), + id = flowId, blob = DBCheckpointStorage.DBFlowCheckpointBlob( checkpoint = ByteArray(8192), flowStack = ByteArray(8192), @@ -78,8 +78,8 @@ class CordaPersistenceServiceTests { compatible = false, progressStep = "", ioRequestType = FlowIORequest.ForceCheckpoint::class.java.simpleName, - checkpointInstant = Instant.now(), - flowMetadata = createMetadataRecord(UUID.randomUUID(), now) + checkpointInstant = now, + flowMetadata = createMetadataRecord(flowId, now) ) ) } @@ -88,18 +88,17 @@ class CordaPersistenceServiceTests { return count } - private fun SessionScope.createMetadataRecord(invocationId: UUID, timestamp: Instant): DBCheckpointStorage.DBFlowMetadata { + private fun SessionScope.createMetadataRecord(flowId: String, timestamp: Instant): DBCheckpointStorage.DBFlowMetadata { val metadata = DBCheckpointStorage.DBFlowMetadata( - invocationId = invocationId.toString(), - flowId = null, + invocationId = UUID.randomUUID().toString(), + flowId = flowId, flowName = "random.flow", userSuppliedIdentifier = null, startType = DBCheckpointStorage.StartReason.RPC, launchingCordapp = "this cordapp", platformVersion = PLATFORM_VERSION, rpcUsername = "Batman", - invocationInstant = Instant.now(), - receivedInstant = Instant.now(), + invocationInstant = timestamp, startInstant = timestamp, finishInstant = null ) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index cd0a465706..8cc1d5e193 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -1,7 +1,10 @@ package net.corda.node.services.persistence +import net.corda.core.context.InvocationContext +import net.corda.core.context.InvocationOrigin import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.internal.uncheckedCast import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.serialize @@ -12,6 +15,7 @@ import net.corda.node.services.statemachine.Checkpoint.FlowStatus import net.corda.node.services.statemachine.CheckpointState import net.corda.node.services.statemachine.ErrorState import net.corda.node.services.statemachine.FlowState +import net.corda.node.services.statemachine.SubFlowVersion import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.nodeapi.internal.persistence.currentDBSession import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY @@ -43,6 +47,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP private const val HMAC_SIZE_BYTES = 16 private const val MAX_PROGRESS_STEP_LENGTH = 256 + private const val MAX_FLOW_NAME_LENGTH = 128 private val NOT_RUNNABLE_CHECKPOINTS = listOf(FlowStatus.COMPLETED, FlowStatus.FAILED, FlowStatus.KILLED) @@ -71,7 +76,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP } enum class StartReason { - RPC, FLOW, SERVICE, SCHEDULED, INITIATED + RPC, SERVICE, SCHEDULED, INITIATED } @Entity @@ -94,7 +99,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP var exceptionDetails: DBFlowException?, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "invocation_id", referencedColumnName = "invocation_id") + @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") var flowMetadata: DBFlowMetadata, @Column(name = "status", nullable = false) @@ -180,12 +185,12 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP class DBFlowMetadata( @Id + @Column(name = "flow_id", nullable = false) + var flowId: String, + @Column(name = "invocation_id", nullable = false) var invocationId: String, - @Column(name = "flow_id", nullable = true) - var flowId: String?, - @Column(name = "flow_name", nullable = false) var flowName: String, @@ -210,16 +215,51 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @Column(name = "invocation_time", nullable = false) var invocationInstant: Instant, - @Column(name = "received_time", nullable = false) - var receivedInstant: Instant, - @Column(name = "start_time", nullable = true) - var startInstant: Instant?, + var startInstant: Instant, @Column(name = "finish_time", nullable = true) var finishInstant: Instant? + ) { + @Suppress("ComplexMethod") + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false - ) + other as DBFlowMetadata + + if (flowId != other.flowId) return false + if (invocationId != other.invocationId) return false + if (flowName != other.flowName) return false + if (userSuppliedIdentifier != other.userSuppliedIdentifier) return false + if (startType != other.startType) return false + if (!initialParameters.contentEquals(other.initialParameters)) return false + if (launchingCordapp != other.launchingCordapp) return false + if (platformVersion != other.platformVersion) return false + if (rpcUsername != other.rpcUsername) return false + if (invocationInstant != other.invocationInstant) return false + if (startInstant != other.startInstant) return false + if (finishInstant != other.finishInstant) return false + + return true + } + + override fun hashCode(): Int { + var result = flowId.hashCode() + result = 31 * result + invocationId.hashCode() + result = 31 * result + flowName.hashCode() + result = 31 * result + (userSuppliedIdentifier?.hashCode() ?: 0) + result = 31 * result + startType.hashCode() + result = 31 * result + initialParameters.contentHashCode() + result = 31 * result + launchingCordapp.hashCode() + result = 31 * result + platformVersion + result = 31 * result + rpcUsername.hashCode() + result = 31 * result + invocationInstant.hashCode() + result = 31 * result + startInstant.hashCode() + result = 31 * result + (finishInstant?.hashCode() ?: 0) + return result + } + } override fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) { currentDBSession().save(createDBCheckpoint(id, checkpoint, serializedFlowState)) @@ -275,25 +315,13 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP ): DBFlowCheckpoint { val flowId = id.uuid.toString() val now = Instant.now() - val invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) - // Need to update the metadata record to join it to the main checkpoint record - // This code needs to be added back in once the metadata record is properly created (remove the code below it) - // val metadata = requireNotNull(currentDBSession().find( - // DBFlowMetadata::class.java, - // invocationId - // )) { "The flow metadata record for flow [$flowId] with invocation id [$invocationId] does not exist"} - val metadata = (currentDBSession().find( - DBFlowMetadata::class.java, - invocationId - )) ?: createTemporaryMetadata(checkpoint) - metadata.flowId = flowId - currentDBSession().update(metadata) + val metadata = createMetadata(flowId, checkpoint) // Most fields are null as they cannot have been set when creating the initial checkpoint return DBFlowCheckpoint( id = flowId, @@ -309,20 +337,24 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP ) } - // Remove this when saving of metadata is properly handled - private fun createTemporaryMetadata(checkpoint: Checkpoint): DBFlowMetadata { + private fun createMetadata(flowId: String, checkpoint: Checkpoint): DBFlowMetadata { + val context = checkpoint.checkpointState.invocationContext + val flowInfo = checkpoint.checkpointState.subFlowStack.first() return DBFlowMetadata( - invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value, - flowId = null, - flowName = "random.flow", + flowId = flowId, + invocationId = context.trace.invocationId.value, + // Truncate the flow name to fit into the database column + // Flow names are unlikely to be this long + flowName = flowInfo.flowClass.name.take(MAX_FLOW_NAME_LENGTH), + // will come from the context userSuppliedIdentifier = null, - startType = DBCheckpointStorage.StartReason.RPC, - launchingCordapp = "this cordapp", + startType = context.getStartedType(), + initialParameters = context.getFlowParameters().storageSerialize().bytes, + launchingCordapp = (flowInfo.subFlowVersion as? SubFlowVersion.CorDappFlow)?.corDappName ?: "Core flow", platformVersion = PLATFORM_VERSION, - rpcUsername = "Batman", - invocationInstant = checkpoint.checkpointState.invocationContext.trace.invocationId.timestamp, - receivedInstant = Instant.now(), - startInstant = null, + rpcUsername = context.principal().name, + invocationInstant = context.trace.invocationId.timestamp, + startInstant = Instant.now(), finishInstant = null ).apply { currentDBSession().save(this) @@ -444,6 +476,24 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP } } + private fun InvocationContext.getStartedType(): StartReason { + return when (origin) { + is InvocationOrigin.RPC, is InvocationOrigin.Shell -> StartReason.RPC + is InvocationOrigin.Peer -> StartReason.INITIATED + is InvocationOrigin.Service -> StartReason.SERVICE + is InvocationOrigin.Scheduled -> StartReason.SCHEDULED + } + } + + private fun InvocationContext.getFlowParameters(): List { + // Only RPC flows have parameters which are found in index 1 + return if(arguments.isNotEmpty()) { + uncheckedCast>(arguments[1]).toList() + } else { + emptyList() + } + } + private fun DBFlowCheckpoint.toSerializedCheckpoint(): Checkpoint.Serialized { return Checkpoint.Serialized( serializedCheckpointState = SerializedBytes(blob.checkpoint), diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/RPCServer.kt b/node/src/main/kotlin/net/corda/node/services/rpc/RPCServer.kt index 9a1e1474d0..63503f8f28 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/RPCServer.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/RPCServer.kt @@ -388,10 +388,11 @@ class RPCServer( val arguments = Try.on { clientToServer.serialisedArguments.deserialize>(context = RPC_SERVER_CONTEXT) } - val context = artemisMessage.context(clientToServer.sessionId) - context.invocation.pushToLoggingContext() + val context: RpcAuthContext when (arguments) { is Try.Success -> { + context = artemisMessage.context(clientToServer.sessionId, arguments.value) + context.invocation.pushToLoggingContext() log.debug { "Arguments: ${arguments.value.toTypedArray().contentDeepToString()}" } rpcExecutor!!.submit { val result = invokeRpc(context, clientToServer.methodName, arguments.value) @@ -399,6 +400,8 @@ class RPCServer( } } is Try.Failure -> { + context = artemisMessage.context(clientToServer.sessionId, emptyList()) + context.invocation.pushToLoggingContext() // We failed to deserialise the arguments, route back the error log.warn("Inbound RPC failed", arguments.exception) sendReply(clientToServer.replyId, clientToServer.clientAddress, arguments) @@ -476,12 +479,12 @@ class RPCServer( observableMap.cleanUp() } - private fun ClientMessage.context(sessionId: Trace.SessionId): RpcAuthContext { + private fun ClientMessage.context(sessionId: Trace.SessionId, arguments: List): RpcAuthContext { val trace = Trace.newInstance(sessionId = sessionId) val externalTrace = externalTrace() val rpcActor = actorFrom(this) val impersonatedActor = impersonatedActor() - return RpcAuthContext(InvocationContext.rpc(rpcActor.first, trace, externalTrace, impersonatedActor), rpcActor.second) + return RpcAuthContext(InvocationContext.rpc(rpcActor.first, trace, externalTrace, impersonatedActor, arguments), rpcActor.second) } private fun actorFrom(message: ClientMessage): Pair { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index e20f3fc290..ca92f11883 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -154,14 +154,22 @@ class TopLevelTransition( private fun suspendTransition(event: Event.Suspend): TransitionResult { return builder { - val newCheckpoint = currentState.checkpoint.copy( + val newCheckpoint = currentState.checkpoint.run { + val newCheckpointState = if (checkpointState.invocationContext.arguments.isNotEmpty()) { + checkpointState.copy( + invocationContext = checkpointState.invocationContext.copy(arguments = emptyList()), + numberOfSuspends = checkpointState.numberOfSuspends + 1 + ) + } else { + checkpointState.copy(numberOfSuspends = checkpointState.numberOfSuspends + 1) + } + copy( flowState = FlowState.Started(event.ioRequest, event.fiber), - checkpointState = currentState.checkpoint.checkpointState.copy( - numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends + 1 - ), + checkpointState = newCheckpointState, flowIoRequest = event.ioRequest::class.java.simpleName, progressStep = event.progressStep?.label - ) + ) + } if (event.maySkipCheckpoint) { actions.addAll(arrayOf( Action.CommitTransaction, diff --git a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml index 482c4d6418..a8e82b3966 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml @@ -9,7 +9,7 @@ - + @@ -26,9 +26,9 @@ constraintName="node_checkpoint_to_result_fk" referencedColumnNames="id" referencedTableName="node_flow_results"/> - + referencedColumnNames="flow_id" referencedTableName="node_flow_metadata"/> diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index acb3557060..15009cd24f 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -19,9 +19,6 @@ - - - @@ -98,11 +95,11 @@ - + - - + + @@ -116,23 +113,20 @@ - + - + - - - - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 266361124e..bdd2727657 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -19,9 +19,6 @@ - - - @@ -98,11 +95,11 @@ - + - - + + @@ -116,23 +113,20 @@ - + - + - - - - + diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 726541775f..f181f74ce0 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -1,10 +1,10 @@ package net.corda.node.services.persistence import net.corda.core.context.InvocationContext +import net.corda.core.context.InvocationOrigin import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.FlowIORequest -import net.corda.core.internal.PLATFORM_VERSION import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize @@ -21,7 +21,6 @@ import net.corda.node.services.statemachine.SubFlowVersion import net.corda.node.services.transactions.PersistentUniquenessProvider import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig -import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity @@ -38,7 +37,6 @@ import org.junit.Before import org.junit.Ignore import org.junit.Rule import org.junit.Test -import java.time.Instant import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -79,7 +77,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { @@ -107,7 +104,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val logic: FlowLogic<*> = object : FlowLogic() { @@ -141,7 +137,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } database.transaction { @@ -163,8 +158,6 @@ class DBCheckpointStorageTests { val (id2, checkpoint2) = newCheckpoint() val serializedFlowState2 = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) - createMetadataRecord(checkpoint2) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) checkpointStorage.addCheckpoint(id2, checkpoint2, serializedFlowState2) checkpointStorage.removeCheckpoint(id) @@ -190,13 +183,11 @@ class DBCheckpointStorageTests { val serializedFirstFlowState = firstCheckpoint.serializeFlowState() database.transaction { - createMetadataRecord(firstCheckpoint) checkpointStorage.addCheckpoint(id, firstCheckpoint, serializedFirstFlowState) } val (id2, secondCheckpoint) = newCheckpoint() val serializedSecondFlowState = secondCheckpoint.serializeFlowState() database.transaction { - createMetadataRecord(secondCheckpoint) checkpointStorage.addCheckpoint(id2, secondCheckpoint, serializedSecondFlowState) } database.transaction { @@ -222,7 +213,6 @@ class DBCheckpointStorageTests { val (id, originalCheckpoint) = newCheckpoint() val serializedOriginalFlowState = originalCheckpoint.serializeFlowState() database.transaction { - createMetadataRecord(originalCheckpoint) checkpointStorage.addCheckpoint(id, originalCheckpoint, serializedOriginalFlowState) } newCheckpointStorage() @@ -242,13 +232,52 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `adding a new checkpoint creates a metadata record`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + database.transaction { + session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()).also { + assertNotNull(it) + } + } + } + + @Test(timeout = 300_000) + fun `updating a checkpoint does not change the metadata record`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val metadata = database.transaction { + session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()).also { + assertNotNull(it) + } + } + val updatedCheckpoint = checkpoint.copy( + checkpointState = checkpoint.checkpointState.copy( + invocationContext = InvocationContext.newInstance(InvocationOrigin.Peer(ALICE_NAME)) + ) + ) + database.transaction { + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, serializedFlowState) + } + val potentiallyUpdatedMetadata = database.transaction { + session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()) + } + assertEquals(metadata, potentiallyUpdatedMetadata) + } + @Test(timeout = 300_000) fun `verify checkpoints compatible`() { val mockServices = MockServices(emptyList(), ALICE.name) database.transaction { val (id, checkpoint) = newCheckpoint(1) val serializedFlowState = checkpoint.serializeFlowState() - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } @@ -259,7 +288,6 @@ class DBCheckpointStorageTests { database.transaction { val (id1, checkpoint1) = newCheckpoint(2) val serializedFlowState1 = checkpoint1.serializeFlowState() - createMetadataRecord(checkpoint1) checkpointStorage.addCheckpoint(id1, checkpoint1, serializedFlowState1) } @@ -278,7 +306,6 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.copy(result = result) @@ -307,7 +334,6 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.copy(result = result) @@ -339,7 +365,6 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.copy(result = result) @@ -367,7 +392,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.addError(exception) @@ -393,7 +417,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint1 = checkpoint.addError(illegalStateException) @@ -421,7 +444,6 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - createMetadataRecord(checkpoint) checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } val updatedCheckpoint = checkpoint.addError(exception) @@ -454,7 +476,7 @@ class DBCheckpointStorageTests { database.transaction { val newCheckpoint = checkpoint.copy(flowIoRequest = FlowIORequest.Sleep::class.java.simpleName) val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( - context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT + context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT ) checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) } @@ -574,22 +596,4 @@ class DBCheckpointStorageTests { ) ) } - - private fun DatabaseTransaction.createMetadataRecord(checkpoint: Checkpoint) { - val metadata = DBCheckpointStorage.DBFlowMetadata( - invocationId = checkpoint.checkpointState.invocationContext.trace.invocationId.value, - flowId = null, - flowName = "random.flow", - userSuppliedIdentifier = null, - startType = DBCheckpointStorage.StartReason.RPC, - launchingCordapp = "this cordapp", - platformVersion = PLATFORM_VERSION, - rpcUsername = "Batman", - invocationInstant = checkpoint.checkpointState.invocationContext.trace.invocationId.timestamp, - receivedInstant = Instant.now(), - startInstant = null, - finishInstant = null - ) - session.save(metadata) - } } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt new file mode 100644 index 0000000000..2326807e9f --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -0,0 +1,512 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.client.rpc.CordaRPCClient +import net.corda.core.context.InvocationContext +import net.corda.core.contracts.BelongsToContract +import net.corda.core.contracts.LinearState +import net.corda.core.contracts.SchedulableState +import net.corda.core.contracts.ScheduledActivity +import net.corda.core.contracts.StateRef +import net.corda.core.contracts.UniqueIdentifier +import net.corda.core.flows.FlowExternalAsyncOperation +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowLogicRefFactory +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.SchedulableFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.flows.StartableByService +import net.corda.core.flows.StateMachineRunId +import net.corda.core.identity.Party +import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.internal.uncheckedCast +import net.corda.core.messaging.startFlow +import net.corda.core.node.AppServiceHub +import net.corda.core.node.services.CordaService +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializationDefaults +import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.serialization.deserialize +import net.corda.core.transactions.TransactionBuilder +import net.corda.core.utilities.getOrThrow +import net.corda.node.services.Permissions +import net.corda.node.services.persistence.DBCheckpointStorage +import net.corda.testing.contracts.DummyContract +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.User +import org.assertj.core.api.Assertions.assertThat +import org.junit.Before +import org.junit.Test +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.concurrent.CompletableFuture +import java.util.concurrent.Executors +import java.util.concurrent.Semaphore +import java.util.function.Supplier +import kotlin.reflect.jvm.jvmName +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class FlowMetadataRecordingTest { + + private val user = User("mark", "dadada", setOf(Permissions.all())) + private val string = "I must be delivered for 4.5" + private val someObject = SomeObject("Store me in the database please", 1234) + + @Before + fun before() { + MyFlow.hookAfterInitialCheckpoint = null + MyFlow.hookAfterSuspend = null + MyResponder.hookAfterInitialCheckpoint = null + MyFlowWithoutParameters.hookAfterInitialCheckpoint = null + } + + @Test(timeout = 300_000) + fun `rpc started flows have metadata recorded`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + var flowId: StateMachineRunId? = null + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + context = contextFromHook + metadata = metadataFromHook + } + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + metadata!!.let { + assertEquals(context!!.trace.invocationId.value, it.invocationId) + assertEquals(flowId!!.uuid.toString(), it.flowId) + assertEquals(MyFlow::class.java.name, it.flowName) + // Should be changed when [userSuppliedIdentifier] gets filled in future changes + assertNull(it.userSuppliedIdentifier) + assertEquals(DBCheckpointStorage.StartReason.RPC, it.startType) + assertEquals( + listOf(nodeBHandle.nodeInfo.singleIdentity(), string, someObject), + it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + assertThat(it.launchingCordapp).contains("custom-cordapp") + assertEquals(PLATFORM_VERSION, it.platformVersion) + assertEquals(user.username, it.rpcUsername) + assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) + assertTrue(it.startInstant >= it.invocationInstant) + assertNull(it.finishInstant) + } + } + } + + @Test(timeout = 300_000) + fun `rpc started flows have metadata recorded - no parameters`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + + var flowId: StateMachineRunId? = null + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlowWithoutParameters.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + context = contextFromHook + metadata = metadataFromHook + } + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::MyFlowWithoutParameters).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + metadata!!.let { + assertEquals(context!!.trace.invocationId.value, it.invocationId) + assertEquals(flowId!!.uuid.toString(), it.flowId) + assertEquals(MyFlowWithoutParameters::class.java.name, it.flowName) + // Should be changed when [userSuppliedIdentifier] gets filled in future changes + assertNull(it.userSuppliedIdentifier) + assertEquals(DBCheckpointStorage.StartReason.RPC, it.startType) + assertEquals( + emptyList(), + it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + assertThat(it.launchingCordapp).contains("custom-cordapp") + assertEquals(PLATFORM_VERSION, it.platformVersion) + assertEquals(user.username, it.rpcUsername) + assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) + assertTrue(it.startInstant >= it.invocationInstant) + assertNull(it.finishInstant) + } + } + } + + @Test(timeout = 300_000) + fun `rpc started flows have their arguments removed from in-memory checkpoint after zero'th checkpoint`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterInitialCheckpoint = + { _, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + context = contextFromHook + metadata = metadataFromHook + } + + var context2: InvocationContext? = null + var metadata2: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterSuspend = + { contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + context2 = contextFromHook + metadata2 = metadataFromHook + } + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + assertEquals( + listOf(nodeBHandle.nodeInfo.singleIdentity(), string, someObject), + uncheckedCast>(context!!.arguments[1]).toList() + ) + assertEquals( + listOf(nodeBHandle.nodeInfo.singleIdentity(), string, someObject), + metadata!!.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + + assertEquals( + emptyList(), + context2!!.arguments + ) + assertEquals( + listOf(nodeBHandle.nodeInfo.singleIdentity(), string, someObject), + metadata2!!.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + } + } + + @Test(timeout = 300_000) + fun `initiated flows have metadata recorded`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + var flowId: StateMachineRunId? = null + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyResponder.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + context = contextFromHook + metadata = metadataFromHook + } + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + metadata!!.let { + assertEquals(context!!.trace.invocationId.value, it.invocationId) + assertEquals(flowId!!.uuid.toString(), it.flowId) + assertEquals(MyResponder::class.java.name, it.flowName) + assertNull(it.userSuppliedIdentifier) + assertEquals(DBCheckpointStorage.StartReason.INITIATED, it.startType) + assertEquals( + emptyList(), + it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + assertThat(it.launchingCordapp).contains("custom-cordapp") + assertEquals(6, it.platformVersion) + assertEquals(nodeAHandle.nodeInfo.singleIdentity().name.toString(), it.rpcUsername) + assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) + assertTrue(it.startInstant >= it.invocationInstant) + assertNull(it.finishInstant) + } + } + } + + @Test(timeout = 300_000) + fun `service started flows have metadata recorded`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + var flowId: StateMachineRunId? = null + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + context = contextFromHook + metadata = metadataFromHook + } + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyServiceStartingFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + metadata!!.let { + assertEquals(context!!.trace.invocationId.value, it.invocationId) + assertEquals(flowId!!.uuid.toString(), it.flowId) + assertEquals(MyFlow::class.java.name, it.flowName) + assertNull(it.userSuppliedIdentifier) + assertEquals(DBCheckpointStorage.StartReason.SERVICE, it.startType) + assertEquals( + emptyList(), + it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + assertThat(it.launchingCordapp).contains("custom-cordapp") + assertEquals(PLATFORM_VERSION, it.platformVersion) + assertEquals(MyService::class.java.name, it.rpcUsername) + assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) + assertTrue(it.startInstant >= it.invocationInstant) + assertNull(it.finishInstant) + } + } + } + + @Test(timeout = 300_000) + fun `scheduled flows have metadata recorded`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + val lock = Semaphore(1) + + var flowId: StateMachineRunId? = null + var context: InvocationContext? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, contextFromHook: InvocationContext, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + context = contextFromHook + metadata = metadataFromHook + // Release the lock so the asserts can be processed + lock.release() + } + + // Acquire the lock to prevent the asserts from being processed too early + lock.acquire() + + CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyStartedScheduledFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + } + + // Block here until released in the hook + lock.acquire() + + metadata!!.let { + assertEquals(context!!.trace.invocationId.value, it.invocationId) + assertEquals(flowId!!.uuid.toString(), it.flowId) + assertEquals(MyFlow::class.java.name, it.flowName) + assertNull(it.userSuppliedIdentifier) + assertEquals(DBCheckpointStorage.StartReason.SCHEDULED, it.startType) + assertEquals( + emptyList(), + it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) + ) + assertThat(it.launchingCordapp).contains("custom-cordapp") + assertEquals(PLATFORM_VERSION, it.platformVersion) + assertEquals("Scheduler", it.rpcUsername) + assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) + assertTrue(it.startInstant >= it.invocationInstant) + assertNull(it.finishInstant) + } + } + } + + @InitiatingFlow + @StartableByRPC + @StartableByService + @SchedulableFlow + @Suppress("UNUSED_PARAMETER") + class MyFlow(private val party: Party, string: String, someObject: SomeObject) : + FlowLogic() { + + companion object { + var hookAfterInitialCheckpoint: (( + flowId: StateMachineRunId, + context: InvocationContext, + metadata: DBCheckpointStorage.DBFlowMetadata + ) -> Unit)? = null + var hookAfterSuspend: (( + context: InvocationContext, + metadata: DBCheckpointStorage.DBFlowMetadata + ) -> Unit)? = null + } + + @Suspendable + override fun call() { + hookAfterInitialCheckpoint?.let { + it( + stateMachine.id, + stateMachine.context, + serviceHub.cordaService(MyService::class.java).findMetadata(stateMachine.id) + ) + } + initiateFlow(party).sendAndReceive("Hello there") + hookAfterSuspend?.let { + it( + stateMachine.context, + serviceHub.cordaService(MyService::class.java).findMetadata(stateMachine.id) + ) + } + } + } + + @InitiatedBy(MyFlow::class) + class MyResponder(private val session: FlowSession) : FlowLogic() { + + companion object { + var hookAfterInitialCheckpoint: (( + flowId: StateMachineRunId, + context: InvocationContext, + metadata: DBCheckpointStorage.DBFlowMetadata + ) -> Unit)? = null + } + + @Suspendable + override fun call() { + session.receive() + hookAfterInitialCheckpoint?.let { + it( + stateMachine.id, + stateMachine.context, + serviceHub.cordaService(MyService::class.java).findMetadata(stateMachine.id) + ) + } + session.send("Hello there") + } + } + + @StartableByRPC + class MyFlowWithoutParameters : FlowLogic() { + + companion object { + var hookAfterInitialCheckpoint: (( + flowId: StateMachineRunId, + context: InvocationContext, + metadata: DBCheckpointStorage.DBFlowMetadata + ) -> Unit)? = null + } + + @Suspendable + override fun call() { + hookAfterInitialCheckpoint?.let { + it( + stateMachine.id, + stateMachine.context, + serviceHub.cordaService(MyService::class.java).findMetadata(stateMachine.id) + ) + } + } + } + + @StartableByRPC + class MyServiceStartingFlow(private val party: Party, private val string: String, private val someObject: SomeObject) : + FlowLogic() { + + @Suspendable + override fun call() { + await(object : FlowExternalAsyncOperation { + override fun execute(deduplicationId: String): CompletableFuture { + return serviceHub.cordaService(MyService::class.java).startFlow(party, string, someObject) + } + }) + } + } + + @StartableByRPC + class MyStartedScheduledFlow(private val party: Party, private val string: String, private val someObject: SomeObject) : + FlowLogic() { + + @Suspendable + override fun call() { + val tx = TransactionBuilder(serviceHub.networkMapCache.notaryIdentities.first()).apply { + addOutputState(ScheduledState(party, string, someObject, listOf(ourIdentity))) + addCommand(DummyContract.Commands.Create(), ourIdentity.owningKey) + } + val stx = serviceHub.signInitialTransaction(tx) + serviceHub.recordTransactions(stx) + } + } + + @CordaService + class MyService(private val services: AppServiceHub) : SingletonSerializeAsToken() { + + private val executorService = Executors.newFixedThreadPool(1) + + fun findMetadata(flowId: StateMachineRunId): DBCheckpointStorage.DBFlowMetadata { + return services.database.transaction { + session.find(DBCheckpointStorage.DBFlowMetadata::class.java, flowId.uuid.toString()) + } + } + + fun startFlow(party: Party, string: String, someObject: SomeObject): CompletableFuture { + return CompletableFuture.supplyAsync( + Supplier { services.startFlow(MyFlow(party, string, someObject)).returnValue.getOrThrow() }, + executorService + ) + } + } + + @CordaSerializable + data class SomeObject(private val string: String, private val number: Int) + + @BelongsToContract(DummyContract::class) + data class ScheduledState( + val party: Party, + val string: String, + val someObject: SomeObject, + override val participants: List, + override val linearId: UniqueIdentifier = UniqueIdentifier() + ) : SchedulableState, LinearState { + + override fun nextScheduledActivity(thisStateRef: StateRef, flowLogicRefFactory: FlowLogicRefFactory): ScheduledActivity? { + val logicRef = flowLogicRefFactory.create(MyFlow::class.jvmName, party, string, someObject) + return ScheduledActivity(logicRef, Instant.now()) + } + } +} \ No newline at end of file From 1c7126afb7d1cc3d4f985ab1150cbb868b6322f8 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Thu, 19 Mar 2020 16:07:00 +0000 Subject: [PATCH 28/49] NOTICK Rename `rpc_user` column to `started_by` in metadata table (#6081) --- .../persistence/CordaPersistenceServiceTests.kt | 2 +- .../node/services/persistence/DBCheckpointStorage.kt | 10 +++++----- .../migration/node-core.changelog-v17-postgres.xml | 2 +- .../resources/migration/node-core.changelog-v17.xml | 2 +- .../services/statemachine/FlowMetadataRecordingTest.kt | 10 +++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index b5c455ce8e..25e7f3af95 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -97,7 +97,7 @@ class CordaPersistenceServiceTests { startType = DBCheckpointStorage.StartReason.RPC, launchingCordapp = "this cordapp", platformVersion = PLATFORM_VERSION, - rpcUsername = "Batman", + startedBy = "Batman", invocationInstant = timestamp, startInstant = timestamp, finishInstant = null diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 8cc1d5e193..176c21a640 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -209,8 +209,8 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP @Column(name = "platform_version", nullable = false) var platformVersion: Int, - @Column(name = "rpc_user", nullable = false) - var rpcUsername: String, + @Column(name = "started_by", nullable = false) + var startedBy: String, @Column(name = "invocation_time", nullable = false) var invocationInstant: Instant, @@ -236,7 +236,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP if (!initialParameters.contentEquals(other.initialParameters)) return false if (launchingCordapp != other.launchingCordapp) return false if (platformVersion != other.platformVersion) return false - if (rpcUsername != other.rpcUsername) return false + if (startedBy != other.startedBy) return false if (invocationInstant != other.invocationInstant) return false if (startInstant != other.startInstant) return false if (finishInstant != other.finishInstant) return false @@ -253,7 +253,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP result = 31 * result + initialParameters.contentHashCode() result = 31 * result + launchingCordapp.hashCode() result = 31 * result + platformVersion - result = 31 * result + rpcUsername.hashCode() + result = 31 * result + startedBy.hashCode() result = 31 * result + invocationInstant.hashCode() result = 31 * result + startInstant.hashCode() result = 31 * result + (finishInstant?.hashCode() ?: 0) @@ -352,7 +352,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP initialParameters = context.getFlowParameters().storageSerialize().bytes, launchingCordapp = (flowInfo.subFlowVersion as? SubFlowVersion.CorDappFlow)?.corDappName ?: "Core flow", platformVersion = PLATFORM_VERSION, - rpcUsername = context.principal().name, + startedBy = context.principal().name, invocationInstant = context.trace.invocationId.timestamp, startInstant = Instant.now(), finishInstant = null diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 15009cd24f..8f558f3b81 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -119,7 +119,7 @@ - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index bdd2727657..513cdd4428 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -119,7 +119,7 @@ - + diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index 2326807e9f..3a203c334b 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -108,7 +108,7 @@ class FlowMetadataRecordingTest { ) assertThat(it.launchingCordapp).contains("custom-cordapp") assertEquals(PLATFORM_VERSION, it.platformVersion) - assertEquals(user.username, it.rpcUsername) + assertEquals(user.username, it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) assertNull(it.finishInstant) @@ -149,7 +149,7 @@ class FlowMetadataRecordingTest { ) assertThat(it.launchingCordapp).contains("custom-cordapp") assertEquals(PLATFORM_VERSION, it.platformVersion) - assertEquals(user.username, it.rpcUsername) + assertEquals(user.username, it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) assertNull(it.finishInstant) @@ -247,7 +247,7 @@ class FlowMetadataRecordingTest { ) assertThat(it.launchingCordapp).contains("custom-cordapp") assertEquals(6, it.platformVersion) - assertEquals(nodeAHandle.nodeInfo.singleIdentity().name.toString(), it.rpcUsername) + assertEquals(nodeAHandle.nodeInfo.singleIdentity().name.toString(), it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) assertNull(it.finishInstant) @@ -293,7 +293,7 @@ class FlowMetadataRecordingTest { ) assertThat(it.launchingCordapp).contains("custom-cordapp") assertEquals(PLATFORM_VERSION, it.platformVersion) - assertEquals(MyService::class.java.name, it.rpcUsername) + assertEquals(MyService::class.java.name, it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) assertNull(it.finishInstant) @@ -349,7 +349,7 @@ class FlowMetadataRecordingTest { ) assertThat(it.launchingCordapp).contains("custom-cordapp") assertEquals(PLATFORM_VERSION, it.platformVersion) - assertEquals("Scheduler", it.rpcUsername) + assertEquals("Scheduler", it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) assertNull(it.finishInstant) From 79b36aea8f8a67bd035e960542aaab8a7818e30c Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Wed, 25 Mar 2020 13:47:00 +0000 Subject: [PATCH 29/49] CORDA-3601 Record a flow's finish time (#6079) * CORDA-3601 Record a flow's finish time Record a flow's finish time by updating its metadata record. It is set in `updateCheckpoint` by checking the status of the checkpoint. If it is `COMPLETED` it will set the `finishInstant` on the metadata object and update it. * CORDA-3601 Record flow finish time for all finished statuses Update the flow finish time for the following statuses: - COMPLETED - KILLED - FAILED * CORDA-3601 Use platform clock in `DBCheckpointStorage` --- .../net/corda/node/internal/AbstractNode.kt | 2 +- .../persistence/DBCheckpointStorage.kt | 28 ++++++-- .../persistence/DBCheckpointStorageTests.kt | 20 +++--- .../services/rpc/CheckpointDumperImplTest.kt | 19 +++--- .../statemachine/FlowFrameworkTests.kt | 35 ++++++++-- .../statemachine/FlowMetadataRecordingTest.kt | 65 +++++++++++++++---- 6 files changed, 128 insertions(+), 41 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 222c579c9f..97c4585cb2 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -328,7 +328,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, }) } val services = ServiceHubInternalImpl().tokenize() - val checkpointStorage = DBCheckpointStorage(DBCheckpointPerformanceRecorder(services.monitoringService.metrics)) + val checkpointStorage = DBCheckpointStorage(DBCheckpointPerformanceRecorder(services.monitoringService.metrics), platformClock) @Suppress("LeakingThis") val smm = makeStateMachineManager() val flowStarter = FlowStarterImpl(smm, flowLogicRefFactory) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 176c21a640..9ed60c6a84 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -22,6 +22,7 @@ import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY import org.hibernate.annotations.Type import java.sql.Connection import java.sql.SQLException +import java.time.Clock import java.time.Instant import java.util.* import java.util.stream.Stream @@ -39,7 +40,10 @@ import javax.persistence.OneToOne * Simple checkpoint key value storage in DB. */ @Suppress("TooManyFunctions") -class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointPerformanceRecorder) : CheckpointStorage { +class DBCheckpointStorage( + private val checkpointPerformanceRecorder: CheckpointPerformanceRecorder, + private val clock: Clock +) : CheckpointStorage { companion object { val log = contextLogger() @@ -314,7 +318,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP serializedFlowState: SerializedBytes ): DBFlowCheckpoint { val flowId = id.uuid.toString() - val now = Instant.now() + val now = clock.instant() val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) @@ -333,7 +337,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP compatible = checkpoint.compatible, progressStep = null, ioRequestType = null, - checkpointInstant = Instant.now() + checkpointInstant = now ) } @@ -354,7 +358,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP platformVersion = PLATFORM_VERSION, startedBy = context.principal().name, invocationInstant = context.trace.invocationId.timestamp, - startInstant = Instant.now(), + startInstant = clock.instant(), finishInstant = null ).apply { currentDBSession().save(this) @@ -367,7 +371,7 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP serializedFlowState: SerializedBytes ): DBFlowCheckpoint { val flowId = id.uuid.toString() - val now = Instant.now() + val now = clock.instant() // Load the previous entity from the hibernate cache so the meta data join does not get updated val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) @@ -380,13 +384,20 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP //val result = updateDBFlowResult(entity, checkpoint, now) val exceptionDetails = updateDBFlowException(entity, checkpoint, now) + val metadata = entity.flowMetadata.apply { + if (checkpoint.isFinished() && finishInstant == null) { + finishInstant = now + currentDBSession().update(this) + } + } + return entity.apply { this.blob = blob //Set the result to null for now. this.result = null this.exceptionDetails = exceptionDetails // Do not update the meta data relationship on updates - this.flowMetadata = entity.flowMetadata + this.flowMetadata = metadata this.status = checkpoint.status this.compatible = checkpoint.compatible this.progressStep = checkpoint.progressStep?.take(MAX_PROGRESS_STEP_LENGTH) @@ -512,4 +523,9 @@ class DBCheckpointStorage(private val checkpointPerformanceRecorder: CheckpointP private fun T.storageSerialize(): SerializedBytes { return serialize(context = SerializationDefaults.STORAGE_CONTEXT) } + + private fun Checkpoint.isFinished() = when(status) { + FlowStatus.COMPLETED, FlowStatus.KILLED, FlowStatus.FAILED -> true + else -> false + } } diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index f181f74ce0..f180f70f50 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -37,6 +37,7 @@ import org.junit.Before import org.junit.Ignore import org.junit.Rule import org.junit.Test +import java.time.Clock import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -546,14 +547,17 @@ class DBCheckpointStorageTests { private fun newCheckpointStorage() { database.transaction { - checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { - override fun record( - serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes - ) { - // do nothing - } - }) + checkpointStorage = DBCheckpointStorage( + object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }, + Clock.systemUTC() + ) } } diff --git a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt index 14c4586ee6..d8c20a6c2c 100644 --- a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt @@ -143,14 +143,17 @@ class CheckpointDumperImplTest { private fun newCheckpointStorage() { database.transaction { - checkpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { - override fun record( - serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes - ) { - // do nothing - } - }) + checkpointStorage = DBCheckpointStorage( + object : CheckpointPerformanceRecorder { + override fun record( + serializedCheckpointState: SerializedBytes, + serializedFlowState: SerializedBytes + ) { + // do nothing + } + }, + Clock.systemUTC() + ) } } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index a58ae9b4e8..2ffbae8e22 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -73,6 +73,7 @@ import org.junit.Test import rx.Notification import rx.Observable import java.sql.SQLException +import java.time.Clock import java.time.Duration import java.time.Instant import java.util.* @@ -80,6 +81,7 @@ import java.util.function.Predicate import kotlin.reflect.KClass import kotlin.streams.toList import kotlin.test.assertFailsWith +import kotlin.test.assertNotNull import kotlin.test.assertTrue class FlowFrameworkTests { @@ -97,14 +99,17 @@ class FlowFrameworkTests { private lateinit var notaryIdentity: Party private val receivedSessionMessages = ArrayList() - private val dbCheckpointStorage = DBCheckpointStorage(object : CheckpointPerformanceRecorder { - override fun record( + private val dbCheckpointStorage = DBCheckpointStorage( + object : CheckpointPerformanceRecorder { + override fun record( serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes - ) { - // do nothing - } - }) + ) { + // do nothing + } + }, + Clock.systemUTC() + ) @Before fun setUpMockNet() { @@ -355,6 +360,24 @@ class FlowFrameworkTests { } } + @Test(timeout = 300_000) + fun `Flow metadata finish time is set in database when the flow finishes`() { + val terminationSignal = Semaphore(0) + val flow = aliceNode.services.startFlow(NoOpFlow(terminateUponSignal = terminationSignal)) + mockNet.waitQuiescent() + aliceNode.database.transaction { + val metadata = session.find(DBCheckpointStorage.DBFlowMetadata::class.java, flow.id.uuid.toString()) + assertNull(metadata.finishInstant) + } + terminationSignal.release() + mockNet.waitQuiescent() + aliceNode.database.transaction { + val metadata = session.find(DBCheckpointStorage.DBFlowMetadata::class.java, flow.id.uuid.toString()) + assertNotNull(metadata.finishInstant) + assertTrue(metadata.finishInstant!! >= metadata.startInstant) + } + } + @Test(timeout = 300_000) fun `Flow persists progress tracker in the database when the flow suspends`() { bobNode.registerCordappFlowFactory(ReceiveFlow::class) { InitiatedReceiveFlow(it) } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index 3a203c334b..2825a05b9f 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -31,6 +31,7 @@ import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.serialization.deserialize import net.corda.core.transactions.TransactionBuilder import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.minutes import net.corda.node.services.Permissions import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.testing.contracts.DummyContract @@ -43,15 +44,14 @@ import net.corda.testing.node.User import org.assertj.core.api.Assertions.assertThat import org.junit.Before import org.junit.Test -import java.time.Duration import java.time.Instant -import java.time.temporal.ChronoUnit import java.util.concurrent.CompletableFuture import java.util.concurrent.Executors import java.util.concurrent.Semaphore import java.util.function.Supplier import kotlin.reflect.jvm.jvmName import kotlin.test.assertEquals +import kotlin.test.assertNotNull import kotlin.test.assertNull import kotlin.test.assertTrue @@ -92,7 +92,7 @@ class FlowMetadataRecordingTest { nodeBHandle.nodeInfo.singleIdentity(), string, someObject - ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + ).returnValue.getOrThrow(1.minutes) } metadata!!.let { @@ -133,7 +133,7 @@ class FlowMetadataRecordingTest { } CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { - it.proxy.startFlow(::MyFlowWithoutParameters).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + it.proxy.startFlow(::MyFlowWithoutParameters).returnValue.getOrThrow(1.minutes) } metadata!!.let { @@ -186,7 +186,7 @@ class FlowMetadataRecordingTest { nodeBHandle.nodeInfo.singleIdentity(), string, someObject - ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + ).returnValue.getOrThrow(1.minutes) } assertEquals( @@ -232,7 +232,7 @@ class FlowMetadataRecordingTest { nodeBHandle.nodeInfo.singleIdentity(), string, someObject - ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + ).returnValue.getOrThrow(1.minutes) } metadata!!.let { @@ -278,7 +278,7 @@ class FlowMetadataRecordingTest { nodeBHandle.nodeInfo.singleIdentity(), string, someObject - ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + ).returnValue.getOrThrow(1.minutes) } metadata!!.let { @@ -308,7 +308,7 @@ class FlowMetadataRecordingTest { val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() - val lock = Semaphore(1) + val lock = Semaphore(0) var flowId: StateMachineRunId? = null var context: InvocationContext? = null @@ -322,16 +322,13 @@ class FlowMetadataRecordingTest { lock.release() } - // Acquire the lock to prevent the asserts from being processed too early - lock.acquire() - CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { it.proxy.startFlow( ::MyStartedScheduledFlow, nodeBHandle.nodeInfo.singleIdentity(), string, someObject - ).returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) + ).returnValue.getOrThrow(1.minutes) } // Block here until released in the hook @@ -357,6 +354,42 @@ class FlowMetadataRecordingTest { } } + @Test(timeout = 300_000) + fun `flows have their finish time recorded when completed`() { + driver(DriverParameters(startNodesInProcess = true)) { + + val nodeAHandle = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val nodeBHandle = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() + + var flowId: StateMachineRunId? = null + var metadata: DBCheckpointStorage.DBFlowMetadata? = null + MyFlow.hookAfterInitialCheckpoint = + { flowIdFromHook: StateMachineRunId, _, metadataFromHook: DBCheckpointStorage.DBFlowMetadata -> + flowId = flowIdFromHook + metadata = metadataFromHook + } + + val finishTime = CordaRPCClient(nodeAHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow( + ::MyFlow, + nodeBHandle.nodeInfo.singleIdentity(), + string, + someObject + ).returnValue.getOrThrow(1.minutes) + it.proxy.startFlow( + ::GetFlowFinishTimeFlow, + flowId!! + ).returnValue.getOrThrow(1.minutes) + } + + metadata!!.let { + assertNull(it.finishInstant) + assertNotNull(finishTime) + assertTrue(finishTime!! >= it.startInstant) + } + } + } + @InitiatingFlow @StartableByRPC @StartableByService @@ -473,6 +506,14 @@ class FlowMetadataRecordingTest { } } + @StartableByRPC + class GetFlowFinishTimeFlow(private val flowId: StateMachineRunId) : FlowLogic() { + @Suspendable + override fun call(): Instant? { + return serviceHub.cordaService(MyService::class.java).findMetadata(flowId).finishInstant + } + } + @CordaService class MyService(private val services: AppServiceHub) : SingletonSerializeAsToken() { From 024d63147dfccf53bd89002efdb67e590199cab4 Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Mon, 30 Mar 2020 16:56:03 +0100 Subject: [PATCH 30/49] CORDA-3491 Remove the flow state when a flow finishes (#6083) Added a new field Completed to the in-memory object FlowState. FlowState.Completed is corresponds to flow_state=Null in the DB. This change will save disk space. --- .../node/services/api/CheckpointStorage.kt | 2 +- .../DBCheckpointPerformanceRecorder.kt | 15 +++++-- .../persistence/DBCheckpointStorage.kt | 16 +++---- .../node/services/rpc/CheckpointDumperImpl.kt | 3 ++ .../statemachine/ActionExecutorImpl.kt | 11 ++++- .../SingleThreadedStateMachineManager.kt | 31 ++++++++------ .../statemachine/StateMachineState.kt | 10 ++++- .../transitions/DoRemainingWorkTransition.kt | 9 ++-- .../transitions/TopLevelTransition.kt | 1 + .../node-core.changelog-v17-postgres.xml | 2 +- .../migration/node-core.changelog-v17.xml | 2 +- .../persistence/DBCheckpointStorageTests.kt | 25 ++++++++++- .../services/rpc/CheckpointDumperImplTest.kt | 42 +++++++++++++++++-- .../statemachine/FlowFrameworkTests.kt | 7 +++- 14 files changed, 134 insertions(+), 42 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index c9d3ee0eb4..59aa3f6300 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -18,7 +18,7 @@ interface CheckpointStorage { /** * Update an existing checkpoint. Will throw if there is not checkpoint for this id. */ - fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) + fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?) /** * Remove existing checkpoint from the store. diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt index 7bc30ec804..d1d713f96a 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointPerformanceRecorder.kt @@ -17,7 +17,7 @@ interface CheckpointPerformanceRecorder { /** * Record performance metrics regarding the serialized size of [CheckpointState] and [FlowState] */ - fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes) + fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes?) } class DBCheckpointPerformanceRecorder(metrics: MetricRegistry) : CheckpointPerformanceRecorder { @@ -44,8 +44,15 @@ class DBCheckpointPerformanceRecorder(metrics: MetricRegistry) : CheckpointPerfo } } - override fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes) { - val totalSize = serializedCheckpointState.size.toLong() + serializedFlowState.size.toLong() + override fun record(serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes?) { + /* For now we don't record states where the serializedFlowState is null and thus the checkpoint is in a completed state. + As this will skew the mean with lots of small checkpoints. For the moment we only measure runnable checkpoints. */ + serializedFlowState?.let { + updateData(serializedCheckpointState.size.toLong() + it.size.toLong()) + } + } + + private fun updateData(totalSize: Long) { checkpointingMeter.mark() checkpointSizesThisSecond.update(totalSize) var lastUpdateTime = lastBandwidthUpdate.get() @@ -57,4 +64,4 @@ class DBCheckpointPerformanceRecorder(metrics: MetricRegistry) : CheckpointPerfo lastUpdateTime = lastBandwidthUpdate.get() } } -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 9ed60c6a84..1d48243f57 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -134,10 +134,9 @@ class DBCheckpointStorage( @Column(name = "checkpoint_value", nullable = false) var checkpoint: ByteArray = EMPTY_BYTE_ARRAY, - // A future task will make this nullable @Type(type = "corda-blob") - @Column(name = "flow_state", nullable = false) - var flowStack: ByteArray = EMPTY_BYTE_ARRAY, + @Column(name = "flow_state") + var flowStack: ByteArray?, @Column(name = "hmac") var hmac: ByteArray, @@ -269,7 +268,7 @@ class DBCheckpointStorage( currentDBSession().save(createDBCheckpoint(id, checkpoint, serializedFlowState)) } - override fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) { + override fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?) { currentDBSession().update(updateDBCheckpoint(id, checkpoint, serializedFlowState)) } @@ -368,7 +367,7 @@ class DBCheckpointStorage( private fun updateDBCheckpoint( id: StateMachineRunId, checkpoint: Checkpoint, - serializedFlowState: SerializedBytes + serializedFlowState: SerializedBytes? ): DBFlowCheckpoint { val flowId = id.uuid.toString() val now = clock.instant() @@ -408,12 +407,12 @@ class DBCheckpointStorage( private fun createDBCheckpointBlob( serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes, + serializedFlowState: SerializedBytes?, now: Instant ): DBFlowCheckpointBlob { return DBFlowCheckpointBlob( checkpoint = serializedCheckpointState.bytes, - flowStack = serializedFlowState.bytes, + flowStack = serializedFlowState?.bytes, hmac = ByteArray(HMAC_SIZE_BYTES), persistedInstant = now ) @@ -506,9 +505,10 @@ class DBCheckpointStorage( } private fun DBFlowCheckpoint.toSerializedCheckpoint(): Checkpoint.Serialized { + val serialisedFlowState = blob.flowStack?.let { SerializedBytes(it) } return Checkpoint.Serialized( serializedCheckpointState = SerializedBytes(blob.checkpoint), - serializedFlowState = SerializedBytes(blob.flowStack), + serializedFlowState = serialisedFlowState, // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint errorState = ErrorState.Clean, // A checkpoint with a result should not normally be loaded (it should be [null] most of the time) diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index 200947538b..d57d415701 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -204,6 +204,9 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri val fiber = flowState.frozenFiber.checkpointDeserialize(context = checkpointSerializationContext) fiber to fiber.logic } + is FlowState.Completed -> { + throw IllegalStateException("Only runnable checkpoints with their flow stack are output by the checkpoint dumper") + } } val flowCallStack = if (fiber != null) { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index ceb3ae157a..9ea7c4c3f7 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -95,11 +95,18 @@ class ActionExecutorImpl( @Suspendable private fun executePersistCheckpoint(action: Action.PersistCheckpoint) { val checkpoint = action.checkpoint - val serializedFlowState = checkpoint.flowState.checkpointSerialize(checkpointSerializationContext) + val flowState = checkpoint.flowState + val serializedFlowState = when(flowState) { + FlowState.Completed -> null + else -> flowState.checkpointSerialize(checkpointSerializationContext) + } if (action.isCheckpointUpdate) { checkpointStorage.updateCheckpoint(action.id, checkpoint, serializedFlowState) } else { - checkpointStorage.addCheckpoint(action.id, checkpoint, serializedFlowState) + if (flowState is FlowState.Completed) { + throw IllegalStateException("A new checkpoint cannot be created with a Completed FlowState.") + } + checkpointStorage.addCheckpoint(action.id, checkpoint, serializedFlowState!!) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 4d9b9ddffb..0d66fb50a1 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -780,11 +780,10 @@ class SingleThreadedStateMachineManager( initialDeduplicationHandler: DeduplicationHandler? ): Flow? { val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id)?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: return null - val flowState = checkpoint.flowState val resultFuture = openFuture() - val fiber = when (flowState) { + val fiber = when (checkpoint.flowState) { is FlowState.Unstarted -> { - val logic = tryCheckpointDeserialize(flowState.frozenFlowLogic, id) ?: return null + val logic = tryCheckpointDeserialize(checkpoint.flowState.frozenFlowLogic, id) ?: return null val state = StateMachineState( checkpoint = checkpoint, pendingDeduplicationHandlers = initialDeduplicationHandler?.let { listOf(it) } ?: emptyList(), @@ -803,7 +802,7 @@ class SingleThreadedStateMachineManager( fiber } is FlowState.Started -> { - val fiber = tryCheckpointDeserialize(flowState.frozenFiber, id) ?: return null + val fiber = tryCheckpointDeserialize(checkpoint.flowState.frozenFiber, id) ?: return null val state = StateMachineState( checkpoint = checkpoint, pendingDeduplicationHandlers = initialDeduplicationHandler?.let { listOf(it) } ?: emptyList(), @@ -820,6 +819,9 @@ class SingleThreadedStateMachineManager( fiber.logic.stateMachine = fiber fiber } + is FlowState.Completed -> { + return null // Places calling this function is rely on it to return null if the flow cannot be created from the checkpoint. + } } verifyFlowLogicIsSuspendable(fiber.logic) @@ -847,18 +849,23 @@ class SingleThreadedStateMachineManager( val flowLogic = flow.fiber.logic if (flowLogic.isEnabledTimedFlow()) scheduleTimeout(id) flow.fiber.scheduleEvent(Event.DoRemainingWork) - when (checkpoint.flowState) { - is FlowState.Unstarted -> { - flow.fiber.start() - } - is FlowState.Started -> { - Fiber.unparkDeserialized(flow.fiber, scheduler) - } - } + startOrResume(checkpoint, flow) } } } + private fun startOrResume(checkpoint: Checkpoint, flow: Flow) { + when (checkpoint.flowState) { + is FlowState.Unstarted -> { + flow.fiber.start() + } + is FlowState.Started -> { + Fiber.unparkDeserialized(flow.fiber, scheduler) + } + is FlowState.Completed -> throw IllegalStateException("Cannot start (or resume) a completed flow.") + } + } + private fun getFlowSessionIds(checkpoint: Checkpoint): Set { val initiatedFlowStart = (checkpoint.flowState as? FlowState.Unstarted)?.flowStart as? FlowStart.Initiated return if (initiatedFlowStart == null) { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 842758e599..34d110747b 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -149,7 +149,7 @@ data class Checkpoint( */ data class Serialized( val serializedCheckpointState: SerializedBytes, - val serializedFlowState: SerializedBytes, + val serializedFlowState: SerializedBytes?, val errorState: ErrorState, val result: SerializedBytes?, val status: FlowStatus, @@ -165,7 +165,7 @@ data class Checkpoint( fun deserialize(checkpointSerializationContext: CheckpointSerializationContext): Checkpoint { return Checkpoint( checkpointState = serializedCheckpointState.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), - flowState = serializedFlowState.checkpointDeserialize(checkpointSerializationContext), + flowState = serializedFlowState?.checkpointDeserialize(checkpointSerializationContext) ?: FlowState.Completed, errorState = errorState, result = result?.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), status = status, @@ -299,6 +299,12 @@ sealed class FlowState { ) : FlowState() { override fun toString() = "Started(flowIORequest=$flowIORequest, frozenFiber=${frozenFiber.hash})" } + + /** + * The flow has completed. It does not have a running fiber that needs to be serialized and checkpointed. + */ + object Completed : FlowState() + } /** diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt index 53214c71fb..1b995c7088 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt @@ -24,10 +24,11 @@ class DoRemainingWorkTransition( // If the flow is clean check the FlowState private fun cleanTransition(): TransitionResult { - val checkpoint = startingState.checkpoint - return when (checkpoint.flowState) { - is FlowState.Unstarted -> UnstartedFlowTransition(context, startingState, checkpoint.flowState).transition() - is FlowState.Started -> StartedFlowTransition(context, startingState, checkpoint.flowState).transition() + val flowState = startingState.checkpoint.flowState + return when (flowState) { + is FlowState.Unstarted -> UnstartedFlowTransition(context, startingState, flowState).transition() + is FlowState.Started -> StartedFlowTransition(context, startingState, flowState).transition() + is FlowState.Completed -> throw IllegalStateException("Cannot transition a state with completed flow state.") } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 31ebaf2fcb..18e9cedb06 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -218,6 +218,7 @@ class TopLevelTransition( checkpointState = checkpoint.checkpointState.copy( numberOfSuspends = checkpoint.checkpointState.numberOfSuspends + 1 ), + flowState = FlowState.Completed, result = event.returnValue, status = Checkpoint.FlowStatus.COMPLETED ), diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 8f558f3b81..2bd17123a3 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -47,7 +47,7 @@ - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index 513cdd4428..d0ae65023f 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -47,7 +47,7 @@ - + diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index f180f70f50..9b9c7ef700 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -133,6 +133,26 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `update a checkpoint to completed`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + + val completedCheckpoint = checkpoint.copy(flowState = FlowState.Completed) + database.transaction { + checkpointStorage.updateCheckpoint(id, completedCheckpoint, null) + } + database.transaction { + assertEquals( + completedCheckpoint, + checkpointStorage.checkpoints().single().deserialize() + ) + } + } + @Test(timeout = 300_000) fun `remove checkpoint`() { val (id, checkpoint) = newCheckpoint() @@ -530,7 +550,8 @@ class DBCheckpointStorageTests { val paused = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) // is considered runnable database.transaction { - val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), runnable, serializedFlowState) checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), hospitalized, serializedFlowState) @@ -551,7 +572,7 @@ class DBCheckpointStorageTests { object : CheckpointPerformanceRecorder { override fun record( serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes + serializedFlowState: SerializedBytes? ) { // do nothing } diff --git a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt index d8c20a6c2c..4ae38eaf21 100644 --- a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt @@ -5,21 +5,26 @@ import com.natpryce.hamkrest.containsSubstring import com.nhaarman.mockito_kotlin.doReturn import com.nhaarman.mockito_kotlin.mock import com.nhaarman.mockito_kotlin.whenever +import junit.framework.TestCase.assertNull import net.corda.core.context.InvocationContext import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.FlowIORequest import net.corda.core.internal.createDirectories import net.corda.core.internal.deleteIfExists import net.corda.core.internal.deleteRecursively import net.corda.core.internal.div import net.corda.core.internal.inputStream +import net.corda.core.internal.isRegularFile +import net.corda.core.internal.list import net.corda.core.internal.readFully import net.corda.core.node.ServiceHub import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.core.utilities.toNonEmptySet import net.corda.nodeapi.internal.lifecycle.NodeServicesContext import net.corda.nodeapi.internal.lifecycle.NodeLifecycleEvent import net.corda.node.internal.NodeStartup @@ -40,10 +45,12 @@ import org.junit.Before import org.junit.Rule import org.junit.Test import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import java.time.Clock import java.time.Instant import java.util.zip.ZipInputStream +import kotlin.test.assertEquals class CheckpointDumperImplTest { @@ -111,7 +118,29 @@ class CheckpointDumperImplTest { } dumper.dumpCheckpoints() - checkDumpFile() + checkDumpFile() + } + + @Test(timeout=300_000) + fun `Checkpoint dumper doesn't output completed checkpoints`() { + val dumper = CheckpointDumperImpl(checkpointStorage, database, services, baseDirectory) + dumper.update(mockAfterStartEvent) + + // add a checkpoint + val (id, checkpoint) = newCheckpoint() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) + } + val newCheckpoint = checkpoint.copy( + flowState = FlowState.Completed, + status = Checkpoint.FlowStatus.COMPLETED + ) + database.transaction { + checkpointStorage.updateCheckpoint(id, newCheckpoint, null) + } + + dumper.dumpCheckpoints() + checkDumpFileEmpty() } private fun checkDumpFile() { @@ -123,6 +152,13 @@ class CheckpointDumperImplTest { } } + private fun checkDumpFileEmpty() { + ZipInputStream(file.inputStream()).use { zip -> + val entry = zip.nextEntry + assertNull(entry) + } + } + // This test will only succeed when the VM startup includes the "checkpoint-agent": // -javaagent:tools/checkpoint-agent/build/libs/checkpoint-agent.jar @Test(timeout=300_000) @@ -147,7 +183,7 @@ class CheckpointDumperImplTest { object : CheckpointPerformanceRecorder { override fun record( serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes + serializedFlowState: SerializedBytes? ) { // do nothing } @@ -171,4 +207,4 @@ class CheckpointDumperImplTest { private fun serializeFlowState(checkpoint: Checkpoint): SerializedBytes { return checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) } -} \ No newline at end of file +} diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 2ffbae8e22..180fc5843f 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -67,6 +67,7 @@ import org.assertj.core.api.Condition import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Before import org.junit.Test @@ -103,7 +104,7 @@ class FlowFrameworkTests { object : CheckpointPerformanceRecorder { override fun record( serializedCheckpointState: SerializedBytes, - serializedFlowState: SerializedBytes + serializedFlowState: SerializedBytes? ) { // do nothing } @@ -342,13 +343,14 @@ class FlowFrameworkTests { //We should update this test when we do the work to persists the flow result. @Test(timeout = 300_000) - fun `Flow status is set to completed in database when the flow finishes`() { + fun `Flow status is set to completed in database when the flow finishes and serialised flow state is null`() { val terminationSignal = Semaphore(0) val flow = aliceNode.services.startFlow(NoOpFlow( terminateUponSignal = terminationSignal)) mockNet.waitQuiescent() // current thread needs to wait fiber running on a different thread, has reached the blocking point aliceNode.database.transaction { val checkpoint = dbCheckpointStorage.getCheckpoint(flow.id) assertNull(checkpoint!!.result) + assertNotNull(checkpoint.serializedFlowState) assertNotEquals(Checkpoint.FlowStatus.COMPLETED, checkpoint.status) } terminationSignal.release() @@ -356,6 +358,7 @@ class FlowFrameworkTests { aliceNode.database.transaction { val checkpoint = dbCheckpointStorage.getCheckpoint(flow.id) assertNull(checkpoint!!.result) + assertNull(checkpoint.serializedFlowState) assertEquals(Checkpoint.FlowStatus.COMPLETED, checkpoint.status) } } From 501b766e71d9fe6c8f5613b53e3145418e6a00c2 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Tue, 7 Apr 2020 17:58:00 +0100 Subject: [PATCH 31/49] CORDA-3604 Store failed and hospitalised errors along with corresponding statuses (#6061) Flows that are kept for overnight observation: - Save their Checkpoint.status as 'HOSPITALIZED' in the database - Save the error that caused the hospitalization in the database A new Event was added for this reason. Whenever the hospital determines a flow for hospitalization, it adds this Event in the flow's fiber queue. When processed it creates a new DB transaction, stores the checkpoint status along with the error, and it adds a 'FlowContinuation.ProcessEvents' continuation so that the fiber keeps processing events (effectively since there are no more events in the fiber's channel, the fiber will suspend). Flows that error: - Their checkpoints are kept in the database - Save their Checkpoint.status as 'FAILED' - Save the error that caused the error in the database Upon erroring, the flow's Checkpoint.status gets updated('FAILED') and the checkpoint is stored in the database instead of getting removed. The flow then propagates the error to counterparties, sets its future with the error and gets removed from memory. --- .../net/corda/node/flows/FlowRetryTest.kt | 18 +- .../persistence/DBCheckpointStorage.kt | 64 ++++-- .../corda/node/services/statemachine/Event.kt | 9 + .../statemachine/StaffedFlowHospital.kt | 14 +- .../transitions/ErrorFlowTransition.kt | 9 +- .../transitions/TopLevelTransition.kt | 12 ++ .../node-core.changelog-v17-postgres.xml | 7 +- .../migration/node-core.changelog-v17.xml | 7 +- .../persistence/DBCheckpointStorageTests.kt | 91 +++++++- .../statemachine/FlowFrameworkTests.kt | 196 +++++++++++++----- 10 files changed, 330 insertions(+), 97 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt index 2a2eece9ec..939d755ad9 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt @@ -142,8 +142,7 @@ class FlowRetryTest { .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) } assertEquals(3, TransientConnectionFailureFlow.retryCount) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get()) } } } @@ -161,8 +160,7 @@ class FlowRetryTest { .returnValue.getOrThrow(Duration.of(10, ChronoUnit.SECONDS)) } assertEquals(3, WrappedTransientConnectionFailureFlow.retryCount) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.HOSPITALIZED).returnValue.get()) } } } @@ -180,8 +178,7 @@ class FlowRetryTest { it.proxy.startFlow(::GeneralExternalFailureFlow, nodeBHandle.nodeInfo.singleIdentity()).returnValue.getOrThrow() } assertEquals(0, GeneralExternalFailureFlow.retryCount) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(1, it.proxy.startFlow(::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + assertEquals(1, it.proxy.startFlow(::GetCheckpointNumberOfStatusFlow, Checkpoint.FlowStatus.FAILED).returnValue.get()) } } } @@ -458,9 +455,14 @@ class GeneralExternalFailureResponder(private val session: FlowSession) : FlowLo } @StartableByRPC -class GetNumberOfUncompletedCheckpointsFlow : FlowLogic() { +class GetCheckpointNumberOfStatusFlow(private val flowStatus: Checkpoint.FlowStatus) : FlowLogic() { override fun call(): Long { - val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})" + val sqlStatement = + "select count(*) " + + "from node_checkpoints " + + "where status = ${flowStatus.ordinal} " + + "and flow_id != '${runId.uuid}' " // don't count in the checkpoint of the current flow + return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> ps.executeQuery().use { rs -> rs.next() diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 1d48243f57..c67b8295e0 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -4,6 +4,7 @@ import net.corda.core.context.InvocationContext import net.corda.core.context.InvocationOrigin import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.PLATFORM_VERSION +import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.uncheckedCast import net.corda.core.serialization.SerializationDefaults import net.corda.core.serialization.SerializedBytes @@ -19,6 +20,7 @@ import net.corda.node.services.statemachine.SubFlowVersion import net.corda.nodeapi.internal.persistence.NODE_DATABASE_PREFIX import net.corda.nodeapi.internal.persistence.currentDBSession import org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY +import org.apache.commons.lang3.exception.ExceptionUtils import org.hibernate.annotations.Type import java.sql.Connection import java.sql.SQLException @@ -50,8 +52,12 @@ class DBCheckpointStorage( private const val HMAC_SIZE_BYTES = 16 - private const val MAX_PROGRESS_STEP_LENGTH = 256 + @VisibleForTesting + const val MAX_STACKTRACE_LENGTH = 4000 + private const val MAX_EXC_MSG_LENGTH = 4000 + private const val MAX_EXC_TYPE_LENGTH = 256 private const val MAX_FLOW_NAME_LENGTH = 128 + private const val MAX_PROGRESS_STEP_LENGTH = 256 private val NOT_RUNNABLE_CHECKPOINTS = listOf(FlowStatus.COMPLETED, FlowStatus.FAILED, FlowStatus.KILLED) @@ -170,15 +176,18 @@ class DBCheckpointStorage( var id: Long = 0, @Column(name = "type", nullable = false) - var type: Class, - - @Type(type = "corda-blob") - @Column(name = "exception_value", nullable = false) - var value: ByteArray = EMPTY_BYTE_ARRAY, + var type: String, @Column(name = "exception_message") var message: String? = null, + @Column(name = "stack_trace", nullable = false) + var stackTrace: String, + + @Type(type = "corda-blob") + @Column(name = "exception_value") + var value: ByteArray? = null, + @Column(name = "timestamp") val persistedInstant: Instant ) @@ -307,7 +316,8 @@ class DBCheckpointStorage( } } - private fun getDBCheckpoint(id: StateMachineRunId): DBFlowCheckpoint? { + @VisibleForTesting + internal fun getDBCheckpoint(id: StateMachineRunId): DBFlowCheckpoint? { return currentDBSession().find(DBFlowCheckpoint::class.java, id.uuid.toString()) } @@ -375,10 +385,15 @@ class DBCheckpointStorage( // Load the previous entity from the hibernate cache so the meta data join does not get updated val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) - val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() - checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + // Do not update in DB [Checkpoint.checkpointState] or [Checkpoint.flowState] if flow failed or got hospitalized + val blob = if (checkpoint.status == FlowStatus.FAILED || checkpoint.status == FlowStatus.HOSPITALIZED) { + entity.blob + } else { + val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() + checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) + } - val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) //This code needs to be added back in when we want to persist the result. For now this requires the result to be @CordaSerializable. //val result = updateDBFlowResult(entity, checkpoint, now) val exceptionDetails = updateDBFlowException(entity, checkpoint, now) @@ -478,9 +493,10 @@ class DBCheckpointStorage( private fun createDBFlowException(errorState: ErrorState.Errored, now: Instant): DBFlowException { return errorState.errors.last().exception.let { DBFlowException( - type = it::class.java, - message = it.message, - value = it.storageSerialize().bytes, + type = it::class.java.name.truncate(MAX_EXC_TYPE_LENGTH, true), + message = it.message?.truncate(MAX_EXC_MSG_LENGTH, false), + stackTrace = it.stackTraceToString(), + value = null, // TODO to be populated upon implementing https://r3-cev.atlassian.net/browse/CORDA-3681 persistedInstant = now ) } @@ -528,4 +544,26 @@ class DBCheckpointStorage( FlowStatus.COMPLETED, FlowStatus.KILLED, FlowStatus.FAILED -> true else -> false } + + private fun String.truncate(maxLength: Int, withWarnings: Boolean): String { + var str = this + if (length > maxLength) { + if (withWarnings) { + log.warn("Truncating long string before storing it into the database. String: $str.") + } + str = str.substring(0, maxLength) + } + return str + } + + private fun Throwable.stackTraceToString(): String { + var stackTraceStr = ExceptionUtils.getStackTrace(this) + if (stackTraceStr.length > MAX_STACKTRACE_LENGTH) { + // cut off the last line, which will be a half line + val lineBreak = System.getProperty("line.separator") + val truncateIndex = stackTraceStr.lastIndexOf(lineBreak, MAX_STACKTRACE_LENGTH - 1) + stackTraceStr = stackTraceStr.substring(0, truncateIndex + lineBreak.length) // include last line break in + } + return stackTraceStr + } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt index d0f96925d2..144496b396 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt @@ -154,6 +154,15 @@ sealed class Event { override fun toString() = "RetryFlowFromSafePoint" } + /** + * Keeps a flow for overnight observation. Overnight observation practically sends the fiber to get suspended, + * in [FlowStateMachineImpl.processEventsUntilFlowIsResumed]. Since the fiber's channel will have no more events to process, + * the fiber gets suspended (i.e. hospitalized). + */ + object OvernightObservation : Event() { + override fun toString() = "OvernightObservation" + } + /** * Indicates that an event was generated by an external event and that external event needs to be replayed if we retry the flow, * even if it has not yet been processed and placed on the pending de-duplication handlers list. diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt index d78b56b578..956108f78a 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt @@ -223,7 +223,7 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, log.info("Flow error kept for overnight observation by ${report.by} (error was ${report.error.message})") // We don't schedule a next event for the flow - it will automatically retry from its checkpoint on node restart onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(flowFiber.id, report.by.map{it.toString()}) } - Triple(Outcome.OVERNIGHT_OBSERVATION, null, 0.seconds) + Triple(Outcome.OVERNIGHT_OBSERVATION, Event.OvernightObservation, 0.seconds) } Diagnosis.NOT_MY_SPECIALTY, Diagnosis.TERMINAL -> { // None of the staff care for these errors, or someone decided it is a terminal condition, so we let them propagate @@ -239,14 +239,12 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, Pair(event, backOffForChronicCondition) } - if (event != null) { - if (backOffForChronicCondition.isZero) { + if (backOffForChronicCondition.isZero) { + flowFiber.scheduleEvent(event) + } else { + hospitalJobTimer.schedule(timerTask { flowFiber.scheduleEvent(event) - } else { - hospitalJobTimer.schedule(timerTask { - flowFiber.scheduleEvent(event) - }, backOffForChronicCondition.toMillis()) - } + }, backOffForChronicCondition.toMillis()) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt index 9692a374ba..551807fcdf 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/ErrorFlowTransition.kt @@ -59,11 +59,11 @@ class ErrorFlowTransition( // If we haven't been removed yet remove the flow. if (!currentState.isRemoved) { - actions.add(Action.CreateTransaction) - if (currentState.isAnyCheckpointPersisted) { - actions.add(Action.RemoveCheckpoint(context.id)) - } + val newCheckpoint = startingState.checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) + actions.addAll(arrayOf( + Action.CreateTransaction, + Action.PersistCheckpoint(context.id, newCheckpoint, isCheckpointUpdate = currentState.isAnyCheckpointPersisted), Action.PersistDeduplicationFacts(currentState.pendingDeduplicationHandlers), Action.ReleaseSoftLocks(context.id.uuid), Action.CommitTransaction, @@ -72,6 +72,7 @@ class ErrorFlowTransition( )) currentState = currentState.copy( + checkpoint = newCheckpoint, pendingDeduplicationHandlers = emptyList(), isRemoved = true ) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 18e9cedb06..79598910c0 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -48,6 +48,7 @@ class TopLevelTransition( is Event.AsyncOperationCompletion -> asyncOperationCompletionTransition(event) is Event.AsyncOperationThrows -> asyncOperationThrowsTransition(event) is Event.RetryFlowFromSafePoint -> retryFlowFromSafePointTransition(startingState) + is Event.OvernightObservation -> overnightObservationTransition() } } @@ -309,4 +310,15 @@ class TopLevelTransition( FlowContinuation.Abort } } + + private fun overnightObservationTransition(): TransitionResult { + return builder { + val newCheckpoint = startingState.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) + actions.add(Action.CreateTransaction) + actions.add(Action.PersistCheckpoint(context.id, newCheckpoint, isCheckpointUpdate = currentState.isAnyCheckpointPersisted)) + actions.add(Action.CommitTransaction) + currentState = currentState.copy(checkpoint = newCheckpoint) + FlowContinuation.ProcessEvents + } + } } diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml index 2bd17123a3..723cfb1951 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml @@ -81,10 +81,13 @@ - + + + + - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v17.xml index d0ae65023f..02a8d16f7b 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v17.xml @@ -81,10 +81,13 @@ - + + + + - + diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 9b9c7ef700..0da98cf202 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -8,6 +8,7 @@ import net.corda.core.internal.FlowIORequest import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.core.utilities.contextLogger import net.corda.node.internal.CheckpointIncompatibleException import net.corda.node.internal.CheckpointVerifier import net.corda.node.services.api.CheckpointStorage @@ -28,6 +29,7 @@ import net.corda.testing.internal.LogHelper import net.corda.testing.internal.configureDatabase import net.corda.testing.node.MockServices import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties +import org.apache.commons.lang3.exception.ExceptionUtils import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.After @@ -38,18 +40,22 @@ import org.junit.Ignore import org.junit.Rule import org.junit.Test import java.time.Clock +import java.util.ArrayList import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertTrue internal fun CheckpointStorage.checkpoints(): List { - return getRunnableCheckpoints().use { + return getAllCheckpoints().use { it.map { it.second }.toList() } } class DBCheckpointStorageTests { private companion object { + + val log = contextLogger() + val ALICE = TestIdentity(ALICE_NAME, 70).party } @@ -423,7 +429,7 @@ class DBCheckpointStorageTests { assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) val exceptionDetails = session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails assertNotNull(exceptionDetails) - assertEquals(exception::class.java, exceptionDetails!!.type) + assertEquals(exception::class.java.name, exceptionDetails!!.type) assertEquals(exception.message, exceptionDetails.message) val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) @@ -451,7 +457,7 @@ class DBCheckpointStorageTests { assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) val exceptionDetails = session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails assertNotNull(exceptionDetails) - assertEquals(illegalArgumentException::class.java, exceptionDetails!!.type) + assertEquals(illegalArgumentException::class.java.name, exceptionDetails!!.type) assertEquals(illegalArgumentException.message, exceptionDetails.message) val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) @@ -566,6 +572,85 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `-not greater than DBCheckpointStorage_MAX_STACKTRACE_LENGTH- stackTrace gets persisted as a whole`() { + val smallerDummyStackTrace = ArrayList() + val dummyStackTraceElement = StackTraceElement("class", "method", "file", 0) + + for (i in 0 until iterationsBasedOnLineSeparatorLength()) { + smallerDummyStackTrace.add(dummyStackTraceElement) + } + + val smallerStackTraceException = java.lang.IllegalStateException() + .apply { + this.stackTrace = smallerDummyStackTrace.toTypedArray() + } + val smallerStackTraceSize = ExceptionUtils.getStackTrace(smallerStackTraceException).length + assertTrue(smallerStackTraceSize < DBCheckpointStorage.MAX_STACKTRACE_LENGTH) + + val (id, checkpoint) = newCheckpoint() + database.transaction { + val serializedFlowState = checkpoint.serializeFlowState() + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(id, checkpoint.addError(smallerStackTraceException), serializedFlowState) + } + database.transaction { + val persistedCheckpoint = checkpointStorage.getDBCheckpoint(id) + val persistedStackTrace = persistedCheckpoint!!.exceptionDetails!!.stackTrace + assertEquals(smallerStackTraceSize, persistedStackTrace.length) + assertEquals(ExceptionUtils.getStackTrace(smallerStackTraceException), persistedStackTrace) + } + } + + @Test(timeout = 300_000) + fun `-greater than DBCheckpointStorage_MAX_STACKTRACE_LENGTH- stackTrace gets truncated to MAX_LENGTH_VARCHAR, and persisted`() { + val smallerDummyStackTrace = ArrayList() + val dummyStackTraceElement = StackTraceElement("class", "method", "file", 0) + + for (i in 0 until iterationsBasedOnLineSeparatorLength()) { + smallerDummyStackTrace.add(dummyStackTraceElement) + } + + val smallerStackTraceException = java.lang.IllegalStateException() + .apply { + this.stackTrace = smallerDummyStackTrace.toTypedArray() + } + val smallerStackTraceSize = ExceptionUtils.getStackTrace(smallerStackTraceException).length + log.info("smallerStackTraceSize = $smallerStackTraceSize") + assertTrue(smallerStackTraceSize < DBCheckpointStorage.MAX_STACKTRACE_LENGTH) + + val biggerStackTraceException = java.lang.IllegalStateException() + .apply { + this.stackTrace = (smallerDummyStackTrace + dummyStackTraceElement).toTypedArray() + } + val biggerStackTraceSize = ExceptionUtils.getStackTrace(biggerStackTraceException).length + log.info("biggerStackTraceSize = $biggerStackTraceSize") + assertTrue(biggerStackTraceSize > DBCheckpointStorage.MAX_STACKTRACE_LENGTH) + + val (id, checkpoint) = newCheckpoint() + database.transaction { + val serializedFlowState = checkpoint.serializeFlowState() + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(id, checkpoint.addError(biggerStackTraceException), serializedFlowState) + } + database.transaction { + val persistedCheckpoint = checkpointStorage.getDBCheckpoint(id) + val persistedStackTrace = persistedCheckpoint!!.exceptionDetails!!.stackTrace + // last line of DBFlowException.stackTrace was a half line; will be truncated to the last whole line, + // therefore it will have the exact same length as 'notGreaterThanDummyException' exception + assertEquals(smallerStackTraceSize, persistedStackTrace.length) + assertEquals(ExceptionUtils.getStackTrace(smallerStackTraceException), persistedStackTrace) + } + } + + private fun iterationsBasedOnLineSeparatorLength() = when { + System.getProperty("line.separator").length == 1 -> // Linux or Mac + 158 + System.getProperty("line.separator").length == 2 -> // Windows + 152 + else -> throw IllegalStateException("Unknown line.separator") + } + private fun newCheckpointStorage() { database.transaction { checkpointStorage = DBCheckpointStorage( diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 180fc5843f..9eefaec2e2 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -15,20 +15,22 @@ import net.corda.core.flows.FlowException import net.corda.core.flows.FlowInfo import net.corda.core.flows.FlowLogic import net.corda.core.flows.FlowSession +import net.corda.core.flows.HospitalizeFlowException import net.corda.core.flows.InitiatingFlow import net.corda.core.flows.ReceiveFinalityFlow +import net.corda.core.flows.StateMachineRunId import net.corda.core.flows.UnexpectedFlowEndException import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField import net.corda.core.internal.FlowIORequest import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.concurrent.flatMap +import net.corda.core.internal.concurrent.openFuture import net.corda.core.messaging.MessageRecipients import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize -import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.serialize import net.corda.core.toFuture import net.corda.core.transactions.SignedTransaction @@ -41,9 +43,6 @@ import net.corda.core.utilities.unwrap import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints -import net.corda.nodeapi.internal.persistence.contextDatabase -import net.corda.nodeapi.internal.persistence.contextTransaction -import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState import net.corda.testing.core.ALICE_NAME @@ -60,6 +59,7 @@ import net.corda.testing.node.internal.InternalMockNodeParameters import net.corda.testing.node.internal.TestStartedNode import net.corda.testing.node.internal.getMessage import net.corda.testing.node.internal.startFlow +import org.apache.commons.lang3.exception.ExceptionUtils import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatIllegalArgumentException import org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType @@ -73,11 +73,12 @@ import org.junit.Before import org.junit.Test import rx.Notification import rx.Observable -import java.sql.SQLException +import java.sql.SQLTransientConnectionException import java.time.Clock import java.time.Duration import java.time.Instant import java.util.* +import java.util.concurrent.TimeoutException import java.util.function.Predicate import kotlin.reflect.KClass import kotlin.streams.toList @@ -138,6 +139,9 @@ class FlowFrameworkTests { fun cleanUp() { mockNet.stopNodes() receivedSessionMessages.clear() + + SuspendingFlow.hookBeforeCheckpoint = {} + SuspendingFlow.hookAfterCheckpoint = {} } @Test(timeout=300_000) @@ -297,7 +301,8 @@ class FlowFrameworkTests { .withStackTraceContaining(ReceiveFlow::class.java.name) // Make sure the stack trace is that of the receiving flow .withStackTraceContaining("Received counter-flow exception from peer") bobNode.database.transaction { - assertThat(bobNode.internals.checkpointStorage.checkpoints()).isEmpty() + val checkpoint = bobNode.internals.checkpointStorage.checkpoints().single() + assertEquals(Checkpoint.FlowStatus.FAILED, checkpoint.status) } assertThat(receivingFiber.state).isEqualTo(Strand.State.WAITING) @@ -660,83 +665,160 @@ class FlowFrameworkTests { @Test(timeout=300_000) fun `Checkpoint status changes to RUNNABLE when flow is loaded from checkpoint - FlowState Unstarted`() { var firstExecution = true - var checkpointStatusInDBBeforeSuspension: Checkpoint.FlowStatus? = null - var checkpointStatusInDBAfterSuspension: Checkpoint.FlowStatus? = null - var checkpointStatusInMemoryBeforeSuspension: Checkpoint.FlowStatus? = null + var flowState: FlowState? = null + var dbCheckpointStatusBeforeSuspension: Checkpoint.FlowStatus? = null + var dbCheckpointStatusAfterSuspension: Checkpoint.FlowStatus? = null + var inMemoryCheckpointStatusBeforeSuspension: Checkpoint.FlowStatus? = null + val futureFiber = openFuture>().toCompletableFuture() SuspendingFlow.hookBeforeCheckpoint = { val flowFiber = this as? FlowStateMachineImpl<*> - assertTrue(flowFiber!!.transientState!!.value.checkpoint.flowState is FlowState.Unstarted) + flowState = flowFiber!!.transientState!!.value.checkpoint.flowState if (firstExecution) { - // the following manual persisting Checkpoint.status to FAILED should be removed when implementing CORDA-3604. - manuallyFailCheckpointInDB(aliceNode) - - firstExecution = false - throw SQLException("deadlock") // will cause flow to retry + throw HospitalizeFlowException() } else { - // The persisted Checkpoint should be still failed here -> it should change to RUNNABLE after suspension - checkpointStatusInDBBeforeSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status - checkpointStatusInMemoryBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status + dbCheckpointStatusBeforeSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + inMemoryCheckpointStatusBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status + + futureFiber.complete(flowFiber) } } - SuspendingFlow.hookAfterCheckpoint = { - checkpointStatusInDBAfterSuspension = aliceNode.internals.checkpointStorage.getRunnableCheckpoints().toList().single().second.status + dbCheckpointStatusAfterSuspension = aliceNode.internals.checkpointStorage.getRunnableCheckpoints().toList().single() + .second.status } - aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow() - - assertEquals(Checkpoint.FlowStatus.FAILED, checkpointStatusInDBBeforeSuspension) - assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInMemoryBeforeSuspension) - assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInDBAfterSuspension) - - SuspendingFlow.hookBeforeCheckpoint = {} - SuspendingFlow.hookAfterCheckpoint = {} + assertFailsWith { + aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow(30.seconds) // wait till flow gets hospitalized + } + // flow is in hospital + assertTrue(flowState is FlowState.Unstarted) + val inMemoryHospitalizedCheckpointStatus = aliceNode.internals.smm.snapshot().first().transientState?.value?.checkpoint?.status + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, inMemoryHospitalizedCheckpointStatus) + aliceNode.database.transaction { + val checkpoint = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, checkpoint.status) + } + // restart Node - flow will be loaded from checkpoint + firstExecution = false + aliceNode = mockNet.restartNode(aliceNode) + futureFiber.get().resultFuture.getOrThrow() // wait until the flow has completed + // checkpoint states ,after flow retried, before and after suspension + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, dbCheckpointStatusBeforeSuspension) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, inMemoryCheckpointStatusBeforeSuspension) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, dbCheckpointStatusAfterSuspension) } @Test(timeout=300_000) fun `Checkpoint status changes to RUNNABLE when flow is loaded from checkpoint - FlowState Started`() { var firstExecution = true - var checkpointStatusInDB: Checkpoint.FlowStatus? = null - var checkpointStatusInMemory: Checkpoint.FlowStatus? = null + var flowState: FlowState? = null + var dbCheckpointStatus: Checkpoint.FlowStatus? = null + var inMemoryCheckpointStatus: Checkpoint.FlowStatus? = null + val futureFiber = openFuture>().toCompletableFuture() SuspendingFlow.hookAfterCheckpoint = { val flowFiber = this as? FlowStateMachineImpl<*> - assertTrue(flowFiber!!.transientState!!.value.checkpoint.flowState is FlowState.Started) + flowState = flowFiber!!.transientState!!.value.checkpoint.flowState if (firstExecution) { - // the following manual persisting Checkpoint.status to FAILED should be removed when implementing CORDA-3604. - manuallyFailCheckpointInDB(aliceNode) - - firstExecution = false - throw SQLException("deadlock") // will cause flow to retry + throw HospitalizeFlowException() } else { - checkpointStatusInDB = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status - checkpointStatusInMemory = flowFiber.transientState!!.value.checkpoint.status + dbCheckpointStatus = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + inMemoryCheckpointStatus = flowFiber.transientState!!.value.checkpoint.status + + futureFiber.complete(flowFiber) + } + } + + assertFailsWith { + aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow(30.seconds) // wait till flow gets hospitalized + } + // flow is in hospital + assertTrue(flowState is FlowState.Started) + aliceNode.database.transaction { + val checkpoint = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, checkpoint.status) + } + // restart Node - flow will be loaded from checkpoint + firstExecution = false + aliceNode = mockNet.restartNode(aliceNode) + futureFiber.get().resultFuture.getOrThrow() // wait until the flow has completed + // checkpoint states ,after flow retried, after suspension + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, dbCheckpointStatus) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, inMemoryCheckpointStatus) + } + + @Test(timeout=300_000) + fun `Checkpoint is updated in DB with FAILED status and the error when flow fails`() { + var flowId: StateMachineRunId? = null + + val e = assertFailsWith { + val fiber = aliceNode.services.startFlow(ExceptionFlow { FlowException("Just an exception") }) + flowId = fiber.id + fiber.resultFuture.getOrThrow() + } + + aliceNode.database.transaction { + val checkpoint = aliceNode.internals.checkpointStorage.checkpoints().single() + assertEquals(Checkpoint.FlowStatus.FAILED, checkpoint.status) + + // assert all fields of DBFlowException + val persistedException = aliceNode.internals.checkpointStorage.getDBCheckpoint(flowId!!)!!.exceptionDetails + assertEquals(FlowException::class.java.name, persistedException!!.type) + assertEquals("Just an exception", persistedException.message) + assertEquals(ExceptionUtils.getStackTrace(e), persistedException.stackTrace) + assertEquals(null, persistedException.value) + } + } + + @Test(timeout=300_000) + fun `Checkpoint is updated in DB with HOSPITALIZED status and the error when flow is kept for overnight observation` () { + var flowId: StateMachineRunId? = null + + assertFailsWith { + val fiber = aliceNode.services.startFlow(ExceptionFlow { HospitalizeFlowException("Overnight observation") }) + flowId = fiber.id + fiber.resultFuture.getOrThrow(10.seconds) + } + + aliceNode.database.transaction { + val checkpoint = aliceNode.internals.checkpointStorage.checkpoints().single() + assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, checkpoint.status) + + // assert all fields of DBFlowException + val persistedException = aliceNode.internals.checkpointStorage.getDBCheckpoint(flowId!!)!!.exceptionDetails + assertEquals(HospitalizeFlowException::class.java.name, persistedException!!.type) + assertEquals("Overnight observation", persistedException.message) + assertEquals(null, persistedException.value) + } + } + + @Test(timeout=300_000) + fun `Checkpoint status and error in memory and in DB are not dirtied upon flow retry`() { + var firstExecution = true + var dbCheckpointStatus: Checkpoint.FlowStatus? = null + var inMemoryCheckpointStatus: Checkpoint.FlowStatus? = null + var persistedException: DBCheckpointStorage.DBFlowException? = null + + SuspendingFlow.hookAfterCheckpoint = { + if (firstExecution) { + firstExecution = false + throw SQLTransientConnectionException("connection is not available") + } else { + val flowFiber = this as? FlowStateMachineImpl<*> + dbCheckpointStatus = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + inMemoryCheckpointStatus = flowFiber!!.transientState!!.value.checkpoint.status + persistedException = aliceNode.internals.checkpointStorage.getDBCheckpoint(flowFiber.id)!!.exceptionDetails } } aliceNode.services.startFlow(SuspendingFlow()).resultFuture.getOrThrow() - - assertEquals(Checkpoint.FlowStatus.FAILED, checkpointStatusInDB) - assertEquals(Checkpoint.FlowStatus.RUNNABLE, checkpointStatusInMemory) - - SuspendingFlow.hookAfterCheckpoint = {} - } - - // the following method should be removed when implementing CORDA-3604. - private fun manuallyFailCheckpointInDB(node: TestStartedNode) { - val idCheckpoint = node.internals.checkpointStorage.getRunnableCheckpoints().toList().single() - val checkpoint = idCheckpoint.second - val updatedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) - node.internals.checkpointStorage.updateCheckpoint(idCheckpoint.first, - updatedCheckpoint.deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT), - updatedCheckpoint.serializedFlowState) - contextTransaction.commit() - contextTransaction.close() - contextTransactionOrNull = null - contextDatabase.newTransaction() + // checkpoint states ,after flow retried, after suspension + assertEquals(Checkpoint.FlowStatus.RUNNABLE, dbCheckpointStatus) + assertEquals(Checkpoint.FlowStatus.RUNNABLE, inMemoryCheckpointStatus) + assertEquals(null, persistedException) } //region Helpers From 896b0ab2462e3e16d8d54327d4aaafacedaf791f Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Thu, 9 Apr 2020 16:57:03 +0100 Subject: [PATCH 32/49] CORDA-3691 delete checkpoint record when complete (#6134) * CORDA-3691 Delete checkpoint when flow finishes The checkpoint and its related records in joined tables should be deleted when a flow finishes. Keeping these flows around will be completed in the future. * CORDA-3691 Ignore some flow metadata tests Ignore tests around recording the finish time of flow metadata records since we are not currently keeping COMPLETED flows in the database. --- .../persistence/DBCheckpointStorage.kt | 24 +-- .../transitions/TopLevelTransition.kt | 6 +- .../persistence/DBCheckpointStorageTests.kt | 150 +++++++++++++++--- .../statemachine/FlowFrameworkTests.kt | 26 ++- .../statemachine/FlowMetadataRecordingTest.kt | 3 + 5 files changed, 169 insertions(+), 40 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index c67b8295e0..abcdde622c 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -282,12 +282,18 @@ class DBCheckpointStorage( } override fun removeCheckpoint(id: StateMachineRunId): Boolean { - val session = currentDBSession() - val criteriaBuilder = session.criteriaBuilder - val delete = criteriaBuilder.createCriteriaDelete(DBFlowCheckpoint::class.java) - val root = delete.from(DBFlowCheckpoint::class.java) - delete.where(criteriaBuilder.equal(root.get(DBFlowCheckpoint::id.name), id.uuid.toString())) - return session.createQuery(delete).executeUpdate() > 0 + // This will be changed after performance tuning + return currentDBSession().let { session -> + session.find(DBFlowCheckpoint::class.java, id.uuid.toString())?.run { + result?.let { session.delete(result) } + exceptionDetails?.let { session.delete(exceptionDetails) } + session.delete(blob) + session.delete(this) + // The metadata foreign key might be the wrong way around + session.delete(flowMetadata) + true + } + } ?: false } override fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? { @@ -310,7 +316,7 @@ class DBCheckpointStorage( val criteriaQuery = criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) val root = criteriaQuery.from(DBFlowCheckpoint::class.java) criteriaQuery.select(root) - .where(criteriaBuilder.not(root.get(DBFlowCheckpoint::status.name).`in`(NOT_RUNNABLE_CHECKPOINTS))) + .where(criteriaBuilder.not(root.get(DBFlowCheckpoint::status.name).`in`(NOT_RUNNABLE_CHECKPOINTS))) return session.createQuery(criteriaQuery).stream().map { StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() } @@ -513,7 +519,7 @@ class DBCheckpointStorage( private fun InvocationContext.getFlowParameters(): List { // Only RPC flows have parameters which are found in index 1 - return if(arguments.isNotEmpty()) { + return if (arguments.isNotEmpty()) { uncheckedCast>(arguments[1]).toList() } else { emptyList() @@ -540,7 +546,7 @@ class DBCheckpointStorage( return serialize(context = SerializationDefaults.STORAGE_CONTEXT) } - private fun Checkpoint.isFinished() = when(status) { + private fun Checkpoint.isFinished() = when (status) { FlowStatus.COMPLETED, FlowStatus.KILLED, FlowStatus.FAILED -> true else -> false } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 79598910c0..37ac11489a 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -228,9 +228,11 @@ class TopLevelTransition( isRemoved = true ) val allSourceSessionIds = checkpoint.checkpointState.sessions.keys + if (currentState.isAnyCheckpointPersisted) { + actions.add(Action.RemoveCheckpoint(context.id)) + } actions.addAll(arrayOf( - Action.PersistCheckpoint(context.id, currentState.checkpoint, currentState.isAnyCheckpointPersisted), - Action.PersistDeduplicationFacts(pendingDeduplicationHandlers), + Action.PersistDeduplicationFacts(pendingDeduplicationHandlers), Action.ReleaseSoftLocks(event.softLocksId), Action.CommitTransaction, Action.AcknowledgeMessages(pendingDeduplicationHandlers), diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 0da98cf202..8ec79aa439 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -22,6 +22,7 @@ import net.corda.node.services.statemachine.SubFlowVersion import net.corda.node.services.transactions.PersistentUniquenessProvider import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig +import net.corda.nodeapi.internal.persistence.DatabaseTransaction import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity @@ -153,19 +154,34 @@ class DBCheckpointStorageTests { } database.transaction { assertEquals( - completedCheckpoint, - checkpointStorage.checkpoints().single().deserialize() + completedCheckpoint, + checkpointStorage.checkpoints().single().deserialize() ) } } @Test(timeout = 300_000) - fun `remove checkpoint`() { + fun `removing a checkpoint deletes from all checkpoint tables`() { + val exception = IllegalStateException("I am a naughty exception") val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } + val updatedCheckpoint = checkpoint.addError(exception).copy(result = "The result") + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + + database.transaction { + assertEquals(1, findRecordsFromDatabase().size) + // The result not stored yet + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(2, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + } + database.transaction { checkpointStorage.removeCheckpoint(id) } @@ -176,6 +192,100 @@ class DBCheckpointStorageTests { database.transaction { assertThat(checkpointStorage.checkpoints()).isEmpty() } + + database.transaction { + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(1, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + } + } + + @Test(timeout = 300_000) + fun `removing a checkpoint when there is no result does not fail`() { + val exception = IllegalStateException("I am a naughty exception") + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.addError(exception) + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + + database.transaction { + assertEquals(1, findRecordsFromDatabase().size) + // The result not stored yet + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(2, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + } + + database.transaction { + checkpointStorage.removeCheckpoint(id) + } + database.transaction { + assertThat(checkpointStorage.checkpoints()).isEmpty() + } + newCheckpointStorage() + database.transaction { + assertThat(checkpointStorage.checkpoints()).isEmpty() + } + + database.transaction { + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(1, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + } + } + + @Test(timeout = 300_000) + fun `removing a checkpoint when there is no exception does not fail`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + val updatedCheckpoint = checkpoint.copy(result = "The result") + val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + + database.transaction { + assertEquals(0, findRecordsFromDatabase().size) + // The result not stored yet + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(2, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) + } + + database.transaction { + checkpointStorage.removeCheckpoint(id) + } + database.transaction { + assertThat(checkpointStorage.checkpoints()).isEmpty() + } + newCheckpointStorage() + database.transaction { + assertThat(checkpointStorage.checkpoints()).isEmpty() + } + + database.transaction { + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + // The saving of checkpoint blobs needs to be fixed + assertEquals(1, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + } } @Test(timeout = 300_000) @@ -346,9 +456,7 @@ class DBCheckpointStorageTests { checkpointStorage.getCheckpoint(id)!!.deserialize().result ) assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) - assertEquals(1, session.createQuery(criteria).resultList.size) + assertEquals(1, findRecordsFromDatabase().size) } } @@ -379,9 +487,7 @@ class DBCheckpointStorageTests { checkpointStorage.getCheckpoint(id)!!.deserialize().result ) assertNotNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) - assertEquals(1, session.createQuery(criteria).resultList.size) + assertEquals(1, findRecordsFromDatabase().size) } } @@ -407,9 +513,7 @@ class DBCheckpointStorageTests { database.transaction { assertNull(checkpointStorage.getCheckpoint(id)!!.deserialize().result) assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).result) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowResult::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowResult::class.java)) - assertEquals(0, session.createQuery(criteria).resultList.size) + assertEquals(0, findRecordsFromDatabase().size) } } @@ -431,9 +535,7 @@ class DBCheckpointStorageTests { assertNotNull(exceptionDetails) assertEquals(exception::class.java.name, exceptionDetails!!.type) assertEquals(exception.message, exceptionDetails.message) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) - assertEquals(1, session.createQuery(criteria).resultList.size) + assertEquals(1, findRecordsFromDatabase().size) } } @@ -459,9 +561,7 @@ class DBCheckpointStorageTests { assertNotNull(exceptionDetails) assertEquals(illegalArgumentException::class.java.name, exceptionDetails!!.type) assertEquals(illegalArgumentException.message, exceptionDetails.message) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) - assertEquals(1, session.createQuery(criteria).resultList.size) + assertEquals(1, findRecordsFromDatabase().size) } } @@ -485,9 +585,7 @@ class DBCheckpointStorageTests { database.transaction { assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) - val criteria = session.criteriaBuilder.createQuery(DBCheckpointStorage.DBFlowException::class.java) - criteria.select(criteria.from(DBCheckpointStorage.DBFlowException::class.java)) - assertEquals(0, session.createQuery(criteria).resultList.size) + assertEquals(0, findRecordsFromDatabase().size) } } @@ -532,7 +630,7 @@ class DBCheckpointStorageTests { database.transaction { val newCheckpoint = checkpoint.copy(progressStep = longString) val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( - context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT + context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT ) checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) } @@ -557,7 +655,7 @@ class DBCheckpointStorageTests { database.transaction { val serializedFlowState = - checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), runnable, serializedFlowState) checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), hospitalized, serializedFlowState) @@ -706,4 +804,10 @@ class DBCheckpointStorageTests { ) ) } + + private inline fun DatabaseTransaction.findRecordsFromDatabase(): List { + val criteria = session.criteriaBuilder.createQuery(T::class.java) + criteria.select(criteria.from(T::class.java)) + return session.createQuery(criteria).resultList + } } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 9eefaec2e2..05955f343b 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -31,6 +31,7 @@ import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize +import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.serialize import net.corda.core.toFuture import net.corda.core.transactions.SignedTransaction @@ -43,6 +44,10 @@ import net.corda.core.utilities.unwrap import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints +import net.corda.nodeapi.internal.persistence.DatabaseTransaction +import net.corda.nodeapi.internal.persistence.contextDatabase +import net.corda.nodeapi.internal.persistence.contextTransaction +import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState import net.corda.testing.core.ALICE_NAME @@ -70,6 +75,7 @@ import org.junit.Assert.assertNotEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Before +import org.junit.Ignore import org.junit.Test import rx.Notification import rx.Observable @@ -77,13 +83,12 @@ import java.sql.SQLTransientConnectionException import java.time.Clock import java.time.Duration import java.time.Instant -import java.util.* +import java.util.ArrayList import java.util.concurrent.TimeoutException import java.util.function.Predicate import kotlin.reflect.KClass import kotlin.streams.toList import kotlin.test.assertFailsWith -import kotlin.test.assertNotNull import kotlin.test.assertTrue class FlowFrameworkTests { @@ -348,7 +353,7 @@ class FlowFrameworkTests { //We should update this test when we do the work to persists the flow result. @Test(timeout = 300_000) - fun `Flow status is set to completed in database when the flow finishes and serialised flow state is null`() { + fun `Checkpoint and all its related records are deleted when the flow finishes`() { val terminationSignal = Semaphore(0) val flow = aliceNode.services.startFlow(NoOpFlow( terminateUponSignal = terminationSignal)) mockNet.waitQuiescent() // current thread needs to wait fiber running on a different thread, has reached the blocking point @@ -362,12 +367,15 @@ class FlowFrameworkTests { mockNet.waitQuiescent() aliceNode.database.transaction { val checkpoint = dbCheckpointStorage.getCheckpoint(flow.id) - assertNull(checkpoint!!.result) - assertNull(checkpoint.serializedFlowState) - assertEquals(Checkpoint.FlowStatus.COMPLETED, checkpoint.status) + assertNull(checkpoint) + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) } } + // Ignoring test since completed flows are not currently keeping their checkpoints in the database + @Ignore @Test(timeout = 300_000) fun `Flow metadata finish time is set in database when the flow finishes`() { val terminationSignal = Semaphore(0) @@ -821,6 +829,12 @@ class FlowFrameworkTests { assertEquals(null, persistedException) } + private inline fun DatabaseTransaction.findRecordsFromDatabase(): List { + val criteria = session.criteriaBuilder.createQuery(T::class.java) + criteria.select(criteria.from(T::class.java)) + return session.createQuery(criteria).resultList + } + //region Helpers private val normalEnd = ExistingSessionMessage(SessionId(0), EndSessionMessage) // NormalSessionEnd(0) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index 2825a05b9f..b1246c739b 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -43,6 +43,7 @@ import net.corda.testing.driver.driver import net.corda.testing.node.User import org.assertj.core.api.Assertions.assertThat import org.junit.Before +import org.junit.Ignore import org.junit.Test import java.time.Instant import java.util.concurrent.CompletableFuture @@ -354,6 +355,8 @@ class FlowMetadataRecordingTest { } } + // Ignoring test since completed flows are not currently keeping their checkpoints in the database + @Ignore @Test(timeout = 300_000) fun `flows have their finish time recorded when completed`() { driver(DriverParameters(startNodesInProcess = true)) { From 7fe284527229c82b4130c11a43d546cdb69fc21c Mon Sep 17 00:00:00 2001 From: LankyDan Date: Tue, 5 May 2020 17:15:54 +0100 Subject: [PATCH 33/49] NOTICK Fix `KilledFlowTransition` after merge --- .../statemachine/transitions/KilledFlowTransition.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt index de25acac71..5c7b095e80 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/KilledFlowTransition.kt @@ -25,10 +25,11 @@ class KilledFlowTransition( val killedFlowErrorMessage = createErrorMessageFromError(killedFlowError) val errorMessages = listOf(killedFlowErrorMessage) - val (initiatedSessions, newSessions) = bufferErrorMessagesInInitiatingSessions(startingState.checkpoint.sessions, errorMessages) - val newCheckpoint = startingState.checkpoint.copy( - sessions = newSessions + val (initiatedSessions, newSessions) = bufferErrorMessagesInInitiatingSessions( + startingState.checkpoint.checkpointState.sessions, + errorMessages ) + val newCheckpoint = startingState.checkpoint.setSessions(sessions = newSessions) currentState = currentState.copy(checkpoint = newCheckpoint) actions.add( Action.PropagateErrors( @@ -42,7 +43,7 @@ class KilledFlowTransition( actions.add(Action.CreateTransaction) } // The checkpoint and soft locks are also removed directly in [StateMachineManager.killFlow] - if(startingState.isAnyCheckpointPersisted) { + if (startingState.isAnyCheckpointPersisted) { actions.add(Action.RemoveCheckpoint(context.id)) } actions.addAll( @@ -51,7 +52,7 @@ class KilledFlowTransition( Action.ReleaseSoftLocks(context.id.uuid), Action.CommitTransaction, Action.AcknowledgeMessages(currentState.pendingDeduplicationHandlers), - Action.RemoveSessionBindings(currentState.checkpoint.sessions.keys) + Action.RemoveSessionBindings(currentState.checkpoint.checkpointState.sessions.keys) ) ) From 217fd906b3bb0640a61681870dddf3de496c8361 Mon Sep 17 00:00:00 2001 From: LankyDan Date: Tue, 5 May 2020 17:22:42 +0100 Subject: [PATCH 34/49] NOTICK Remove unused imports --- .../node/services/statemachine/ActionExecutorImpl.kt | 7 ------- .../node/services/rpc/CheckpointDumperImplTest.kt | 10 ++-------- .../node/services/statemachine/FlowFrameworkTests.kt | 4 ---- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 4b86ff8091..925fdc99a7 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -2,13 +2,8 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable import com.codahale.metrics.Gauge -import com.codahale.metrics.Histogram -import com.codahale.metrics.MetricRegistry import com.codahale.metrics.Reservoir -import com.codahale.metrics.SlidingTimeWindowArrayReservoir -import com.codahale.metrics.SlidingTimeWindowReservoir import net.corda.core.internal.concurrent.thenMatch -import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.checkpointSerialize import net.corda.core.utilities.contextLogger @@ -19,8 +14,6 @@ import net.corda.nodeapi.internal.persistence.contextDatabase import net.corda.nodeapi.internal.persistence.contextTransaction import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import java.time.Duration -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong /** * This is the bottom execution engine of flow side-effects. diff --git a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt index 4ae38eaf21..4eb5bd5a98 100644 --- a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt @@ -10,23 +10,17 @@ import net.corda.core.context.InvocationContext import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.CordaX500Name -import net.corda.core.internal.FlowIORequest import net.corda.core.internal.createDirectories import net.corda.core.internal.deleteIfExists import net.corda.core.internal.deleteRecursively import net.corda.core.internal.div import net.corda.core.internal.inputStream -import net.corda.core.internal.isRegularFile -import net.corda.core.internal.list import net.corda.core.internal.readFully import net.corda.core.node.ServiceHub import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize -import net.corda.core.utilities.toNonEmptySet -import net.corda.nodeapi.internal.lifecycle.NodeServicesContext -import net.corda.nodeapi.internal.lifecycle.NodeLifecycleEvent import net.corda.node.internal.NodeStartup import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage @@ -35,6 +29,8 @@ import net.corda.node.services.statemachine.CheckpointState import net.corda.node.services.statemachine.FlowStart import net.corda.node.services.statemachine.FlowState import net.corda.node.services.statemachine.SubFlowVersion +import net.corda.nodeapi.internal.lifecycle.NodeLifecycleEvent +import net.corda.nodeapi.internal.lifecycle.NodeServicesContext import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.TestIdentity @@ -45,12 +41,10 @@ import org.junit.Before import org.junit.Rule import org.junit.Test import java.nio.file.Files -import java.nio.file.Path import java.nio.file.Paths import java.time.Clock import java.time.Instant import java.util.zip.ZipInputStream -import kotlin.test.assertEquals class CheckpointDumperImplTest { diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 05955f343b..72ea1fbdd6 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -31,7 +31,6 @@ import net.corda.core.node.services.PartyInfo import net.corda.core.node.services.queryBy import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize -import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.serialize import net.corda.core.toFuture import net.corda.core.transactions.SignedTransaction @@ -45,9 +44,6 @@ import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints import net.corda.nodeapi.internal.persistence.DatabaseTransaction -import net.corda.nodeapi.internal.persistence.contextDatabase -import net.corda.nodeapi.internal.persistence.contextTransaction -import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState import net.corda.testing.core.ALICE_NAME From 565afc5fdbeef84c0a27c3a521c6a91ed4f3d64a Mon Sep 17 00:00:00 2001 From: LankyDan Date: Wed, 6 May 2020 10:02:40 +0100 Subject: [PATCH 35/49] NOTICK Fix kill flow tests due to storing failed flows Failed flows are stored after the checkpoint table rework. This meant that some of the asserts in `FlowIsKilledTest` and `KillFlowTest` were wrong. --- .../corda/coretests/flows/FlowIsKilledTest.kt | 29 ++++++--- .../net/corda/node/flows/KillFlowTest.kt | 63 ++++++++++--------- 2 files changed, 56 insertions(+), 36 deletions(-) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt index 0f496d6364..95d0af17e2 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/FlowIsKilledTest.kt @@ -14,6 +14,7 @@ import net.corda.core.messaging.startFlow import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.minutes import net.corda.core.utilities.seconds +import net.corda.node.services.statemachine.Checkpoint import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.CHARLIE_NAME @@ -82,10 +83,9 @@ class FlowIsKilledTest { assertEquals(11, AFlowThatWantsToDieAndKillsItsFriends.position) assertTrue(AFlowThatWantsToDieAndKillsItsFriendsResponder.receivedKilledExceptions[BOB_NAME]!!) assertTrue(AFlowThatWantsToDieAndKillsItsFriendsResponder.receivedKilledExceptions[CHARLIE_NAME]!!) - val aliceCheckpoints = alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, aliceCheckpoints) - val bobCheckpoints = bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, bobCheckpoints) + assertEquals(1, alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, bob.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -113,10 +113,9 @@ class FlowIsKilledTest { } assertTrue(AFlowThatGetsMurderedByItsFriend.receivedKilledException) assertEquals(11, AFlowThatGetsMurderedByItsFriendResponder.position) - val aliceCheckpoints = alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, aliceCheckpoints) - val bobCheckpoints = bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, bobCheckpoints) + assertEquals(2, alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, alice.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } @@ -361,4 +360,18 @@ class FlowIsKilledTest { } } } + + @StartableByRPC + class GetNumberOfFailedCheckpointsFlow : FlowLogic() { + override fun call(): Long { + return serviceHub.jdbcSession() + .prepareStatement("select count(*) from node_checkpoints where status = ${Checkpoint.FlowStatus.FAILED.ordinal}") + .use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + } + } + } } \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt index 59ed762343..fff4b2e25f 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/KillFlowTest.kt @@ -25,6 +25,7 @@ import net.corda.core.utilities.seconds import net.corda.finance.DOLLARS import net.corda.finance.contracts.asset.Cash import net.corda.finance.flows.CashIssueFlow +import net.corda.node.services.statemachine.Checkpoint import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.CHARLIE_NAME @@ -67,8 +68,7 @@ class KillFlowTest { assertFailsWith { handle.returnValue.getOrThrow(1.minutes) } - val checkpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, checkpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -94,12 +94,11 @@ class KillFlowTest { AFlowThatGetsMurderedWhenItTriesToSuspendAndSomehowKillsItsFriendsResponder.locks.forEach { it.value.acquire() } assertTrue(AFlowThatGetsMurderedWhenItTriesToSuspendAndSomehowKillsItsFriendsResponder.receivedKilledExceptions[BOB_NAME]!!) assertTrue(AFlowThatGetsMurderedWhenItTriesToSuspendAndSomehowKillsItsFriendsResponder.receivedKilledExceptions[CHARLIE_NAME]!!) - val aliceCheckpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, aliceCheckpoints) - val bobCheckpoints = bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, bobCheckpoints) - val charlieCheckpoints = charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, charlieCheckpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, bob.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, charlie.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -119,8 +118,7 @@ class KillFlowTest { } assertTrue(time < 1.minutes.toMillis(), "It should at a minimum, take less than a minute to kill this flow") assertTrue(time < 5.seconds.toMillis(), "Really, it should take less than a few seconds to kill a flow") - val checkpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, checkpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -156,8 +154,7 @@ class KillFlowTest { } assertTrue(time < 1.minutes.toMillis(), "It should at a minimum, take less than a minute to kill this flow") assertTrue(time < 5.seconds.toMillis(), "Really, it should take less than a few seconds to kill a flow") - val checkpoints = startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, checkpoints) + assertEquals(1, startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } @Test(timeout = 300_000) @@ -175,8 +172,7 @@ class KillFlowTest { } assertTrue(time < 1.minutes.toMillis(), "It should at a minimum, take less than a minute to kill this flow") assertTrue(time < 5.seconds.toMillis(), "Really, it should take less than a few seconds to kill a flow") - val checkpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, checkpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -196,8 +192,7 @@ class KillFlowTest { } assertTrue(time < 1.minutes.toMillis(), "It should at a minimum, take less than a minute to kill this flow") assertTrue(time < 5.seconds.toMillis(), "Really, it should take less than a few seconds to kill a flow") - val checkpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, checkpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -225,12 +220,11 @@ class KillFlowTest { } assertTrue(AFlowThatGetsMurderedAndSomehowKillsItsFriendsResponder.receivedKilledExceptions[BOB_NAME]!!) assertTrue(AFlowThatGetsMurderedAndSomehowKillsItsFriendsResponder.receivedKilledExceptions[CHARLIE_NAME]!!) - val aliceCheckpoints = rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, aliceCheckpoints) - val bobCheckpoints = bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, bobCheckpoints) - val charlieCheckpoints = charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, charlieCheckpoints) + assertEquals(1, rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, bob.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, charlie.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } } @@ -259,12 +253,11 @@ class KillFlowTest { assertTrue(AFlowThatGetsMurderedByItsFriend.receivedKilledException) assertFalse(AFlowThatGetsMurderedByItsFriendResponder.receivedKilledExceptions[BOB_NAME]!!) assertTrue(AFlowThatGetsMurderedByItsFriendResponder.receivedKilledExceptions[CHARLIE_NAME]!!) - val aliceCheckpoints = alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, aliceCheckpoints) - val bobCheckpoints = bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, bobCheckpoints) - val charlieCheckpoints = charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) - assertEquals(1, charlieCheckpoints) + assertEquals(2, alice.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, alice.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, bob.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(2, charlie.rpc.startFlow(::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)) + assertEquals(1, charlie.rpc.startFlow(::GetNumberOfFailedCheckpointsFlow).returnValue.getOrThrow(20.seconds)) } } @@ -597,4 +590,18 @@ class KillFlowTest { } } } + + @StartableByRPC + class GetNumberOfFailedCheckpointsFlow : FlowLogic() { + override fun call(): Long { + return serviceHub.jdbcSession() + .prepareStatement("select count(*) from node_checkpoints where status = ${Checkpoint.FlowStatus.FAILED.ordinal}") + .use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + } + } + } } \ No newline at end of file From 8125b9bdb69079feab06133dc380ff98dd58fcf7 Mon Sep 17 00:00:00 2001 From: LankyDan Date: Wed, 6 May 2020 10:03:05 +0100 Subject: [PATCH 36/49] NOTICK Change platform version assert in `FlowMetadataRecordingTest` --- .../node/services/statemachine/FlowMetadataRecordingTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt index b1246c739b..ddac3afba8 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowMetadataRecordingTest.kt @@ -247,7 +247,7 @@ class FlowMetadataRecordingTest { it.initialParameters.deserialize(context = SerializationDefaults.STORAGE_CONTEXT) ) assertThat(it.launchingCordapp).contains("custom-cordapp") - assertEquals(6, it.platformVersion) + assertEquals(7, it.platformVersion) assertEquals(nodeAHandle.nodeInfo.singleIdentity().name.toString(), it.startedBy) assertEquals(context!!.trace.invocationId.timestamp, it.invocationInstant) assertTrue(it.startInstant >= it.invocationInstant) From cc8ce3ca99d3c372c00206ab10c094f84c0d6f70 Mon Sep 17 00:00:00 2001 From: nikinagy <61757742+nikinagy@users.noreply.github.com> Date: Tue, 19 May 2020 14:04:19 +0100 Subject: [PATCH 37/49] empty list checks (#6262) --- .../vault/HibernateQueryCriteriaParser.kt | 28 +++++++++++++------ .../node/services/vault/VaultQueryTests.kt | 21 ++++++++++++++ 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt b/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt index 2a52f9b948..5b13e476e0 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt @@ -766,8 +766,14 @@ class HibernateQueryCriteriaParser(val contractStateType: Class) .values.map { participant -> (participant as LiteralExpression<*>).literal } log.warn("Adding new participants: $participants to existing participants: $existingParticipants") - commonPredicates.replace(predicateID, criteriaBuilder.and( - getPersistentPartyRoot().get("x500Name").`in`(existingParticipants + participants))) + commonPredicates.replace( + predicateID, + checkIfListIsEmpty( + args = existingParticipants + participants, + criteriaBuilder = criteriaBuilder, + predicate = criteriaBuilder.and(getPersistentPartyRoot().get("x500Name").`in`(existingParticipants + participants)) + ) + ) } else { // Get the persistent party entity. @@ -796,12 +802,18 @@ class HibernateQueryCriteriaParser(val contractStateType: Class("stateRef"), - subRoot.get("compositeKey").get("stateRef"))), - criteriaBuilder.not(subRoot.get("x500Name").`in`(exactParticipants))) - val subQueryNotExistsPredicate = criteriaBuilder.and(criteriaBuilder.not(criteriaBuilder.exists(subQueryNotExists))) - constraintPredicates.add(subQueryNotExistsPredicate) + + //if the list of exact participants is empty, we return nothing with 1=0 + if (exactParticipants.isEmpty()) { + constraintPredicates.add(criteriaBuilder.and(criteriaBuilder.equal(criteriaBuilder.literal(1), 0))) + } else { + subQueryNotExists.where(criteriaBuilder.and( + criteriaBuilder.equal(vaultStates.get("stateRef"), + subRoot.get("compositeKey").get("stateRef"))), + criteriaBuilder.not(subRoot.get("x500Name").`in`(exactParticipants))) + val subQueryNotExistsPredicate = criteriaBuilder.and(criteriaBuilder.not(criteriaBuilder.exists(subQueryNotExists))) + constraintPredicates.add(subQueryNotExistsPredicate) + } // join with transactions for each matching participant (only required where more than one) if (exactParticipants.size > 1) diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index ef3b1f5a02..b4c0ec98bf 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -249,6 +249,27 @@ abstract class VaultQueryTestsBase : VaultQueryParties { } } + @Test(timeout=300_000) + fun `returns zero states when exact participants list is empty`() { + database.transaction { + identitySvc.verifyAndRegisterIdentity(BIG_CORP_IDENTITY) + vaultFiller.fillWithDummyState(participants = listOf(MEGA_CORP)) + vaultFiller.fillWithDummyState(participants = listOf(MEGA_CORP, BIG_CORP)) + + val criteria = VaultQueryCriteria(exactParticipants = emptyList()) + val results = vaultService.queryBy(criteria) + assertThat(results.states).hasSize(0) + + val criteriaWithOneExactParticipant = VaultQueryCriteria(exactParticipants = listOf(MEGA_CORP)) + val resultsWithOneExactParticipant = vaultService.queryBy(criteriaWithOneExactParticipant) + assertThat(resultsWithOneExactParticipant.states).hasSize(1) + + val criteriaWithMoreExactParticipants = VaultQueryCriteria(exactParticipants = listOf(MEGA_CORP, BIG_CORP)) + val resultsWithMoreExactParticipants = vaultService.queryBy(criteriaWithMoreExactParticipants) + assertThat(resultsWithMoreExactParticipants.states).hasSize(1) + } + } + @Test(timeout=300_000) fun `unconsumed base contract states for two participants`() { database.transaction { From eb52de1b4007ab2c9f725c8c31383a62b1572874 Mon Sep 17 00:00:00 2001 From: williamvigorr3 <58432369+williamvigorr3@users.noreply.github.com> Date: Tue, 19 May 2020 16:27:41 +0100 Subject: [PATCH 38/49] CORDA-3490 Add option to start node without starting checkpointed flows (#6136) Added command-line option: `--pause-all-flows` to the Node to control this. This mode causes all checkpoints to be set to status PAUSED when the state machine starts up (in StartMode.Safe mode). Changed the state machine so that PAUSED checkpoints are loaded into memory (the checkpoint is deserialised but the flow state is left serialised) but not started. Messages from peers are queued whilst the flow is paused and processed once the flow is resumed. --- .../net/corda/common/logging/Constants.kt | 2 +- .../internal/messaging/InternalCordaRPCOps.kt | 8 + .../services/statemachine/FlowPausingTest.kt | 113 ++++++++ .../net/corda/node/NodeCmdLineOptions.kt | 10 + .../net/corda/node/internal/AbstractNode.kt | 4 +- .../corda/node/internal/CheckpointVerifier.kt | 2 +- .../corda/node/internal/CordaRPCOpsImpl.kt | 2 + .../node/services/api/CheckpointStorage.kt | 23 +- .../node/services/config/NodeConfiguration.kt | 3 + .../services/config/NodeConfigurationImpl.kt | 5 +- .../schema/v1/V1NodeConfigurationSpec.kt | 5 +- .../persistence/DBCheckpointStorage.kt | 64 ++++- .../node/services/rpc/CheckpointDumperImpl.kt | 9 +- .../node/services/statemachine/FlowCreator.kt | 191 +++++++++++++ .../SingleThreadedStateMachineManager.kt | 264 ++++++------------ .../statemachine/StateMachineManager.kt | 15 +- .../statemachine/StateMachineState.kt | 12 +- .../transitions/DoRemainingWorkTransition.kt | 1 + .../corda/node/internal/NodeStartupCliTest.kt | 1 + .../node/messaging/TwoPartyTradeFlowTests.kt | 2 +- .../persistence/DBCheckpointStorageTests.kt | 117 +++++++- .../statemachine/FlowFrameworkTests.kt | 12 +- .../services/statemachine/FlowPausingTests.kt | 77 +++++ .../node/internal/InternalMockNetwork.kt | 1 + 24 files changed, 723 insertions(+), 220 deletions(-) create mode 100644 node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowPausingTest.kt create mode 100644 node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt create mode 100644 node/src/test/kotlin/net/corda/node/services/statemachine/FlowPausingTests.kt diff --git a/common/logging/src/main/kotlin/net/corda/common/logging/Constants.kt b/common/logging/src/main/kotlin/net/corda/common/logging/Constants.kt index b08dda81d7..5eb0817584 100644 --- a/common/logging/src/main/kotlin/net/corda/common/logging/Constants.kt +++ b/common/logging/src/main/kotlin/net/corda/common/logging/Constants.kt @@ -9,4 +9,4 @@ package net.corda.common.logging * (originally added to source control for ease of use) */ -internal const val CURRENT_MAJOR_RELEASE = "4.6-SNAPSHOT" +internal const val CURRENT_MAJOR_RELEASE = "4.6-SNAPSHOT" \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/messaging/InternalCordaRPCOps.kt b/core/src/main/kotlin/net/corda/core/internal/messaging/InternalCordaRPCOps.kt index 8f92d54c32..e3ab065422 100644 --- a/core/src/main/kotlin/net/corda/core/internal/messaging/InternalCordaRPCOps.kt +++ b/core/src/main/kotlin/net/corda/core/internal/messaging/InternalCordaRPCOps.kt @@ -1,5 +1,6 @@ package net.corda.core.internal.messaging +import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.AttachmentTrustInfo import net.corda.core.messaging.CordaRPCOps @@ -13,4 +14,11 @@ interface InternalCordaRPCOps : CordaRPCOps { /** Get all attachment trust information */ val attachmentTrustInfos: List + + /** + * Resume a paused flow. + * + * @return whether the flow was successfully resumed. + */ + fun unPauseFlow(id: StateMachineRunId): Boolean } \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowPausingTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowPausingTest.kt new file mode 100644 index 0000000000..c2961a8045 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/FlowPausingTest.kt @@ -0,0 +1,113 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.flows.StateMachineRunId +import net.corda.core.identity.Party +import net.corda.core.internal.messaging.InternalCordaRPCOps +import net.corda.core.messaging.CordaRPCOps +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.unwrap +import net.corda.node.services.Permissions +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.NodeParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.User +import org.junit.Test +import java.time.Duration +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FlowPausingTest { + + companion object { + val TOTAL_MESSAGES = 100 + val SLEEP_BETWEEN_MESSAGES_MS = 10L + } + + @Test(timeout = 300_000) + fun `Paused flows can recieve session messages`() { + val rpcUser = User("demo", "demo", setOf(Permissions.startFlow(), Permissions.all())) + driver(DriverParameters(startNodesInProcess = true, inMemoryDB = false)) { + val alice = startNode(NodeParameters(providedName = ALICE_NAME, rpcUsers = listOf(rpcUser))).getOrThrow() + val bob = startNode(NodeParameters(providedName = BOB_NAME, rpcUsers = listOf(rpcUser))) + val startedBob = bob.getOrThrow() + val aliceFlow = alice.rpc.startFlow(::HeartbeatFlow, startedBob.nodeInfo.legalIdentities[0]) + // We wait here for the initiated flow to start running on bob + val initiatedFlowId = startedBob.rpc.waitForFlowToStart(150) + assertNotNull(initiatedFlowId) + /* We shut down bob, we want this to happen before bob has finished receiving all of the heartbeats. + This is a Race but if bob finishes too quickly then we will fail to unpause the initiated flow running on BOB latter + and this test will fail.*/ + startedBob.stop() + //Start bob backup in Safe mode. This means no flows will run but BOB should receive messages and queue these up. + val restartedBob = startNode(NodeParameters( + providedName = BOB_NAME, + rpcUsers = listOf(rpcUser), + customOverrides = mapOf("smmStartMode" to "Safe"))).getOrThrow() + + //Sleep for long enough so BOB has time to receive all the messages. + //All messages in this period should be queued up and replayed when the flow is unpaused. + Thread.sleep(TOTAL_MESSAGES * SLEEP_BETWEEN_MESSAGES_MS) + //ALICE should not have finished yet as the HeartbeatResponderFlow should not have sent the final message back (as it is paused). + assertEquals(false, aliceFlow.returnValue.isDone) + assertEquals(true, (restartedBob.rpc as InternalCordaRPCOps).unPauseFlow(initiatedFlowId!!)) + + assertEquals(true, aliceFlow.returnValue.getOrThrow()) + alice.stop() + restartedBob.stop() + } + } + + fun CordaRPCOps.waitForFlowToStart(maxTrys: Int): StateMachineRunId? { + for (i in 1..maxTrys) { + val snapshot = this.stateMachinesSnapshot().singleOrNull() + if (snapshot == null) { + Thread.sleep(SLEEP_BETWEEN_MESSAGES_MS) + } else { + return snapshot.id + } + } + return null + } + + @StartableByRPC + @InitiatingFlow + class HeartbeatFlow(private val otherParty: Party): FlowLogic() { + var sequenceNumber = 0 + @Suspendable + override fun call(): Boolean { + val session = initiateFlow(otherParty) + for (i in 1..TOTAL_MESSAGES) { + session.send(sequenceNumber++) + sleep(Duration.ofMillis(10)) + } + val success = session.receive().unwrap{data -> data} + return success + } + } + + @InitiatedBy(HeartbeatFlow::class) + class HeartbeatResponderFlow(val session: FlowSession): FlowLogic() { + var sequenceNumber : Int = 0 + @Suspendable + override fun call() { + var pass = true + for (i in 1..TOTAL_MESSAGES) { + val receivedSequenceNumber = session.receive().unwrap{data -> data} + if (receivedSequenceNumber != sequenceNumber) { + pass = false + } + sequenceNumber++ + } + session.send(pass) + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt b/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt index d530ae4d41..f98d782ca3 100644 --- a/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt +++ b/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt @@ -14,6 +14,7 @@ import net.corda.node.services.config.ConfigHelper import net.corda.node.services.config.NodeConfiguration import net.corda.node.services.config.Valid import net.corda.node.services.config.parseAsNodeConfiguration +import net.corda.node.services.statemachine.StateMachineManager import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy import picocli.CommandLine.Option import java.nio.file.Path @@ -48,6 +49,12 @@ open class SharedNodeCmdLineOptions { ) var devMode: Boolean? = null + @Option( + names = ["--pause-all-flows"], + description = ["Do not run any flows on startup. Sets all flows to paused, which can be unpaused via RPC."] + ) + var safeMode: Boolean = false + open fun parseConfiguration(configuration: Config): Valid { val option = Configuration.Options(strict = unknownConfigKeysPolicy == UnknownConfigKeysPolicy.FAIL) return configuration.parseAsNodeConfiguration(option) @@ -186,6 +193,9 @@ open class NodeCmdLineOptions : SharedNodeCmdLineOptions() { devMode?.let { configOverrides += "devMode" to it } + if (safeMode) { + configOverrides += "smmStartMode" to StateMachineManager.StartMode.Safe.toString() + } return try { valid(ConfigHelper.loadConfig(baseDirectory, configFile, configOverrides = ConfigFactory.parseMap(configOverrides))) } catch (e: ConfigException) { diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index d281677a7d..105dab3c15 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -543,7 +543,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, tokenizableServices = null verifyCheckpointsCompatible(frozenTokenizableServices) - val smmStartedFuture = smm.start(frozenTokenizableServices) + val smmStartedFuture = smm.start(frozenTokenizableServices, configuration.smmStartMode) // Shut down the SMM so no Fibers are scheduled. runOnStop += { smm.stop(acceptableLiveFiberCountOnStop()) } val flowMonitor = FlowMonitor( @@ -1379,4 +1379,4 @@ fun clientSslOptionsCompatibleWith(nodeRpcOptions: NodeRpcOptions): ClientRpcSsl } // Here we're using the node's RPC key store as the RPC client's trust store. return ClientRpcSslOptions(trustStorePath = nodeRpcOptions.sslConfig!!.keyStorePath, trustStorePassword = nodeRpcOptions.sslConfig!!.keyStorePassword) -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt index bebb451d74..6c14a53a54 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CheckpointVerifier.kt @@ -35,7 +35,7 @@ object CheckpointVerifier { val cordappsByHash = currentCordapps.associateBy { it.jarHash } - checkpointStorage.getRunnableCheckpoints().use { + checkpointStorage.getCheckpointsToRun().use { it.forEach { (_, serializedCheckpoint) -> val checkpoint = try { serializedCheckpoint.deserialize(checkpointSerializationContext) diff --git a/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt b/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt index 571c97b82c..3ee3126c29 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt @@ -169,6 +169,8 @@ internal class CordaRPCOpsImpl( override fun killFlow(id: StateMachineRunId): Boolean = smm.killFlow(id) + override fun unPauseFlow(id: StateMachineRunId): Boolean = smm.unPauseFlow(id) + override fun stateMachinesFeed(): DataFeed, StateMachineUpdate> { val (allStateMachines, changes) = smm.track() diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index 59aa3f6300..b9f67d8bd7 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -20,6 +20,12 @@ interface CheckpointStorage { */ fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?) + /** + * Update all persisted checkpoints with status [Checkpoint.FlowStatus.RUNNABLE] or [Checkpoint.FlowStatus.HOSPITALIZED], + * changing the status to [Checkpoint.FlowStatus.PAUSED]. + */ + fun markAllPaused() + /** * Remove existing checkpoint from the store. * @return whether the id matched a checkpoint that was removed. @@ -37,14 +43,23 @@ interface CheckpointStorage { fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? /** - * Stream all checkpoints from the store. If this is backed by a database the stream will be valid until the - * underlying database connection is closed, so any processing should happen before it is closed. + * Stream all checkpoints with statuses [statuses] from the store. If this is backed by a database the stream will be valid + * until the underlying database connection is closed, so any processing should happen before it is closed. */ - fun getAllCheckpoints(): Stream> + fun getCheckpoints( + statuses: Collection = Checkpoint.FlowStatus.values().toSet() + ): Stream> /** * Stream runnable checkpoints from the store. If this is backed by a database the stream will be valid * until the underlying database connection is closed, so any processing should happen before it is closed. */ - fun getRunnableCheckpoints(): Stream> + fun getCheckpointsToRun(): Stream> + + /** + * Stream paused checkpoints from the store. If this is backed by a database the stream will be valid + * until the underlying database connection is closed, so any processing should happen before it is closed. + * This method does not fetch [Checkpoint.Serialized.serializedFlowState] to save memory. + */ + fun getPausedCheckpoints(): Stream> } diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt index f2dc3f16cb..39d2c04a93 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt @@ -11,6 +11,7 @@ import net.corda.core.internal.notary.NotaryServiceFlow import net.corda.core.utilities.NetworkHostAndPort import net.corda.node.services.config.rpc.NodeRpcOptions import net.corda.node.services.config.schema.v1.V1NodeConfigurationSpec +import net.corda.node.services.statemachine.StateMachineManager import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier import net.corda.nodeapi.internal.config.MutualSslConfiguration import net.corda.nodeapi.internal.config.User @@ -93,6 +94,8 @@ interface NodeConfiguration : ConfigurationWithOptionsContainer { val quasarExcludePackages: List + val smmStartMode: StateMachineManager.StartMode + companion object { // default to at least 8MB and a bit extra for larger heap sizes val defaultTransactionCacheSize: Long = 8.MB + getAdditionalCacheMemory() diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt index e1dcc86903..6a9128b503 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt @@ -8,6 +8,7 @@ import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.loggerFor import net.corda.core.utilities.seconds import net.corda.node.services.config.rpc.NodeRpcOptions +import net.corda.node.services.statemachine.StateMachineManager import net.corda.nodeapi.BrokerRpcSslOptions import net.corda.nodeapi.internal.DEV_PUB_KEY_HASHES import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier @@ -84,7 +85,8 @@ data class NodeConfigurationImpl( override val blacklistedAttachmentSigningKeys: List = Defaults.blacklistedAttachmentSigningKeys, override val configurationWithOptions: ConfigurationWithOptions, override val flowExternalOperationThreadPoolSize: Int = Defaults.flowExternalOperationThreadPoolSize, - override val quasarExcludePackages: List = Defaults.quasarExcludePackages + override val quasarExcludePackages: List = Defaults.quasarExcludePackages, + override val smmStartMode : StateMachineManager.StartMode = Defaults.smmStartMode ) : NodeConfiguration { internal object Defaults { val jmxMonitoringHttpPort: Int? = null @@ -123,6 +125,7 @@ data class NodeConfigurationImpl( val blacklistedAttachmentSigningKeys: List = emptyList() const val flowExternalOperationThreadPoolSize: Int = 1 val quasarExcludePackages: List = emptyList() + val smmStartMode : StateMachineManager.StartMode = StateMachineManager.StartMode.ExcludingPaused fun cordappsDirectories(baseDirectory: Path) = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT) diff --git a/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt b/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt index b4c5477e14..a5bde3e836 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt @@ -9,6 +9,7 @@ import net.corda.common.validation.internal.Validated.Companion.valid import net.corda.node.services.config.* import net.corda.node.services.config.NodeConfigurationImpl.Defaults import net.corda.node.services.config.schema.parsers.* +import net.corda.node.services.statemachine.StateMachineManager internal object V1NodeConfigurationSpec : Configuration.Specification("NodeConfiguration") { private val myLegalName by string().mapValid(::toCordaX500Name) @@ -66,6 +67,7 @@ internal object V1NodeConfigurationSpec : Configuration.Specification @@ -300,33 +309,39 @@ class DBCheckpointStorage( return getDBCheckpoint(id)?.toSerializedCheckpoint() } - override fun getAllCheckpoints(): Stream> { - val session = currentDBSession() - val criteriaQuery = session.criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) - val root = criteriaQuery.from(DBFlowCheckpoint::class.java) - criteriaQuery.select(root) - return session.createQuery(criteriaQuery).stream().map { - StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() - } - } - - override fun getRunnableCheckpoints(): Stream> { + override fun getCheckpoints(statuses: Collection): Stream> { val session = currentDBSession() val criteriaBuilder = session.criteriaBuilder val criteriaQuery = criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) val root = criteriaQuery.from(DBFlowCheckpoint::class.java) criteriaQuery.select(root) - .where(criteriaBuilder.not(root.get(DBFlowCheckpoint::status.name).`in`(NOT_RUNNABLE_CHECKPOINTS))) + .where(criteriaBuilder.isTrue(root.get(DBFlowCheckpoint::status.name).`in`(statuses))) return session.createQuery(criteriaQuery).stream().map { StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() } } + override fun getCheckpointsToRun(): Stream> { + return getCheckpoints(RUNNABLE_CHECKPOINTS) + } + @VisibleForTesting internal fun getDBCheckpoint(id: StateMachineRunId): DBFlowCheckpoint? { return currentDBSession().find(DBFlowCheckpoint::class.java, id.uuid.toString()) } + override fun getPausedCheckpoints(): Stream> { + val session = currentDBSession() + val jpqlQuery = """select new ${DBPausedFields::class.java.name}(checkpoint.id, blob.checkpoint, checkpoint.status, + checkpoint.progressStep, checkpoint.ioRequestType, checkpoint.compatible) from ${DBFlowCheckpoint::class.java.name} + checkpoint join ${DBFlowCheckpointBlob::class.java.name} blob on checkpoint.blob = blob.id where + checkpoint.status = ${FlowStatus.PAUSED.ordinal}""".trimIndent() + val query = session.createQuery(jpqlQuery, DBPausedFields::class.java) + return query.resultList.stream().map { + StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() + } + } + private fun createDBCheckpoint( id: StateMachineRunId, checkpoint: Checkpoint, @@ -542,6 +557,29 @@ class DBCheckpointStorage( ) } + private class DBPausedFields( + val id: String, + val checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + val status: FlowStatus, + val progressStep: String?, + val ioRequestType: String?, + val compatible: Boolean + ) { + fun toSerializedCheckpoint(): Checkpoint.Serialized { + return Checkpoint.Serialized( + serializedCheckpointState = SerializedBytes(checkpoint), + serializedFlowState = null, + // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint + errorState = ErrorState.Clean, + result = null, + status = status, + progressStep = progressStep, + flowIoRequest = ioRequestType, + compatible = compatible + ) + } + } + private fun T.storageSerialize(): SerializedBytes { return serialize(context = SerializationDefaults.STORAGE_CONTEXT) } diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt index d57d415701..2d57f8947e 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumperImpl.kt @@ -90,6 +90,11 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri companion object { internal val TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss").withZone(UTC) private val log = contextLogger() + private val DUMPABLE_CHECKPOINTS = setOf( + Checkpoint.FlowStatus.RUNNABLE, + Checkpoint.FlowStatus.HOSPITALIZED, + Checkpoint.FlowStatus.PAUSED + ) } override val priority: Int = SERVICE_PRIORITY_NORMAL @@ -141,7 +146,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri try { if (lock.getAndIncrement() == 0 && !file.exists()) { database.transaction { - checkpointStorage.getRunnableCheckpoints().use { stream -> + checkpointStorage.getCheckpoints(DUMPABLE_CHECKPOINTS).use { stream -> ZipOutputStream(file.outputStream()).use { zip -> stream.forEach { (runId, serialisedCheckpoint) -> @@ -204,7 +209,7 @@ class CheckpointDumperImpl(private val checkpointStorage: CheckpointStorage, pri val fiber = flowState.frozenFiber.checkpointDeserialize(context = checkpointSerializationContext) fiber to fiber.logic } - is FlowState.Completed -> { + else -> { throw IllegalStateException("Only runnable checkpoints with their flow stack are output by the checkpoint dumper") } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt new file mode 100644 index 0000000000..be8026b73f --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowCreator.kt @@ -0,0 +1,191 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.FiberScheduler +import co.paralleluniverse.fibers.Suspendable +import co.paralleluniverse.strands.channels.Channels +import net.corda.core.concurrent.CordaFuture +import net.corda.core.context.InvocationContext +import net.corda.core.flows.FlowException +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.StateMachineRunId +import net.corda.core.identity.Party +import net.corda.core.internal.concurrent.OpenFuture +import net.corda.core.internal.concurrent.openFuture +import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.internal.CheckpointSerializationContext +import net.corda.core.serialization.internal.checkpointDeserialize +import net.corda.core.serialization.internal.checkpointSerialize +import net.corda.core.utilities.contextLogger +import net.corda.node.services.api.CheckpointStorage +import net.corda.node.services.api.ServiceHubInternal +import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.statemachine.transitions.StateMachine +import net.corda.node.utilities.isEnabledTimedFlow +import net.corda.nodeapi.internal.persistence.CordaPersistence +import org.apache.activemq.artemis.utils.ReusableLatch +import java.security.SecureRandom + +class Flow(val fiber: FlowStateMachineImpl, val resultFuture: OpenFuture) + +class NonResidentFlow(val runId: StateMachineRunId, val checkpoint: Checkpoint) { + val externalEvents = mutableListOf() + + fun addExternalEvent(message: Event.DeliverSessionMessage) { + externalEvents.add(message) + } +} + +class FlowCreator( + val checkpointSerializationContext: CheckpointSerializationContext, + private val checkpointStorage: CheckpointStorage, + val scheduler: FiberScheduler, + val database: CordaPersistence, + val transitionExecutor: TransitionExecutor, + val actionExecutor: ActionExecutor, + val secureRandom: SecureRandom, + val serviceHub: ServiceHubInternal, + val unfinishedFibers: ReusableLatch, + val resetCustomTimeout: (StateMachineRunId, Long) -> Unit) { + + companion object { + private val logger = contextLogger() + } + + fun createFlowFromNonResidentFlow(nonResidentFlow: NonResidentFlow): Flow<*>? { + // As for paused flows we don't extract the serialized flow state we need to re-extract the checkpoint from the database. + val checkpoint = when (nonResidentFlow.checkpoint.status) { + Checkpoint.FlowStatus.PAUSED -> { + val serialized = database.transaction { + checkpointStorage.getCheckpoint(nonResidentFlow.runId) + } + serialized?.copy(status = Checkpoint.FlowStatus.RUNNABLE)?.deserialize(checkpointSerializationContext) ?: return null + } + else -> nonResidentFlow.checkpoint + } + return createFlowFromCheckpoint(nonResidentFlow.runId, checkpoint) + } + + fun createFlowFromCheckpoint(runId: StateMachineRunId, oldCheckpoint: Checkpoint): Flow<*>? { + val checkpoint = oldCheckpoint.copy(status = Checkpoint.FlowStatus.RUNNABLE) + val fiber = checkpoint.getFiberFromCheckpoint(runId) ?: return null + val resultFuture = openFuture() + fiber.transientValues = TransientReference(createTransientValues(runId, resultFuture)) + fiber.logic.stateMachine = fiber + verifyFlowLogicIsSuspendable(fiber.logic) + val state = createStateMachineState(checkpoint, fiber, true) + fiber.transientState = TransientReference(state) + return Flow(fiber, resultFuture) + } + + @Suppress("LongParameterList") + fun createFlowFromLogic( + flowId: StateMachineRunId, + invocationContext: InvocationContext, + flowLogic: FlowLogic, + flowStart: FlowStart, + ourIdentity: Party, + existingCheckpoint: Checkpoint?, + deduplicationHandler: DeduplicationHandler?, + senderUUID: String?): Flow { + // Before we construct the state machine state by freezing the FlowLogic we need to make sure that lazy properties + // have access to the fiber (and thereby the service hub) + val flowStateMachineImpl = FlowStateMachineImpl(flowId, flowLogic, scheduler) + val resultFuture = openFuture() + flowStateMachineImpl.transientValues = TransientReference(createTransientValues(flowId, resultFuture)) + flowLogic.stateMachine = flowStateMachineImpl + val frozenFlowLogic = (flowLogic as FlowLogic<*>).checkpointSerialize(context = checkpointSerializationContext) + val flowCorDappVersion = FlowStateMachineImpl.createSubFlowVersion( + serviceHub.cordappProvider.getCordappForFlow(flowLogic), serviceHub.myInfo.platformVersion) + + val checkpoint = existingCheckpoint?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: Checkpoint.create( + invocationContext, + flowStart, + flowLogic.javaClass, + frozenFlowLogic, + ourIdentity, + flowCorDappVersion, + flowLogic.isEnabledTimedFlow() + ).getOrThrow() + + val state = createStateMachineState( + checkpoint, + flowStateMachineImpl, + existingCheckpoint != null, + deduplicationHandler, + senderUUID) + flowStateMachineImpl.transientState = TransientReference(state) + return Flow(flowStateMachineImpl, resultFuture) + } + + private fun Checkpoint.getFiberFromCheckpoint(runId: StateMachineRunId): FlowStateMachineImpl<*>? { + return when (this.flowState) { + is FlowState.Unstarted -> { + val logic = tryCheckpointDeserialize(this.flowState.frozenFlowLogic, runId) ?: return null + FlowStateMachineImpl(runId, logic, scheduler) + } + is FlowState.Started -> tryCheckpointDeserialize(this.flowState.frozenFiber, runId) ?: return null + // Places calling this function is rely on it to return null if the flow cannot be created from the checkpoint. + else -> { + return null + } + } + } + + @Suppress("TooGenericExceptionCaught") + private inline fun tryCheckpointDeserialize(bytes: SerializedBytes, flowId: StateMachineRunId): T? { + return try { + bytes.checkpointDeserialize(context = checkpointSerializationContext) + } catch (e: Exception) { + logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) + null + } + } + + private fun verifyFlowLogicIsSuspendable(logic: FlowLogic) { + // Quasar requires (in Java 8) that at least the call method be annotated suspendable. Unfortunately, it's + // easy to forget to add this when creating a new flow, so we check here to give the user a better error. + // + // The Kotlin compiler can sometimes generate a synthetic bridge method from a single call declaration, which + // forwards to the void method and then returns Unit. However annotations do not get copied across to this + // bridge, so we have to do a more complex scan here. + val call = logic.javaClass.methods.first { !it.isSynthetic && it.name == "call" && it.parameterCount == 0 } + if (call.getAnnotation(Suspendable::class.java) == null) { + throw FlowException("${logic.javaClass.name}.call() is not annotated as @Suspendable. Please fix this.") + } + } + + private fun createTransientValues(id: StateMachineRunId, resultFuture: CordaFuture): FlowStateMachineImpl.TransientValues { + return FlowStateMachineImpl.TransientValues( + eventQueue = Channels.newChannel(-1, Channels.OverflowPolicy.BLOCK), + resultFuture = resultFuture, + database = database, + transitionExecutor = transitionExecutor, + actionExecutor = actionExecutor, + stateMachine = StateMachine(id, secureRandom), + serviceHub = serviceHub, + checkpointSerializationContext = checkpointSerializationContext, + unfinishedFibers = unfinishedFibers, + waitTimeUpdateHook = { flowId, timeout -> resetCustomTimeout(flowId, timeout) } + ) + } + + private fun createStateMachineState( + checkpoint: Checkpoint, + fiber: FlowStateMachineImpl<*>, + anyCheckpointPersisted: Boolean, + deduplicationHandler: DeduplicationHandler? = null, + senderUUID: String? = null): StateMachineState { + return StateMachineState( + checkpoint = checkpoint, + pendingDeduplicationHandlers = deduplicationHandler?.let { listOf(it) } ?: emptyList(), + isFlowResumed = false, + future = null, + isWaitingForFuture = false, + isAnyCheckpointPersisted = anyCheckpointPersisted, + isStartIdempotent = false, + isRemoved = false, + isKilled = false, + flowLogic = fiber.logic, + senderUUID = senderUUID) + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 692afc5e2b..985b44fff4 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -2,9 +2,7 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Fiber import co.paralleluniverse.fibers.FiberExecutorScheduler -import co.paralleluniverse.fibers.Suspendable import co.paralleluniverse.fibers.instrument.JavaAgent -import co.paralleluniverse.strands.channels.Channels import com.codahale.metrics.Gauge import com.google.common.util.concurrent.ThreadFactoryBuilder import net.corda.core.concurrent.CordaFuture @@ -24,12 +22,9 @@ import net.corda.core.internal.concurrent.mapError import net.corda.core.internal.concurrent.openFuture import net.corda.core.internal.mapNotNull import net.corda.core.messaging.DataFeed -import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.CheckpointSerializationDefaults -import net.corda.core.serialization.internal.checkpointDeserialize -import net.corda.core.serialization.internal.checkpointSerialize import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.Try import net.corda.core.utilities.contextLogger @@ -39,13 +34,11 @@ import net.corda.node.services.api.CheckpointStorage import net.corda.node.services.api.ServiceHubInternal import net.corda.node.services.config.shouldCheckCheckpoints import net.corda.node.services.messaging.DeduplicationHandler -import net.corda.node.services.statemachine.FlowStateMachineImpl.Companion.createSubFlowVersion import net.corda.node.services.statemachine.interceptors.DumpHistoryOnErrorInterceptor import net.corda.node.services.statemachine.interceptors.FiberDeserializationChecker import net.corda.node.services.statemachine.interceptors.FiberDeserializationCheckingInterceptor import net.corda.node.services.statemachine.interceptors.HospitalisingInterceptor import net.corda.node.services.statemachine.interceptors.PrintingInterceptor -import net.corda.node.services.statemachine.transitions.StateMachine import net.corda.node.utilities.AffinityExecutor import net.corda.node.utilities.errorAndTerminate import net.corda.node.utilities.injectOldProgressTracker @@ -61,6 +54,7 @@ import java.lang.Integer.min import java.security.SecureRandom import java.time.Duration import java.util.HashSet +import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -90,8 +84,6 @@ class SingleThreadedStateMachineManager( private val logger = contextLogger() } - private class Flow(val fiber: FlowStateMachineImpl<*>, val resultFuture: OpenFuture) - private data class ScheduledTimeout( /** Will fire a [FlowTimeoutException] indicating to the flow hospital to restart the flow. */ val scheduledFuture: ScheduledFuture<*>, @@ -105,7 +97,8 @@ class SingleThreadedStateMachineManager( val changesPublisher = PublishSubject.create()!! /** True if we're shutting down, so don't resume anything. */ var stopping = false - val flows = HashMap() + val flows = HashMap>() + val pausedFlows = HashMap() val startedFutures = HashMap>() /** Flows scheduled to be retried if not finished within the specified timeout period. */ val timedFlows = HashMap() @@ -127,7 +120,7 @@ class SingleThreadedStateMachineManager( private val ourSenderUUID = serviceHub.networkService.ourSenderUUID private var checkpointSerializationContext: CheckpointSerializationContext? = null - private var actionExecutor: ActionExecutor? = null + private lateinit var flowCreator: FlowCreator override val flowHospital: StaffedFlowHospital = makeFlowHospital() private val transitionExecutor = makeTransitionExecutor() @@ -146,7 +139,7 @@ class SingleThreadedStateMachineManager( */ override val changes: Observable = mutex.content.changesPublisher - override fun start(tokenizableServices: List) : CordaFuture { + override fun start(tokenizableServices: List, startMode: StateMachineManager.StartMode): CordaFuture { checkQuasarJavaAgentPresence() val checkpointSerializationContext = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT.withTokenContext( CheckpointSerializeAsTokenContextImpl( @@ -157,8 +150,24 @@ class SingleThreadedStateMachineManager( ) ) this.checkpointSerializationContext = checkpointSerializationContext - this.actionExecutor = makeActionExecutor(checkpointSerializationContext) + val actionExecutor = makeActionExecutor(checkpointSerializationContext) fiberDeserializationChecker?.start(checkpointSerializationContext) + when (startMode) { + StateMachineManager.StartMode.ExcludingPaused -> {} + StateMachineManager.StartMode.Safe -> markAllFlowsAsPaused() + } + this.flowCreator = FlowCreator( + checkpointSerializationContext, + checkpointStorage, + scheduler, + database, + transitionExecutor, + actionExecutor, + secureRandom, + serviceHub, + unfinishedFibers, + ::resetCustomTimeout) + val fibers = restoreFlowsFromCheckpoints() metrics.register("Flows.InFlight", Gauge { mutex.content.flows.size }) Fiber.setDefaultUncaughtExceptionHandler { fiber, throwable -> @@ -168,6 +177,17 @@ class SingleThreadedStateMachineManager( (fiber as FlowStateMachineImpl<*>).logger.warn("Caught exception from flow", throwable) } } + + val pausedFlows = restoreNonResidentFlowsFromPausedCheckpoints() + mutex.locked { + this.pausedFlows.putAll(pausedFlows) + for ((id, flow) in pausedFlows) { + val checkpoint = flow.checkpoint + for (sessionId in getFlowSessionIds(checkpoint)) { + sessionToFlow[sessionId] = id + } + } + } return serviceHub.networkMapCache.nodeReady.map { logger.info("Node ready, info: ${serviceHub.myInfo}") resumeRestoredFlows(fibers) @@ -241,8 +261,7 @@ class SingleThreadedStateMachineManager( flowLogic = flowLogic, flowStart = FlowStart.Explicit, ourIdentity = ourIdentity ?: ourFirstIdentity, - deduplicationHandler = deduplicationHandler, - isStartIdempotent = false + deduplicationHandler = deduplicationHandler ) } @@ -282,6 +301,22 @@ class SingleThreadedStateMachineManager( } } + private fun markAllFlowsAsPaused() { + return checkpointStorage.markAllPaused() + } + + override fun unPauseFlow(id: StateMachineRunId): Boolean { + mutex.locked { + val pausedFlow = pausedFlows.remove(id) ?: return false + val flow = flowCreator.createFlowFromNonResidentFlow(pausedFlow) ?: return false + addAndStartFlow(flow.fiber.id, flow) + for (event in pausedFlow.externalEvents) { + flow.fiber.scheduleEvent(event) + } + } + return true + } + override fun addSessionBinding(flowId: StateMachineRunId, sessionId: SessionId) { val previousFlowId = sessionToFlow.put(sessionId, flowId) if (previousFlowId != null) { @@ -352,23 +387,28 @@ class SingleThreadedStateMachineManager( liveFibers.countUp() } - private fun restoreFlowsFromCheckpoints(): List { - return checkpointStorage.getRunnableCheckpoints().use { + private fun restoreFlowsFromCheckpoints(): List> { + return checkpointStorage.getCheckpointsToRun().use { it.mapNotNull { (id, serializedCheckpoint) -> // If a flow is added before start() then don't attempt to restore it mutex.locked { if (id in flows) return@mapNotNull null } - createFlowFromCheckpoint( - id = id, - serializedCheckpoint = serializedCheckpoint, - initialDeduplicationHandler = null, - isAnyCheckpointPersisted = true, - isStartIdempotent = false - ) + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id) ?: return@mapNotNull null + flowCreator.createFlowFromCheckpoint(id, checkpoint) }.toList() } } - private fun resumeRestoredFlows(flows: List) { + private fun restoreNonResidentFlowsFromPausedCheckpoints(): Map { + return checkpointStorage.getPausedCheckpoints().use { + it.mapNotNull { (id, serializedCheckpoint) -> + // If a flow is added before start() then don't attempt to restore it + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id) ?: return@mapNotNull null + id to NonResidentFlow(id, checkpoint) + }.toList().toMap() + } + } + + private fun resumeRestoredFlows(flows: List>) { for (flow in flows) { addAndStartFlow(flow.fiber.id, flow) } @@ -393,14 +433,10 @@ class SingleThreadedStateMachineManager( logger.error("Unable to find database checkpoint for flow $flowId. Something is very wrong. The flow will not retry.") return } + + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) ?: return // Resurrect flow - createFlowFromCheckpoint( - id = flowId, - serializedCheckpoint = serializedCheckpoint, - initialDeduplicationHandler = null, - isAnyCheckpointPersisted = true, - isStartIdempotent = false - ) ?: return + flowCreator.createFlowFromCheckpoint(flowId, checkpoint) ?: return } else { // Just flow initiation message null @@ -503,9 +539,13 @@ class SingleThreadedStateMachineManager( logger.info("Cannot find flow corresponding to session ID - $recipientId.") } } else { - mutex.locked { flows[flowId] }?.run { - fiber.scheduleEvent(Event.DeliverSessionMessage(sessionMessage, deduplicationHandler, sender)) - } ?: logger.info("Cannot find fiber corresponding to flow ID $flowId") + val event = Event.DeliverSessionMessage(sessionMessage, deduplicationHandler, sender) + mutex.locked { + flows[flowId]?.run { fiber.scheduleEvent(event) } + // If flow is not running add it to the list of external events to be processed if/when the flow resumes. + ?: pausedFlows[flowId]?.run { addExternalEvent(event) } + ?: logger.info("Cannot find fiber corresponding to flow ID $flowId") + } } } catch (exception: Exception) { logger.error("Exception while routing $sessionMessage", exception) @@ -582,8 +622,7 @@ class SingleThreadedStateMachineManager( flowLogic, flowStart, ourIdentity, - initiatingMessageDeduplicationHandler, - isStartIdempotent = false + initiatingMessageDeduplicationHandler ) } @@ -594,20 +633,9 @@ class SingleThreadedStateMachineManager( flowLogic: FlowLogic, flowStart: FlowStart, ourIdentity: Party, - deduplicationHandler: DeduplicationHandler?, - isStartIdempotent: Boolean + deduplicationHandler: DeduplicationHandler? ): CordaFuture> { - // Before we construct the state machine state by freezing the FlowLogic we need to make sure that lazy properties - // have access to the fiber (and thereby the service hub) - val flowStateMachineImpl = FlowStateMachineImpl(flowId, flowLogic, scheduler) - val resultFuture = openFuture() - flowStateMachineImpl.transientValues = TransientReference(createTransientValues(flowId, resultFuture)) - flowLogic.stateMachine = flowStateMachineImpl - val frozenFlowLogic = (flowLogic as FlowLogic<*>).checkpointSerialize(context = checkpointSerializationContext!!) - - val flowCorDappVersion = createSubFlowVersion(serviceHub.cordappProvider.getCordappForFlow(flowLogic), serviceHub.myInfo.platformVersion) - val flowAlreadyExists = mutex.locked { flows[flowId] != null } val existingCheckpoint = if (flowAlreadyExists) { @@ -629,37 +657,15 @@ class SingleThreadedStateMachineManager( // This is a brand new flow null } - val checkpoint = existingCheckpoint?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: Checkpoint.create( - invocationContext, - flowStart, - flowLogic.javaClass, - frozenFlowLogic, - ourIdentity, - flowCorDappVersion, - flowLogic.isEnabledTimedFlow() - ).getOrThrow() + val flow = flowCreator.createFlowFromLogic(flowId, invocationContext, flowLogic, flowStart, ourIdentity, existingCheckpoint, deduplicationHandler, ourSenderUUID) val startedFuture = openFuture() - val initialState = StateMachineState( - checkpoint = checkpoint, - pendingDeduplicationHandlers = deduplicationHandler?.let { listOf(it) } ?: emptyList(), - isFlowResumed = false, - isWaitingForFuture = false, - future = null, - isAnyCheckpointPersisted = existingCheckpoint != null, - isStartIdempotent = isStartIdempotent, - isRemoved = false, - isKilled = false, - flowLogic = flowLogic, - senderUUID = ourSenderUUID - ) - flowStateMachineImpl.transientState = TransientReference(initialState) mutex.locked { startedFutures[flowId] = startedFuture } totalStartedFlows.inc() - addAndStartFlow(flowId, Flow(flowStateMachineImpl, resultFuture)) - return startedFuture.map { flowStateMachineImpl as FlowStateMachine } + addAndStartFlow(flowId, flow) + return startedFuture.map { flow.fiber as FlowStateMachine } } override fun scheduleFlowTimeout(flowId: StateMachineRunId) { @@ -739,7 +745,7 @@ class SingleThreadedStateMachineManager( } /** Schedules a [FlowTimeoutException] to be fired in order to restart the flow. */ - private fun scheduleTimeoutException(flow: Flow, delay: Long): ScheduledFuture<*> { + private fun scheduleTimeoutException(flow: Flow<*>, delay: Long): ScheduledFuture<*> { return with(serviceHub.configuration.flowTimeout) { scheduledFutureExecutor.schedule({ val event = Event.Error(FlowTimeoutException()) @@ -767,43 +773,6 @@ class SingleThreadedStateMachineManager( } } - private fun verifyFlowLogicIsSuspendable(logic: FlowLogic) { - // Quasar requires (in Java 8) that at least the call method be annotated suspendable. Unfortunately, it's - // easy to forget to add this when creating a new flow, so we check here to give the user a better error. - // - // The Kotlin compiler can sometimes generate a synthetic bridge method from a single call declaration, which - // forwards to the void method and then returns Unit. However annotations do not get copied across to this - // bridge, so we have to do a more complex scan here. - val call = logic.javaClass.methods.first { !it.isSynthetic && it.name == "call" && it.parameterCount == 0 } - if (call.getAnnotation(Suspendable::class.java) == null) { - throw FlowException("${logic.javaClass.name}.call() is not annotated as @Suspendable. Please fix this.") - } - } - - private fun createTransientValues(id: StateMachineRunId, resultFuture: CordaFuture): FlowStateMachineImpl.TransientValues { - return FlowStateMachineImpl.TransientValues( - eventQueue = Channels.newChannel(-1, Channels.OverflowPolicy.BLOCK), - resultFuture = resultFuture, - database = database, - transitionExecutor = transitionExecutor, - actionExecutor = actionExecutor!!, - stateMachine = StateMachine(id, secureRandom), - serviceHub = serviceHub, - checkpointSerializationContext = checkpointSerializationContext!!, - unfinishedFibers = unfinishedFibers, - waitTimeUpdateHook = { flowId, timeout -> resetCustomTimeout(flowId, timeout) } - ) - } - - private inline fun tryCheckpointDeserialize(bytes: SerializedBytes, flowId: StateMachineRunId): T? { - return try { - bytes.checkpointDeserialize(context = checkpointSerializationContext!!) - } catch (e: Exception) { - logger.error("Unable to deserialize checkpoint for flow $flowId. Something is very wrong and this flow will be ignored.", e) - null - } - } - private fun tryDeserializeCheckpoint(serializedCheckpoint: Checkpoint.Serialized, flowId: StateMachineRunId): Checkpoint? { return try { serializedCheckpoint.deserialize(checkpointSerializationContext!!) @@ -813,68 +782,7 @@ class SingleThreadedStateMachineManager( } } - private fun createFlowFromCheckpoint( - id: StateMachineRunId, - serializedCheckpoint: Checkpoint.Serialized, - isAnyCheckpointPersisted: Boolean, - isStartIdempotent: Boolean, - initialDeduplicationHandler: DeduplicationHandler? - ): Flow? { - val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id)?.copy(status = Checkpoint.FlowStatus.RUNNABLE) ?: return null - val resultFuture = openFuture() - val fiber = when (checkpoint.flowState) { - is FlowState.Unstarted -> { - val logic = tryCheckpointDeserialize(checkpoint.flowState.frozenFlowLogic, id) ?: return null - val state = StateMachineState( - checkpoint = checkpoint, - pendingDeduplicationHandlers = initialDeduplicationHandler?.let { listOf(it) } ?: emptyList(), - isFlowResumed = false, - isWaitingForFuture = false, - future = null, - isAnyCheckpointPersisted = isAnyCheckpointPersisted, - isStartIdempotent = isStartIdempotent, - isRemoved = false, - isKilled = false, - flowLogic = logic, - senderUUID = null - ) - val fiber = FlowStateMachineImpl(id, logic, scheduler) - fiber.transientValues = TransientReference(createTransientValues(id, resultFuture)) - fiber.transientState = TransientReference(state) - fiber.logic.stateMachine = fiber - fiber - } - is FlowState.Started -> { - val fiber = tryCheckpointDeserialize(checkpoint.flowState.frozenFiber, id) ?: return null - val state = StateMachineState( - checkpoint = checkpoint, - pendingDeduplicationHandlers = initialDeduplicationHandler?.let { listOf(it) } ?: emptyList(), - isFlowResumed = false, - isWaitingForFuture = false, - future = null, - isAnyCheckpointPersisted = isAnyCheckpointPersisted, - isStartIdempotent = isStartIdempotent, - isRemoved = false, - isKilled = false, - flowLogic = fiber.logic, - senderUUID = null - ) - fiber.transientValues = TransientReference(createTransientValues(id, resultFuture)) - fiber.transientState = TransientReference(state) - fiber.logic.stateMachine = fiber - fiber - } - is FlowState.Completed -> { - return null // Places calling this function is rely on it to return null if the flow cannot be created from the checkpoint. - } - } - - verifyFlowLogicIsSuspendable(fiber.logic) - - return Flow(fiber, resultFuture) - } - - private fun addAndStartFlow(id: StateMachineRunId, flow: Flow) { + private fun addAndStartFlow(id: StateMachineRunId, flow: Flow<*>) { val checkpoint = flow.fiber.snapshot().checkpoint for (sessionId in getFlowSessionIds(checkpoint)) { sessionToFlow[sessionId] = id @@ -899,7 +807,7 @@ class SingleThreadedStateMachineManager( } } - private fun startOrResume(checkpoint: Checkpoint, flow: Flow) { + private fun startOrResume(checkpoint: Checkpoint, flow: Flow<*>) { when (checkpoint.flowState) { is FlowState.Unstarted -> { flow.fiber.start() @@ -953,7 +861,7 @@ class SingleThreadedStateMachineManager( } private fun InnerState.removeFlowOrderly( - flow: Flow, + flow: Flow<*>, removalReason: FlowRemovalReason.OrderlyFinish, lastState: StateMachineState ) { @@ -969,7 +877,7 @@ class SingleThreadedStateMachineManager( } private fun InnerState.removeFlowError( - flow: Flow, + flow: Flow<*>, removalReason: FlowRemovalReason.ErrorFinish, lastState: StateMachineState ) { @@ -983,7 +891,7 @@ class SingleThreadedStateMachineManager( } // The flow's event queue may be non-empty in case it shut down abruptly. We handle outstanding events here. - private fun drainFlowEventQueue(flow: Flow) { + private fun drainFlowEventQueue(flow: Flow<*>) { while (true) { val event = flow.fiber.transientValues!!.value.eventQueue.tryReceive() ?: return when (event) { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt index 7fa4c22e9b..6079fbccf1 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt @@ -30,12 +30,18 @@ import java.time.Duration * TODO: Don't store all active flows in memory, load from the database on demand. */ interface StateMachineManager { + + enum class StartMode { + ExcludingPaused, // Resume all flows except paused flows. + Safe // Mark all flows as paused. + } + /** * Starts the state machine manager, loading and starting the state machines in storage. * * @return `Future` which completes when SMM is fully started */ - fun start(tokenizableServices: List) : CordaFuture + fun start(tokenizableServices: List, startMode: StartMode = StartMode.ExcludingPaused) : CordaFuture /** * Stops the state machine manager gracefully, waiting until all but [allowedUnsuspendedFiberCount] flows reach the @@ -80,6 +86,13 @@ interface StateMachineManager { */ fun killFlow(id: StateMachineRunId): Boolean + /** + * Start a paused flow. + * + * @return whether the flow was successfully started. + */ + fun unPauseFlow(id: StateMachineRunId): Boolean + /** * Deliver an external event to the state machine. Such an event might be a new P2P message, or a request to start a flow. * The event may be replayed if a flow fails and attempts to retry. diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 6ef6775c61..6c88e97ea1 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -170,9 +170,14 @@ data class Checkpoint( * @return A [Checkpoint] with all its fields filled in from [Checkpoint.Serialized] */ fun deserialize(checkpointSerializationContext: CheckpointSerializationContext): Checkpoint { + val flowState = when(status) { + FlowStatus.PAUSED -> FlowState.Paused + FlowStatus.COMPLETED -> FlowState.Completed + else -> serializedFlowState!!.checkpointDeserialize(checkpointSerializationContext) + } return Checkpoint( checkpointState = serializedCheckpointState.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), - flowState = serializedFlowState?.checkpointDeserialize(checkpointSerializationContext) ?: FlowState.Completed, + flowState = flowState, errorState = errorState, result = result?.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), status = status, @@ -307,6 +312,11 @@ sealed class FlowState { override fun toString() = "Started(flowIORequest=$flowIORequest, frozenFiber=${frozenFiber.hash})" } + /** + * The flow is paused. To save memory we don't store the FlowState + */ + object Paused: FlowState() + /** * The flow has completed. It does not have a running fiber that needs to be serialized and checkpointed. */ diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt index 1b995c7088..21b06c6e40 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt @@ -29,6 +29,7 @@ class DoRemainingWorkTransition( is FlowState.Unstarted -> UnstartedFlowTransition(context, startingState, flowState).transition() is FlowState.Started -> StartedFlowTransition(context, startingState, flowState).transition() is FlowState.Completed -> throw IllegalStateException("Cannot transition a state with completed flow state.") + is FlowState.Paused -> throw IllegalStateException("Cannot transition a state with paused flow state.") } } diff --git a/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt b/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt index 8fc0154f37..125d38f81b 100644 --- a/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt +++ b/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt @@ -32,6 +32,7 @@ class NodeStartupCliTest { Assertions.assertThat(startup.verbose).isEqualTo(false) Assertions.assertThat(startup.loggingLevel).isEqualTo(Level.INFO) Assertions.assertThat(startup.cmdLineOptions.noLocalShell).isEqualTo(false) + Assertions.assertThat(startup.cmdLineOptions.safeMode).isEqualTo(false) Assertions.assertThat(startup.cmdLineOptions.sshdServer).isEqualTo(false) Assertions.assertThat(startup.cmdLineOptions.justGenerateNodeInfo).isEqualTo(false) Assertions.assertThat(startup.cmdLineOptions.justGenerateRpcSslCerts).isEqualTo(false) diff --git a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt index 9c95fb7b10..6c8ce1d7bd 100644 --- a/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt +++ b/node/src/test/kotlin/net/corda/node/messaging/TwoPartyTradeFlowTests.kt @@ -63,7 +63,7 @@ import kotlin.test.assertFailsWith import kotlin.test.assertTrue internal fun CheckpointStorage.getAllIncompleteCheckpoints(): List { - return getRunnableCheckpoints().use { + return getCheckpointsToRun().use { it.map { it.second }.toList() }.filter { it.status != Checkpoint.FlowStatus.COMPLETED } } diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index 8ec79aa439..f47a363c16 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -5,6 +5,7 @@ import net.corda.core.context.InvocationOrigin import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.FlowIORequest +import net.corda.core.internal.toSet import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationDefaults import net.corda.core.serialization.internal.checkpointSerialize @@ -41,13 +42,13 @@ import org.junit.Ignore import org.junit.Rule import org.junit.Test import java.time.Clock -import java.util.ArrayList +import java.util.* import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertTrue internal fun CheckpointStorage.checkpoints(): List { - return getAllCheckpoints().use { + return getCheckpoints().use { it.map { it.second }.toList() } } @@ -148,18 +149,38 @@ class DBCheckpointStorageTests { checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) } - val completedCheckpoint = checkpoint.copy(flowState = FlowState.Completed) + val completedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.COMPLETED) database.transaction { checkpointStorage.updateCheckpoint(id, completedCheckpoint, null) } database.transaction { assertEquals( - completedCheckpoint, + completedCheckpoint.copy(flowState = FlowState.Completed), checkpointStorage.checkpoints().single().deserialize() ) } } + @Test(timeout = 300_000) + fun `update a checkpoint to paused`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + } + + val pausedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) + database.transaction { + checkpointStorage.updateCheckpoint(id, pausedCheckpoint, null) + } + database.transaction { + assertEquals( + pausedCheckpoint.copy(flowState = FlowState.Paused), + checkpointStorage.checkpoints().single().deserialize() + ) + } + } + @Test(timeout = 300_000) fun `removing a checkpoint deletes from all checkpoint tables`() { val exception = IllegalStateException("I am a naughty exception") @@ -641,7 +662,7 @@ class DBCheckpointStorageTests { } @Test(timeout = 300_000) - fun `fetch runnable checkpoints`() { + fun `Checkpoints can be fetched from the database by status`() { val (_, checkpoint) = newCheckpoint(1) // runnables val runnable = checkpoint.copy(status = Checkpoint.FlowStatus.RUNNABLE) @@ -650,8 +671,8 @@ class DBCheckpointStorageTests { val completed = checkpoint.copy(status = Checkpoint.FlowStatus.COMPLETED) val failed = checkpoint.copy(status = Checkpoint.FlowStatus.FAILED) val killed = checkpoint.copy(status = Checkpoint.FlowStatus.KILLED) - // tentative - val paused = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) // is considered runnable + // paused + val paused = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) database.transaction { val serializedFlowState = @@ -666,7 +687,15 @@ class DBCheckpointStorageTests { } database.transaction { - assertEquals(3, checkpointStorage.getRunnableCheckpoints().count()) + val toRunStatuses = setOf(Checkpoint.FlowStatus.RUNNABLE, Checkpoint.FlowStatus.HOSPITALIZED) + val pausedStatuses = setOf(Checkpoint.FlowStatus.PAUSED) + val customStatuses = setOf(Checkpoint.FlowStatus.RUNNABLE, Checkpoint.FlowStatus.KILLED) + val customStatuses1 = setOf(Checkpoint.FlowStatus.PAUSED, Checkpoint.FlowStatus.HOSPITALIZED, Checkpoint.FlowStatus.FAILED) + + assertEquals(toRunStatuses, checkpointStorage.getCheckpointsToRun().map { it.second.status }.toSet()) + assertEquals(pausedStatuses, checkpointStorage.getPausedCheckpoints().map { it.second.status }.toSet()) + assertEquals(customStatuses, checkpointStorage.getCheckpoints(customStatuses).map { it.second.status }.toSet()) + assertEquals(customStatuses1, checkpointStorage.getCheckpoints(customStatuses1).map { it.second.status }.toSet()) } } @@ -749,6 +778,78 @@ class DBCheckpointStorageTests { else -> throw IllegalStateException("Unknown line.separator") } + @Test(timeout = 300_000) + fun `paused checkpoints can be extracted`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + val pausedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) + database.transaction { + checkpointStorage.addCheckpoint(id, pausedCheckpoint, serializedFlowState) + } + + database.transaction { + val (extractedId, extractedCheckpoint) = checkpointStorage.getPausedCheckpoints().toList().single() + assertEquals(id, extractedId) + //We don't extract the result or the flowstate from a paused checkpoint + assertEquals(null, extractedCheckpoint.serializedFlowState) + assertEquals(null, extractedCheckpoint.result) + + assertEquals(pausedCheckpoint.status, extractedCheckpoint.status) + assertEquals(pausedCheckpoint.progressStep, extractedCheckpoint.progressStep) + assertEquals(pausedCheckpoint.flowIoRequest, extractedCheckpoint.flowIoRequest) + + val deserialisedCheckpoint = extractedCheckpoint.deserialize() + assertEquals(pausedCheckpoint.checkpointState, deserialisedCheckpoint.checkpointState) + assertEquals(FlowState.Paused, deserialisedCheckpoint.flowState) + } + } + + @Test(timeout = 300_000) + fun `checkpoints correctly change there status to paused`() { + val (_, checkpoint) = newCheckpoint(1) + // runnables + val runnable = changeStatus(checkpoint, Checkpoint.FlowStatus.RUNNABLE) + val hospitalized = changeStatus(checkpoint, Checkpoint.FlowStatus.HOSPITALIZED) + // not runnables + val completed = changeStatus(checkpoint, Checkpoint.FlowStatus.COMPLETED) + val failed = changeStatus(checkpoint, Checkpoint.FlowStatus.FAILED) + val killed = changeStatus(checkpoint, Checkpoint.FlowStatus.KILLED) + // paused + val paused = changeStatus(checkpoint, Checkpoint.FlowStatus.PAUSED) + database.transaction { + val serializedFlowState = + checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + + checkpointStorage.addCheckpoint(runnable.id, runnable.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(hospitalized.id, hospitalized.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(completed.id, completed.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(failed.id, failed.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(killed.id, killed.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(paused.id, paused.checkpoint, serializedFlowState) + } + + database.transaction { + checkpointStorage.markAllPaused() + } + + database.transaction { + //Hospitalised and paused checkpoints status should update + assertEquals(Checkpoint.FlowStatus.PAUSED, checkpointStorage.getDBCheckpoint(runnable.id)!!.status) + assertEquals(Checkpoint.FlowStatus.PAUSED, checkpointStorage.getDBCheckpoint(hospitalized.id)!!.status) + //Other checkpoints should not be updated + assertEquals(Checkpoint.FlowStatus.COMPLETED, checkpointStorage.getDBCheckpoint(completed.id)!!.status) + assertEquals(Checkpoint.FlowStatus.FAILED, checkpointStorage.getDBCheckpoint(failed.id)!!.status) + assertEquals(Checkpoint.FlowStatus.KILLED, checkpointStorage.getDBCheckpoint(killed.id)!!.status) + assertEquals(Checkpoint.FlowStatus.PAUSED, checkpointStorage.getDBCheckpoint(paused.id)!!.status) + } + } + + data class IdAndCheckpoint(val id: StateMachineRunId, val checkpoint: Checkpoint) + + private fun changeStatus(oldCheckpoint: Checkpoint, status: Checkpoint.FlowStatus): IdAndCheckpoint { + return IdAndCheckpoint(StateMachineRunId.createRandom(), oldCheckpoint.copy(status = status)) + } + private fun newCheckpointStorage() { database.transaction { checkpointStorage = DBCheckpointStorage( diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 72ea1fbdd6..2b84537bdc 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -682,14 +682,14 @@ class FlowFrameworkTests { if (firstExecution) { throw HospitalizeFlowException() } else { - dbCheckpointStatusBeforeSuspension = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + dbCheckpointStatusBeforeSuspension = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status inMemoryCheckpointStatusBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status futureFiber.complete(flowFiber) } } SuspendingFlow.hookAfterCheckpoint = { - dbCheckpointStatusAfterSuspension = aliceNode.internals.checkpointStorage.getRunnableCheckpoints().toList().single() + dbCheckpointStatusAfterSuspension = aliceNode.internals.checkpointStorage.getCheckpointsToRun().toList().single() .second.status } @@ -701,7 +701,7 @@ class FlowFrameworkTests { val inMemoryHospitalizedCheckpointStatus = aliceNode.internals.smm.snapshot().first().transientState?.value?.checkpoint?.status assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, inMemoryHospitalizedCheckpointStatus) aliceNode.database.transaction { - val checkpoint = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second + val checkpoint = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, checkpoint.status) } // restart Node - flow will be loaded from checkpoint @@ -729,7 +729,7 @@ class FlowFrameworkTests { if (firstExecution) { throw HospitalizeFlowException() } else { - dbCheckpointStatus = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + dbCheckpointStatus = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status inMemoryCheckpointStatus = flowFiber.transientState!!.value.checkpoint.status futureFiber.complete(flowFiber) @@ -742,7 +742,7 @@ class FlowFrameworkTests { // flow is in hospital assertTrue(flowState is FlowState.Started) aliceNode.database.transaction { - val checkpoint = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second + val checkpoint = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second assertEquals(Checkpoint.FlowStatus.HOSPITALIZED, checkpoint.status) } // restart Node - flow will be loaded from checkpoint @@ -812,7 +812,7 @@ class FlowFrameworkTests { throw SQLTransientConnectionException("connection is not available") } else { val flowFiber = this as? FlowStateMachineImpl<*> - dbCheckpointStatus = aliceNode.internals.checkpointStorage.getAllCheckpoints().toList().single().second.status + dbCheckpointStatus = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status inMemoryCheckpointStatus = flowFiber!!.transientState!!.value.checkpoint.status persistedException = aliceNode.internals.checkpointStorage.getDBCheckpoint(flowFiber.id)!!.exceptionDetails } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowPausingTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowPausingTests.kt new file mode 100644 index 0000000000..1a0415892b --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowPausingTests.kt @@ -0,0 +1,77 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import net.corda.core.flows.FlowLogic +import net.corda.core.internal.FlowStateMachine +import net.corda.node.services.config.NodeConfiguration +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.node.internal.InternalMockNetwork +import net.corda.testing.node.internal.InternalMockNodeParameters +import net.corda.testing.node.internal.TestStartedNode +import net.corda.testing.node.internal.startFlow +import org.junit.After +import org.junit.Before +import org.junit.Test +import java.time.Duration +import kotlin.test.assertEquals + +class FlowPausingTests { + + companion object { + const val NUMBER_OF_FLOWS = 4 + const val SLEEP_TIME = 1000L + } + + private lateinit var mockNet: InternalMockNetwork + private lateinit var aliceNode: TestStartedNode + private lateinit var bobNode: TestStartedNode + + @Before + fun setUpMockNet() { + mockNet = InternalMockNetwork() + aliceNode = mockNet.createNode(InternalMockNodeParameters(legalName = ALICE_NAME)) + bobNode = mockNet.createNode(InternalMockNodeParameters(legalName = BOB_NAME)) + } + + @After + fun cleanUp() { + mockNet.stopNodes() + } + + private fun restartNode(node: TestStartedNode, smmStartMode: StateMachineManager.StartMode) : TestStartedNode { + val parameters = InternalMockNodeParameters(configOverrides = { + conf: NodeConfiguration -> + doReturn(smmStartMode).whenever(conf).smmStartMode + }) + return mockNet.restartNode(node, parameters = parameters) + } + + @Test(timeout = 300_000) + fun `All are paused when the node is restarted in safe start mode`() { + val flows = ArrayList>() + for (i in 1..NUMBER_OF_FLOWS) { + flows += aliceNode.services.startFlow(CheckpointingFlow()) + } + //All of the flows must not resume before the node restarts. + val restartedAlice = restartNode(aliceNode, StateMachineManager.StartMode.Safe) + assertEquals(0, restartedAlice.smm.snapshot().size) + //We need to wait long enough here so any running flows would finish. + Thread.sleep(NUMBER_OF_FLOWS * SLEEP_TIME) + restartedAlice.database.transaction { + for (flow in flows) { + val checkpoint = restartedAlice.internals.checkpointStorage.getCheckpoint(flow.id) + assertEquals(Checkpoint.FlowStatus.PAUSED, checkpoint!!.status) + } + } + } + + internal class CheckpointingFlow: FlowLogic() { + @Suspendable + override fun call() { + sleep(Duration.ofMillis(SLEEP_TIME)) + } + } +} diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt index aea0e9d5d0..857a171373 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt @@ -638,6 +638,7 @@ private fun mockNodeConfiguration(certificatesDirectory: Path): NodeConfiguratio doReturn(NetworkParameterAcceptanceSettings()).whenever(it).networkParameterAcceptanceSettings doReturn(rigorousMock()).whenever(it).configurationWithOptions doReturn(2).whenever(it).flowExternalOperationThreadPoolSize + doReturn(StateMachineManager.StartMode.ExcludingPaused).whenever(it).smmStartMode } } From e34930f9a8fba962bfb6d43cb7d07dc1e5888ada Mon Sep 17 00:00:00 2001 From: Ramzi El-Yafi Date: Wed, 20 May 2020 22:07:04 +0100 Subject: [PATCH 39/49] [INFRA-351] Jenkinsfile for code check jobs (#6272) --- .ci/dev/pr-code-checks/Jenkinsfile | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .ci/dev/pr-code-checks/Jenkinsfile diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile new file mode 100644 index 0000000000..0fdd5b0055 --- /dev/null +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -0,0 +1,69 @@ +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +pipeline { + agent { label 'k8s' } + options { + timestamps() + timeout(time: 3, unit: 'HOURS') + } + + environment { + PR_CONTEXT_STRING = "PR Code Checks" + } + + stages { + stage('Detekt check') { + steps { + script { + pullRequest.createStatus( + status: 'pending', + context: "${PR_CONTEXT_STRING}", + description: "Running code checks", + targetUrl: "${env.BUILD_URL}") + } + sh "./gradlew --no-daemon clean detekt" + } + } + + stage('Compilation warnings check') { + steps { + sh "./gradlew --no-daemon -Pcompilation.warningsAsErrors=true compileAll" + } + } + + stage('No API change check') { + steps { + sh "./gradlew --no-daemon generateApi" + sh ".ci/check-api-changes.sh" + } + } + } + + post { + success { + script { + pullRequest.createStatus( + status: 'success', + context: "${PR_CONTEXT_STRING}", + description: 'Code checks passed', + targetUrl: "${env.BUILD_URL}") + } + } + + failure { + script { + pullRequest.createStatus( + status: 'failure', + context: "${PR_CONTEXT_STRING}", + description: 'Code checks failed', + targetUrl: "${env.BUILD_URL}") + } + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} From 326afacedbbeaf0f24a9507b08836e36c884dacc Mon Sep 17 00:00:00 2001 From: Walter Oggioni <6357328+woggioni@users.noreply.github.com> Date: Thu, 21 May 2020 09:27:13 +0100 Subject: [PATCH 40/49] improved interactive shell flow lookup logic (#6271) the current logic when a user type ``` flow start SomeFlow ``` is to search for the string `SomeFlow` in all registered flow names, if there is at exactly flow that ends with that string, it will be selected, if there is more than one, the first in aphabetical order will be selected. If there are multiple flows that contains the string but no one that ends with it, it is considered ambiguous, if there is exactly one containing the string it is instead selected (it doesn't matter whether it ends or not with it). All other cases are considered errors. The goal of this change is to address the case where the node contains ``` net.corda.cordappp.AnotherSomeFlow net.corda.cordappp.ImprovedSomeFlow net.corda.cordappp.SomeFlow ``` typing ``` flow start SomeFlow ``` currently results in running `net.corda.cordappp.AnotherSomeFlow` because it comes first in alphabetical order, while `net.corda.cordappp.SomeFlow` is a better matches because the input substring matches a bigger portion of the full flow name --- .../src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt b/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt index ee26e6016b..bc00b7b53a 100644 --- a/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt +++ b/tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt @@ -336,7 +336,7 @@ object InteractiveShell { ansiProgressRenderer: ANSIProgressRenderer, inputObjectMapper: ObjectMapper = createYamlInputMapper(rpcOps)) { val matches = try { - rpcOps.registeredFlows().filter { nameFragment in it } + rpcOps.registeredFlows().filter { nameFragment in it }.sortedBy { it.length } } catch (e: PermissionException) { output.println(e.message ?: "Access denied", Decoration.bold, Color.red) return From 8e74eea6072f06b571f7bac1cc3310e166fd7057 Mon Sep 17 00:00:00 2001 From: nikinagy <61757742+nikinagy@users.noreply.github.com> Date: Thu, 21 May 2020 13:26:55 +0100 Subject: [PATCH 41/49] CORDA-3587 - adding kdocs for current behaviour of VaultQueryCriteria (#6242) * adding kdocs for current behaviour of VaultQueryCriteria * improving the kdocs * address PR comments --- .../core/node/services/vault/QueryCriteria.kt | 92 +++++++++++++++++-- 1 file changed, 86 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt b/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt index 91bfe9bc0d..059e801730 100644 --- a/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt +++ b/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt @@ -169,19 +169,93 @@ sealed class QueryCriteria : GenericQueryCriteria>): VaultQueryCriteria = copy(contractStateTypes = contractStateTypes) - fun withStateRefs(stateRefs: List): VaultQueryCriteria = copy(stateRefs = stateRefs) - fun withNotary(notary: List): VaultQueryCriteria = copy(notary = notary) - fun withSoftLockingCondition(softLockingCondition: SoftLockingCondition): VaultQueryCriteria = copy(softLockingCondition = softLockingCondition) - fun withTimeCondition(timeCondition: TimeCondition): VaultQueryCriteria = copy(timeCondition = timeCondition) + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [relevancyStatus]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withRelevancyStatus(relevancyStatus: Vault.RelevancyStatus): VaultQueryCriteria = copy(relevancyStatus = relevancyStatus) + + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [constraintTypes]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withConstraintTypes(constraintTypes: Set): VaultQueryCriteria = copy(constraintTypes = constraintTypes) + + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [constraints]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withConstraints(constraints: Set): VaultQueryCriteria = copy(constraints = constraints) + + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [participants]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withParticipants(participants: List): VaultQueryCriteria = copy(participants = participants) + + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [externalIds]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withExternalIds(externalIds: List): VaultQueryCriteria = copy(externalIds = externalIds) + + /** + * This function creates a new [VaultQueryCriteria] object with default values, and sets the value of [exactParticipants]. + * Please use only one function in this group at a time to make sure they are not overwriting each other. + */ fun withExactParticipants(exactParticipants: List): VaultQueryCriteria = copy(exactParticipants = exactParticipants) + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [status]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withStatus(status: Vault.StateStatus): VaultQueryCriteria = copy(status = status) + + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [contractStateTypes]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withContractStateTypes(contractStateTypes: Set>): VaultQueryCriteria = copy(contractStateTypes = contractStateTypes) + + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [stateRefs]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withStateRefs(stateRefs: List): VaultQueryCriteria = copy(stateRefs = stateRefs) + + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [notary]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withNotary(notary: List): VaultQueryCriteria = copy(notary = notary) + + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [softLockingCondition]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withSoftLockingCondition(softLockingCondition: SoftLockingCondition): VaultQueryCriteria = copy(softLockingCondition = softLockingCondition) + + /** + * This function copies the existing [VaultQueryCriteria] object and sets the given value for [timeCondition]. + * You can use more than one of the functions in this group together. + * In case you are also using a function that creates a new [VaultQueryCriteria] object, make sure that you are + * calling that method first. + */ + fun withTimeCondition(timeCondition: TimeCondition): VaultQueryCriteria = copy(timeCondition = timeCondition) + + /** + * This function creates a [VaultQueryCriteria] object with the given values. All other fields have the default values set. + */ fun copy( status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED, contractStateTypes: Set>? = null, @@ -211,6 +285,9 @@ sealed class QueryCriteria : GenericQueryCriteria>? = null, @@ -238,6 +315,9 @@ sealed class QueryCriteria : GenericQueryCriteria>? = this.contractStateTypes, From 57b1a5e0fc4af55dea02d4066c74cd3f01b34480 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Fri, 22 May 2020 10:15:51 +0100 Subject: [PATCH 42/49] ENT-5339 Failing tests against Oracle in VaultObserverExceptionTest (#6275) * Fix erroneous sql statement for oracle; It was failing tests with 'ORA-00933: SQL command not properly ended' * Fixed flaky test; it didn't wait for counter party flow to get hospitalized as the test implied --- .../vault/VaultObserverExceptionTest.kt | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt index 5cd4529c6d..86bb3b2931 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt @@ -24,6 +24,7 @@ import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.seconds import net.corda.node.services.Permissions import net.corda.node.services.statemachine.StaffedFlowHospital +import net.corda.node.services.transactions.PersistentUniquenessProvider import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.singleIdentity @@ -598,7 +599,12 @@ class VaultObserverExceptionTest { @Test(timeout=300_000) fun `Throw user error in VaultService rawUpdates during counterparty FinalityFlow blows up the flow but does not break the Observer`() { var observationCounter = 0 - StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> ++observationCounter } + // Semaphore is used to wait until [PassErroneousOwnableStateReceiver] gets hospitalized, only after that moment let testing thread assert 'observationCounter' + val counterPartyHospitalized = Semaphore(0) + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + ++observationCounter + counterPartyHospitalized.release() + } val rawUpdatesCount = ConcurrentHashMap() DbListenerService.onNextVisited = { party -> @@ -644,6 +650,7 @@ class VaultObserverExceptionTest { assertEquals(1, aliceNode.getStatesById(stateId, Vault.StateStatus.CONSUMED).size) assertEquals(0, bobNode.getStatesById(stateId, Vault.StateStatus.UNCONSUMED).size) assertEquals(1, notary.getNotarisedTransactionIds().size) + counterPartyHospitalized.acquire() assertEquals(1, observationCounter) assertEquals(2, rawUpdatesCount[aliceNode.nodeInfo.singleIdentity()]) assertEquals(1, rawUpdatesCount[bobNode.nodeInfo.singleIdentity()]) @@ -653,6 +660,7 @@ class VaultObserverExceptionTest { assertEquals(2, aliceNode.getAllStates(Vault.StateStatus.CONSUMED).size) assertEquals(0, bobNode.getStatesById(stateId2, Vault.StateStatus.UNCONSUMED).size) assertEquals(2, notary.getNotarisedTransactionIds().size) + counterPartyHospitalized.acquire() assertEquals(2, observationCounter) assertEquals(4, rawUpdatesCount[aliceNode.nodeInfo.singleIdentity()]) assertEquals(2, rawUpdatesCount[bobNode.nodeInfo.singleIdentity()]) @@ -833,14 +841,13 @@ class VaultObserverExceptionTest { @StartableByRPC class NotarisedTxs : FlowLogic>() { override fun call(): List { - val session = serviceHub.jdbcSession() - val statement = session.createStatement() - statement.execute("SELECT TRANSACTION_ID FROM NODE_NOTARY_COMMITTED_TXS;") - val result = mutableListOf() - while (statement.resultSet.next()) { - result.add(statement.resultSet.getString(1)) + return serviceHub.withEntityManager { + val criteriaQuery = this.criteriaBuilder.createQuery(String::class.java) + val root = criteriaQuery.from(PersistentUniquenessProvider.CommittedTransaction::class.java) + criteriaQuery.select(root.get(PersistentUniquenessProvider.CommittedTransaction::transactionId.name)) + val query = this.createQuery(criteriaQuery) + query.resultList } - return result } } From 330a95cb6872ffdf8d2d1ecd3110a828286d9923 Mon Sep 17 00:00:00 2001 From: Dan Newton Date: Tue, 26 May 2020 11:32:42 +0100 Subject: [PATCH 43/49] ENT-4627 Log transaction context missing exception (#6278) When a flow is missing its database transaction the _real_ stacktrace of the exception is missing in the logs. This is due to the normal error handling path way does not work due the transaction being missing. When it goes to process the next error event (due to the original transaction context missing error) it will fail for the same error as it needs the context to be able to process the error event. The extra logging should help diagnose future errors. --- .../node/services/statemachine/FlowStateMachineImpl.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index b86aae1d81..c76d4aa2e9 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -303,7 +303,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, if(t.isUnrecoverable()) { errorAndTerminate("Caught unrecoverable error from flow. Forcibly terminating the JVM, this might leave resources open, and most likely will.", t) } - logger.info("Flow raised an error: ${t.message}. Sending it to flow hospital to be triaged.") + logFlowError(t) Try.Failure(t) } val softLocksId = if (hasSoftLockedStates) logic.runId.uuid else null @@ -342,6 +342,14 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, ) } + private fun logFlowError(throwable: Throwable) { + if (contextTransactionOrNull != null) { + logger.info("Flow raised an error: ${throwable.message}. Sending it to flow hospital to be triaged.") + } else { + logger.error("Flow raised an error: ${throwable.message}. The flow's database transaction is missing.", throwable) + } + } + @Suspendable override fun subFlow(currentFlow: FlowLogic<*>, subFlow: FlowLogic): R { subFlow.stateMachine = this From 4ed57506c823d5b23658a6a53fb192782e8015a7 Mon Sep 17 00:00:00 2001 From: Ramzi El-Yafi Date: Tue, 26 May 2020 15:26:55 +0100 Subject: [PATCH 44/49] [INFRA-352] Artifactory publication in Jenkins (#6276) * [INFRA-352] Artifactory publication in Jenkins * Address review comments --- .ci/dev/publish-branch/Jenkinsfile.nightly | 69 ++++++++++++++++++++++ .ci/dev/publish-branch/Jenkinsfile.preview | 65 ++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 .ci/dev/publish-branch/Jenkinsfile.nightly create mode 100644 .ci/dev/publish-branch/Jenkinsfile.preview diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly new file mode 100644 index 0000000000..460117e500 --- /dev/null +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -0,0 +1,69 @@ +#!groovy +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +pipeline { + agent { label 'k8s' } + + options { + timestamps() + ansiColor('xterm') + overrideIndexTriggers(false) + buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) + timeout(time: 3, unit: 'HOURS') + } + + triggers { + cron '@midnight' + } + + environment { + // Replace / with :: as links from Jenkins to Artifactory are broken if we use slashes + // in the name + ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Nightly to Artifactory" + .replaceAll("/", " :: ") + } + + stages { + stage('Publish to Artifactory') { + steps { + rtServer ( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer ( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'corda-dev', + ) + withCredentials([ + usernamePassword(credentialsId: 'artifactory-credentials', + usernameVariable: 'CORDA_ARTIFACTORY_USERNAME', + passwordVariable: 'CORDA_ARTIFACTORY_PASSWORD')]) { + rtGradleRun ( + usesPlugin: true, + useWrapper: true, + switches: "--no-daemon -s", + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + rtPublishBuildInfo ( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } + } + + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} diff --git a/.ci/dev/publish-branch/Jenkinsfile.preview b/.ci/dev/publish-branch/Jenkinsfile.preview new file mode 100644 index 0000000000..1b39ae3237 --- /dev/null +++ b/.ci/dev/publish-branch/Jenkinsfile.preview @@ -0,0 +1,65 @@ +#!groovy +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +pipeline { + agent { label 'k8s' } + + options { + timestamps() + ansiColor('xterm') + overrideIndexTriggers(false) + buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) + timeout(time: 3, unit: 'HOURS') + } + + environment { + // Replace / with :: as links from Jenkins to Artifactory are broken if we use slashes + // in the name + ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Preview to Artifactory" + .replaceAll("/", " :: ") + } + + stages { + stage('Publish to Artifactory') { + steps { + rtServer ( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer ( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'corda-dev', + ) + withCredentials([ + usernamePassword(credentialsId: 'artifactory-credentials', + usernameVariable: 'CORDA_ARTIFACTORY_USERNAME', + passwordVariable: 'CORDA_ARTIFACTORY_PASSWORD')]) { + rtGradleRun ( + usesPlugin: true, + useWrapper: true, + switches: "--no-daemon -s -PversionFromGit", + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + rtPublishBuildInfo ( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } + } + + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} From 6ebc6e9b16575a7afbb781b0d93a8f04b3affba5 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Tue, 26 May 2020 15:46:29 +0100 Subject: [PATCH 45/49] CORDA-3750: Reimplement Corda's Crypto object for use inside the sandbox. (#6193) * CORDA-3750: Use hand-written sandbox Crypto object that delegates to the node. * CORDA-3750: Add integration test for deterministic CashIssueAndPayment flow. * Tidy up generics for Array instances. * Upgrade to DJVM 1.1-RC04. --- constants.properties | 2 +- node/build.gradle | 5 + .../kotlin/net/corda/node/djvm/LtxFactory.kt | 8 +- .../DeterministicCashIssueAndPaymentTest.kt | 68 +++++ .../java/sandbox/java/lang/CharSequence.java | 8 + .../java/sandbox/java/lang/Comparable.java | 8 + .../main/java/sandbox/java/lang/Number.java | 8 + .../java/sandbox/java/math/BigInteger.java | 8 + .../main/java/sandbox/java/security/Key.java | 13 + .../java/sandbox/java/security/KeyPair.java | 8 + .../sandbox/java/security/PrivateKey.java | 8 + .../java/sandbox/java/security/PublicKey.java | 8 + .../security/spec/AlgorithmParameterSpec.java | 8 + .../java/sandbox/java/util/ArrayList.java | 16 ++ .../src/main/java/sandbox/java/util/List.java | 9 + .../sandbox/net/corda/core/crypto/DJVM.java | 62 +++++ .../net/corda/core/crypto/DJVMPublicKey.java | 61 ++++ .../org/bouncycastle/asn1/ASN1Encodable.java | 9 + .../org/bouncycastle/asn1/ASN1Object.java | 16 ++ .../asn1/x509/AlgorithmIdentifier.java | 14 + .../asn1/x509/SubjectPublicKeyInfo.java | 10 + .../corda/node/internal/djvm/Serializer.kt | 2 +- .../DeterministicVerifierFactoryService.kt | 15 +- .../sandbox/net/corda/core/crypto/Crypto.kt | 261 ++++++++++++++++++ .../net/corda/core/crypto/SecureHash.kt | 8 + .../net/corda/core/crypto/SignatureScheme.kt | 26 ++ .../corda/core/crypto/TransactionSignature.kt | 7 + .../corda/core/crypto/internal/ProviderMap.kt | 8 + .../serializers/SandboxPublicKeySerializer.kt | 3 +- .../djvm/DeserializePublicKeyTest.kt | 20 +- 30 files changed, 694 insertions(+), 13 deletions(-) create mode 100644 node/src/integration-test/kotlin/net/corda/node/services/DeterministicCashIssueAndPaymentTest.kt create mode 100644 node/src/main/java/sandbox/java/lang/CharSequence.java create mode 100644 node/src/main/java/sandbox/java/lang/Comparable.java create mode 100644 node/src/main/java/sandbox/java/lang/Number.java create mode 100644 node/src/main/java/sandbox/java/math/BigInteger.java create mode 100644 node/src/main/java/sandbox/java/security/Key.java create mode 100644 node/src/main/java/sandbox/java/security/KeyPair.java create mode 100644 node/src/main/java/sandbox/java/security/PrivateKey.java create mode 100644 node/src/main/java/sandbox/java/security/PublicKey.java create mode 100644 node/src/main/java/sandbox/java/security/spec/AlgorithmParameterSpec.java create mode 100644 node/src/main/java/sandbox/java/util/ArrayList.java create mode 100644 node/src/main/java/sandbox/java/util/List.java create mode 100644 node/src/main/java/sandbox/net/corda/core/crypto/DJVM.java create mode 100644 node/src/main/java/sandbox/net/corda/core/crypto/DJVMPublicKey.java create mode 100644 node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Encodable.java create mode 100644 node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Object.java create mode 100644 node/src/main/java/sandbox/org/bouncycastle/asn1/x509/AlgorithmIdentifier.java create mode 100644 node/src/main/java/sandbox/org/bouncycastle/asn1/x509/SubjectPublicKeyInfo.java create mode 100644 node/src/main/kotlin/sandbox/net/corda/core/crypto/Crypto.kt create mode 100644 node/src/main/kotlin/sandbox/net/corda/core/crypto/SecureHash.kt create mode 100644 node/src/main/kotlin/sandbox/net/corda/core/crypto/SignatureScheme.kt create mode 100644 node/src/main/kotlin/sandbox/net/corda/core/crypto/TransactionSignature.kt create mode 100644 node/src/main/kotlin/sandbox/net/corda/core/crypto/internal/ProviderMap.kt diff --git a/constants.properties b/constants.properties index 6865651962..131f2ae5f8 100644 --- a/constants.properties +++ b/constants.properties @@ -30,7 +30,7 @@ snakeYamlVersion=1.19 caffeineVersion=2.7.0 metricsVersion=4.1.0 metricsNewRelicVersion=1.1.1 -djvmVersion=1.1-RC03 +djvmVersion=1.1-RC04 deterministicRtVersion=1.0-RC02 openSourceBranch=https://github.com/corda/corda/blob/release/os/4.4 openSourceSamplesBranch=https://github.com/corda/samples/blob/release-V4 diff --git a/node/build.gradle b/node/build.gradle index ae98903907..58f7a5498e 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -304,6 +304,11 @@ quasar { jar { baseName 'corda-node' + exclude 'sandbox/java/**' + exclude 'sandbox/org/**' + exclude 'sandbox/net/corda/core/crypto/SecureHash.class' + exclude 'sandbox/net/corda/core/crypto/SignatureScheme.class' + exclude 'sandbox/net/corda/core/crypto/TransactionSignature.class' manifest { attributes('Corda-Deterministic-Runtime': configurations.jdkRt.singleFile.name) attributes('Corda-Deterministic-Classpath': configurations.deterministic.collect { it.name }.join(' ')) diff --git a/node/djvm/src/main/kotlin/net/corda/node/djvm/LtxFactory.kt b/node/djvm/src/main/kotlin/net/corda/node/djvm/LtxFactory.kt index dc3affd9e2..31c1b2aaa8 100644 --- a/node/djvm/src/main/kotlin/net/corda/node/djvm/LtxFactory.kt +++ b/node/djvm/src/main/kotlin/net/corda/node/djvm/LtxFactory.kt @@ -27,12 +27,12 @@ private const val TX_PRIVACY_SALT = 7 private const val TX_NETWORK_PARAMETERS = 8 private const val TX_REFERENCES = 9 -class LtxFactory : Function, LedgerTransaction> { +class LtxFactory : Function, LedgerTransaction> { @Suppress("unchecked_cast") - override fun apply(txArgs: Array): LedgerTransaction { + override fun apply(txArgs: Array): LedgerTransaction { return LedgerTransaction.createForSandbox( - inputs = (txArgs[TX_INPUTS] as Array>).map { it.toStateAndRef() }, + inputs = (txArgs[TX_INPUTS] as Array>).map { it.toStateAndRef() }, outputs = (txArgs[TX_OUTPUTS] as? List>) ?: emptyList(), commands = (txArgs[TX_COMMANDS] as? List>) ?: emptyList(), attachments = (txArgs[TX_ATTACHMENTS] as? List) ?: emptyList(), @@ -41,7 +41,7 @@ class LtxFactory : Function, LedgerTransaction> { timeWindow = txArgs[TX_TIME_WINDOW] as? TimeWindow, privacySalt = txArgs[TX_PRIVACY_SALT] as PrivacySalt, networkParameters = txArgs[TX_NETWORK_PARAMETERS] as NetworkParameters, - references = (txArgs[TX_REFERENCES] as Array>).map { it.toStateAndRef() } + references = (txArgs[TX_REFERENCES] as Array>).map { it.toStateAndRef() } ) } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/DeterministicCashIssueAndPaymentTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/DeterministicCashIssueAndPaymentTest.kt new file mode 100644 index 0000000000..0de1960375 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/DeterministicCashIssueAndPaymentTest.kt @@ -0,0 +1,68 @@ +package net.corda.node.services + +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.loggerFor +import net.corda.finance.DOLLARS +import net.corda.finance.flows.CashIssueAndPaymentFlow +import net.corda.node.DeterministicSourcesRule +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.driver.internal.incrementalPortAllocation +import net.corda.testing.node.NotarySpec +import net.corda.testing.node.internal.findCordapp +import org.junit.ClassRule +import org.junit.Test +import org.junit.jupiter.api.assertDoesNotThrow + +@Suppress("FunctionName") +class DeterministicCashIssueAndPaymentTest { + companion object { + val logger = loggerFor() + + @ClassRule + @JvmField + val djvmSources = DeterministicSourcesRule() + + @JvmField + val CASH_AMOUNT = 500.DOLLARS + + fun parametersFor(djvmSources: DeterministicSourcesRule): DriverParameters { + return DriverParameters( + portAllocation = incrementalPortAllocation(), + startNodesInProcess = false, + notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, validating = true)), + cordappsForAllNodes = listOf( + findCordapp("net.corda.finance.contracts"), + findCordapp("net.corda.finance.workflows") + ), + djvmBootstrapSource = djvmSources.bootstrap, + djvmCordaSource = djvmSources.corda + ) + } + } + + @Test(timeout = 300_000) + fun `test DJVM can issue cash`() { + val reference = OpaqueBytes.of(0x01) + driver(parametersFor(djvmSources)) { + val alice = startNode(providedName = ALICE_NAME).getOrThrow() + val aliceParty = alice.nodeInfo.singleIdentity() + val notaryParty = notaryHandles.single().identity + val txId = assertDoesNotThrow { + alice.rpc.startFlow(::CashIssueAndPaymentFlow, + CASH_AMOUNT, + reference, + aliceParty, + false, + notaryParty + ).returnValue.getOrThrow() + } + logger.info("TX-ID: {}", txId) + } + } +} diff --git a/node/src/main/java/sandbox/java/lang/CharSequence.java b/node/src/main/java/sandbox/java/lang/CharSequence.java new file mode 100644 index 0000000000..9b62762a11 --- /dev/null +++ b/node/src/main/java/sandbox/java/lang/CharSequence.java @@ -0,0 +1,8 @@ +package sandbox.java.lang; + +/** + * This is a dummy class that implements just enough of {@link java.lang.CharSequence} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface CharSequence extends java.lang.CharSequence { +} diff --git a/node/src/main/java/sandbox/java/lang/Comparable.java b/node/src/main/java/sandbox/java/lang/Comparable.java new file mode 100644 index 0000000000..5ca4f4871c --- /dev/null +++ b/node/src/main/java/sandbox/java/lang/Comparable.java @@ -0,0 +1,8 @@ +package sandbox.java.lang; + +/** + * This is a dummy class that implements just enough of {@link java.lang.Comparable} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface Comparable extends java.lang.Comparable { +} diff --git a/node/src/main/java/sandbox/java/lang/Number.java b/node/src/main/java/sandbox/java/lang/Number.java new file mode 100644 index 0000000000..a98d60dbd6 --- /dev/null +++ b/node/src/main/java/sandbox/java/lang/Number.java @@ -0,0 +1,8 @@ +package sandbox.java.lang; + +/** + * This is a dummy class that implements just enough of {@link java.lang.Number} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public class Number extends Object { +} diff --git a/node/src/main/java/sandbox/java/math/BigInteger.java b/node/src/main/java/sandbox/java/math/BigInteger.java new file mode 100644 index 0000000000..d58328fd3c --- /dev/null +++ b/node/src/main/java/sandbox/java/math/BigInteger.java @@ -0,0 +1,8 @@ +package sandbox.java.math; + +/** + * This is a dummy class that implements just enough of {@link java.math.BigInteger} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public class BigInteger extends sandbox.java.lang.Number { +} diff --git a/node/src/main/java/sandbox/java/security/Key.java b/node/src/main/java/sandbox/java/security/Key.java new file mode 100644 index 0000000000..9c5c952852 --- /dev/null +++ b/node/src/main/java/sandbox/java/security/Key.java @@ -0,0 +1,13 @@ +package sandbox.java.security; + +import sandbox.java.lang.String; + +/** + * This is a dummy class that implements just enough of {@link java.security.Key} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface Key { + String getAlgorithm(); + String getFormat(); + byte[] getEncoded(); +} diff --git a/node/src/main/java/sandbox/java/security/KeyPair.java b/node/src/main/java/sandbox/java/security/KeyPair.java new file mode 100644 index 0000000000..653e27de8e --- /dev/null +++ b/node/src/main/java/sandbox/java/security/KeyPair.java @@ -0,0 +1,8 @@ +package sandbox.java.security; + +/** + * This is a dummy class that implements just enough of {@link java.security.KeyPair} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public final class KeyPair extends sandbox.java.lang.Object implements java.io.Serializable { +} \ No newline at end of file diff --git a/node/src/main/java/sandbox/java/security/PrivateKey.java b/node/src/main/java/sandbox/java/security/PrivateKey.java new file mode 100644 index 0000000000..a314aa6234 --- /dev/null +++ b/node/src/main/java/sandbox/java/security/PrivateKey.java @@ -0,0 +1,8 @@ +package sandbox.java.security; + +/** + * This is a dummy class that implements just enough of {@link java.security.PrivateKey} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface PrivateKey extends Key { +} diff --git a/node/src/main/java/sandbox/java/security/PublicKey.java b/node/src/main/java/sandbox/java/security/PublicKey.java new file mode 100644 index 0000000000..451f33141a --- /dev/null +++ b/node/src/main/java/sandbox/java/security/PublicKey.java @@ -0,0 +1,8 @@ +package sandbox.java.security; + +/** + * This is a dummy class that implements just enough of {@link java.security.PublicKey} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface PublicKey extends Key { +} diff --git a/node/src/main/java/sandbox/java/security/spec/AlgorithmParameterSpec.java b/node/src/main/java/sandbox/java/security/spec/AlgorithmParameterSpec.java new file mode 100644 index 0000000000..7943cf4612 --- /dev/null +++ b/node/src/main/java/sandbox/java/security/spec/AlgorithmParameterSpec.java @@ -0,0 +1,8 @@ +package sandbox.java.security.spec; + +/** + * This is a dummy class that implements just enough of {@link java.security.spec.AlgorithmParameterSpec} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface AlgorithmParameterSpec { +} diff --git a/node/src/main/java/sandbox/java/util/ArrayList.java b/node/src/main/java/sandbox/java/util/ArrayList.java new file mode 100644 index 0000000000..7eb5df9f09 --- /dev/null +++ b/node/src/main/java/sandbox/java/util/ArrayList.java @@ -0,0 +1,16 @@ +package sandbox.java.util; + +/** + * This is a dummy class that implements just enough of {@link java.util.ArrayList} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +@SuppressWarnings("unused") +public class ArrayList extends sandbox.java.lang.Object implements List { + public ArrayList(int size) { + } + + @Override + public boolean add(T item) { + throw new UnsupportedOperationException("Dummy class - not implemented"); + } +} diff --git a/node/src/main/java/sandbox/java/util/List.java b/node/src/main/java/sandbox/java/util/List.java new file mode 100644 index 0000000000..0f7dbfe22c --- /dev/null +++ b/node/src/main/java/sandbox/java/util/List.java @@ -0,0 +1,9 @@ +package sandbox.java.util; + +/** + * This is a dummy class that implements just enough of {@link java.util.List} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public interface List { + boolean add(T item); +} diff --git a/node/src/main/java/sandbox/net/corda/core/crypto/DJVM.java b/node/src/main/java/sandbox/net/corda/core/crypto/DJVM.java new file mode 100644 index 0000000000..8004d44666 --- /dev/null +++ b/node/src/main/java/sandbox/net/corda/core/crypto/DJVM.java @@ -0,0 +1,62 @@ +package sandbox.net.corda.core.crypto; + +import org.jetbrains.annotations.NotNull; +import sandbox.java.lang.Integer; +import sandbox.java.lang.String; +import sandbox.java.util.ArrayList; +import sandbox.java.util.List; +import sandbox.org.bouncycastle.asn1.x509.AlgorithmIdentifier; + +import java.io.IOException; + +/** + * Helper class for {@link sandbox.net.corda.core.crypto.Crypto}. + * Deliberately package-private. + */ +final class DJVM { + private DJVM() {} + + @NotNull + static SignatureScheme toDJVM(@NotNull net.corda.core.crypto.SignatureScheme scheme) { + // The AlgorithmParameterSpec is deliberately left as null + // because it is computationally expensive to generate these + // objects inside the sandbox every time. + return new SignatureScheme( + scheme.getSchemeNumberID(), + String.toDJVM(scheme.getSchemeCodeName()), + toDJVM(scheme.getSignatureOID()), + toDJVM(scheme.getAlternativeOIDs()), + String.toDJVM(scheme.getProviderName()), + String.toDJVM(scheme.getAlgorithmName()), + String.toDJVM(scheme.getSignatureName()), + null, + Integer.toDJVM(scheme.getKeySize()), + String.toDJVM(scheme.getDesc()) + ); + } + + static org.bouncycastle.asn1.x509.AlgorithmIdentifier fromDJVM(@NotNull AlgorithmIdentifier oid) { + try { + return org.bouncycastle.asn1.x509.AlgorithmIdentifier.getInstance(oid.getEncoded()); + } catch (IOException e) { + throw sandbox.java.lang.DJVM.toRuleViolationError(e); + } + } + + static AlgorithmIdentifier toDJVM(@NotNull org.bouncycastle.asn1.x509.AlgorithmIdentifier oid) { + try { + return AlgorithmIdentifier.getInstance(oid.getEncoded()); + } catch (IOException e) { + throw sandbox.java.lang.DJVM.toRuleViolationError(e); + } + } + + @NotNull + static List toDJVM(@NotNull java.util.List list) { + ArrayList djvmList = new ArrayList<>(list.size()); + for (org.bouncycastle.asn1.x509.AlgorithmIdentifier oid : list) { + djvmList.add(toDJVM(oid)); + } + return djvmList; + } +} diff --git a/node/src/main/java/sandbox/net/corda/core/crypto/DJVMPublicKey.java b/node/src/main/java/sandbox/net/corda/core/crypto/DJVMPublicKey.java new file mode 100644 index 0000000000..199acc39ce --- /dev/null +++ b/node/src/main/java/sandbox/net/corda/core/crypto/DJVMPublicKey.java @@ -0,0 +1,61 @@ +package sandbox.net.corda.core.crypto; + +import org.jetbrains.annotations.NotNull; +import sandbox.java.lang.Object; +import sandbox.java.lang.String; +import sandbox.java.security.PublicKey; + +/** + * We shall delegate as much cryptography as possible to Corda's + * underlying {@link net.corda.core.crypto.Crypto} object and its + * {@link java.security.Provider} classes. This wrapper only needs + * to implement {@link #equals} and {@link #hashCode}. + */ +final class DJVMPublicKey extends Object implements PublicKey { + private final java.security.PublicKey underlying; + private final String algorithm; + private final String format; + private final int hashCode; + + DJVMPublicKey(@NotNull java.security.PublicKey underlying) { + this.underlying = underlying; + this.algorithm = String.toDJVM(underlying.getAlgorithm()); + this.format = String.toDJVM(underlying.getFormat()); + this.hashCode = underlying.hashCode(); + } + + java.security.PublicKey getUnderlying() { + return underlying; + } + + @Override + public boolean equals(java.lang.Object other) { + if (this == other) { + return true; + } else if (!(other instanceof DJVMPublicKey)) { + return false; + } else { + return underlying.equals(((DJVMPublicKey) other).underlying); + } + } + + @Override + public int hashCode() { + return hashCode; + } + + @Override + public String getAlgorithm() { + return algorithm; + } + + @Override + public String getFormat() { + return format; + } + + @Override + public byte[] getEncoded() { + return underlying.getEncoded(); + } +} diff --git a/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Encodable.java b/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Encodable.java new file mode 100644 index 0000000000..e75ac2e6b4 --- /dev/null +++ b/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Encodable.java @@ -0,0 +1,9 @@ +package sandbox.org.bouncycastle.asn1; + +/** + * This is a dummy class that implements just enough of {@link org.bouncycastle.asn1.ASN1Encodable} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +@SuppressWarnings("WeakerAccess") +public interface ASN1Encodable { +} diff --git a/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Object.java b/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Object.java new file mode 100644 index 0000000000..d71661cc71 --- /dev/null +++ b/node/src/main/java/sandbox/org/bouncycastle/asn1/ASN1Object.java @@ -0,0 +1,16 @@ +package sandbox.org.bouncycastle.asn1; + +import sandbox.java.lang.Object; + +import java.io.IOException; + +/** + * This is a dummy class that implements just enough of {@link org.bouncycastle.asn1.ASN1Object} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +@SuppressWarnings("RedundantThrows") +public class ASN1Object extends Object implements ASN1Encodable { + public byte[] getEncoded() throws IOException { + throw new UnsupportedOperationException("Dummy class - not implemented"); + } +} diff --git a/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/AlgorithmIdentifier.java b/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/AlgorithmIdentifier.java new file mode 100644 index 0000000000..144b5f9260 --- /dev/null +++ b/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/AlgorithmIdentifier.java @@ -0,0 +1,14 @@ +package sandbox.org.bouncycastle.asn1.x509; + +import sandbox.org.bouncycastle.asn1.ASN1Object; + +/** + * This is a dummy class that implements just enough of {@link org.bouncycastle.asn1.x509.AlgorithmIdentifier} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +@SuppressWarnings("unused") +public class AlgorithmIdentifier extends ASN1Object { + public static AlgorithmIdentifier getInstance(Object obj) { + throw new UnsupportedOperationException("Dummy class - not implemented"); + } +} diff --git a/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/SubjectPublicKeyInfo.java b/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/SubjectPublicKeyInfo.java new file mode 100644 index 0000000000..a84fb60772 --- /dev/null +++ b/node/src/main/java/sandbox/org/bouncycastle/asn1/x509/SubjectPublicKeyInfo.java @@ -0,0 +1,10 @@ +package sandbox.org.bouncycastle.asn1.x509; + +import sandbox.org.bouncycastle.asn1.ASN1Object; + +/** + * This is a dummy class that implements just enough of {@link org.bouncycastle.asn1.x509.SubjectPublicKeyInfo} + * to allow us to compile {@link sandbox.net.corda.core.crypto.Crypto}. + */ +public class SubjectPublicKeyInfo extends ASN1Object { +} diff --git a/node/src/main/kotlin/net/corda/node/internal/djvm/Serializer.kt b/node/src/main/kotlin/net/corda/node/internal/djvm/Serializer.kt index 1262ae8710..40a5522a28 100644 --- a/node/src/main/kotlin/net/corda/node/internal/djvm/Serializer.kt +++ b/node/src/main/kotlin/net/corda/node/internal/djvm/Serializer.kt @@ -31,7 +31,7 @@ class Serializer( * [net.corda.node.djvm.LtxFactory] to be transformed finally to * a list of [net.corda.core.contracts.StateAndRef] objects, */ - fun deserialize(stateRefs: List): Array> { + fun deserialize(stateRefs: List): Array> { return stateRefs.map { arrayOf(deserialize(it.serializedState), deserialize(it.ref.serialize())) }.toTypedArray() diff --git a/node/src/main/kotlin/net/corda/node/services/transactions/DeterministicVerifierFactoryService.kt b/node/src/main/kotlin/net/corda/node/services/transactions/DeterministicVerifierFactoryService.kt index 3485e3af71..d514335c92 100644 --- a/node/src/main/kotlin/net/corda/node/services/transactions/DeterministicVerifierFactoryService.kt +++ b/node/src/main/kotlin/net/corda/node/services/transactions/DeterministicVerifierFactoryService.kt @@ -47,7 +47,20 @@ class DeterministicVerifierFactoryService( ConstructorForDeserialization::class.java, DeprecatedConstructorForDeserialization::class.java ), - bootstrapSource = bootstrapSource + bootstrapSource = bootstrapSource, + overrideClasses = setOf( + /** + * These classes are all duplicated into the sandbox + * without the DJVM modifying their byte-code first. + * The goal is to delegate cryptographic operations + * out to the Node rather than perform them inside + * the sandbox, because this is MUCH FASTER. + */ + sandbox.net.corda.core.crypto.Crypto::class.java.name, + "sandbox.net.corda.core.crypto.DJVM", + "sandbox.net.corda.core.crypto.DJVMPublicKey", + "sandbox.net.corda.core.crypto.internal.ProviderMapKt" + ) ) baseSandboxConfiguration = SandboxConfiguration.createFor( diff --git a/node/src/main/kotlin/sandbox/net/corda/core/crypto/Crypto.kt b/node/src/main/kotlin/sandbox/net/corda/core/crypto/Crypto.kt new file mode 100644 index 0000000000..b51a586c93 --- /dev/null +++ b/node/src/main/kotlin/sandbox/net/corda/core/crypto/Crypto.kt @@ -0,0 +1,261 @@ +package sandbox.net.corda.core.crypto + +import sandbox.net.corda.core.crypto.DJVM.fromDJVM +import sandbox.net.corda.core.crypto.DJVM.toDJVM +import sandbox.java.lang.String +import sandbox.java.lang.doCatch +import sandbox.java.math.BigInteger +import sandbox.java.security.KeyPair +import sandbox.java.security.PrivateKey +import sandbox.java.security.PublicKey +import sandbox.java.util.ArrayList +import sandbox.java.util.List +import sandbox.java.lang.Object +import sandbox.org.bouncycastle.asn1.x509.AlgorithmIdentifier +import sandbox.org.bouncycastle.asn1.x509.SubjectPublicKeyInfo +import java.security.GeneralSecurityException +import java.security.SignatureException +import java.security.spec.InvalidKeySpecException + +/** + * This is a hand-written "drop-in" replacement for the version of + * [net.corda.core.crypto.Crypto] found inside core-deterministic. + * This class is used in the DJVM sandbox instead of transforming + * the byte-code for [net.corda.core.crypto.Crypto], with the goal + * of not needing to instantiate some EXPENSIVE elliptic curve + * cryptography classes inside every single sandbox. + * + * The downside is that this class MUST manually be kept consistent + * with the DJVM's byte-code rewriting rules. + */ +@Suppress("unused", "unused_parameter", "TooManyFunctions") +object Crypto : Object() { + @JvmField + val RSA_SHA256: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.RSA_SHA256) + + @JvmField + val ECDSA_SECP256K1_SHA256: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.ECDSA_SECP256K1_SHA256) + + @JvmField + val ECDSA_SECP256R1_SHA256: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.ECDSA_SECP256R1_SHA256) + + @JvmField + val EDDSA_ED25519_SHA512: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.EDDSA_ED25519_SHA512) + + @JvmField + val SPHINCS256_SHA256: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.SPHINCS256_SHA256) + + @JvmField + val COMPOSITE_KEY: SignatureScheme = toDJVM(net.corda.core.crypto.Crypto.COMPOSITE_KEY) + + @JvmField + val DEFAULT_SIGNATURE_SCHEME = EDDSA_ED25519_SHA512 + + /** + * We can use the unsandboxed versions of [Map] and [List] here + * because the [underlyingSchemes] and [djvmSchemes] fields are + * private and not exposed to the rest of the sandbox. + */ + private val underlyingSchemes: Map + = net.corda.core.crypto.Crypto.supportedSignatureSchemes() + .associateBy(net.corda.core.crypto.SignatureScheme::schemeCodeName) + private val djvmSchemes: Map = listOf( + RSA_SHA256, + ECDSA_SECP256K1_SHA256, + ECDSA_SECP256R1_SHA256, + EDDSA_ED25519_SHA512, + SPHINCS256_SHA256, + COMPOSITE_KEY + ).associateByTo(LinkedHashMap(), SignatureScheme::schemeCodeName) + + private fun findUnderlyingSignatureScheme(signatureScheme: SignatureScheme): net.corda.core.crypto.SignatureScheme { + return net.corda.core.crypto.Crypto.findSignatureScheme(String.fromDJVM(signatureScheme.schemeCodeName)) + } + + private fun PublicKey.toUnderlyingKey(): java.security.PublicKey { + return (this as? DJVMPublicKey ?: throw sandbox.java.lang.fail("Unsupported key ${this::class.java.name}")).underlying + } + + @JvmStatic + fun supportedSignatureSchemes(): List { + val schemes = ArrayList(djvmSchemes.size) + for (scheme in djvmSchemes.values) { + schemes.add(scheme) + } + return schemes + } + + @JvmStatic + fun isSupportedSignatureScheme(signatureScheme: SignatureScheme): Boolean { + return String.fromDJVM(signatureScheme.schemeCodeName) in underlyingSchemes + } + + @JvmStatic + fun findSignatureScheme(schemeCodeName: String): SignatureScheme { + return djvmSchemes[schemeCodeName] + ?: throw IllegalArgumentException("Unsupported key/algorithm for schemeCodeName: $schemeCodeName") + } + + @JvmStatic + fun findSignatureScheme(schemeNumberID: Int): SignatureScheme { + val underlyingScheme = net.corda.core.crypto.Crypto.findSignatureScheme(schemeNumberID) + return findSignatureScheme(String.toDJVM(underlyingScheme.schemeCodeName)) + } + + @JvmStatic + fun findSignatureScheme(key: PublicKey): SignatureScheme { + val underlyingScheme = net.corda.core.crypto.Crypto.findSignatureScheme(key.toUnderlyingKey()) + return findSignatureScheme(String.toDJVM(underlyingScheme.schemeCodeName)) + } + + @JvmStatic + fun findSignatureScheme(algorithm: AlgorithmIdentifier): SignatureScheme { + val underlyingScheme = net.corda.core.crypto.Crypto.findSignatureScheme(fromDJVM(algorithm)) + return findSignatureScheme(String.toDJVM(underlyingScheme.schemeCodeName)) + } + + @JvmStatic + fun findSignatureScheme(key: PrivateKey): SignatureScheme { + throw sandbox.java.lang.failApi("Crypto.findSignatureScheme(PrivateKey)") + } + + @JvmStatic + fun decodePrivateKey(signatureScheme: SignatureScheme, encodedKey: ByteArray): PrivateKey { + throw sandbox.java.lang.failApi("Crypto.decodePrivateKey(SignatureScheme, byte[])") + } + + @JvmStatic + fun decodePublicKey(encodedKey: ByteArray): PublicKey { + val underlying = try { + net.corda.core.crypto.Crypto.decodePublicKey(encodedKey) + } catch (e: InvalidKeySpecException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + return DJVMPublicKey(underlying) + } + + @JvmStatic + fun decodePublicKey(schemeCodeName: String, encodedKey: ByteArray): PublicKey { + val underlying = try { + net.corda.core.crypto.Crypto.decodePublicKey(String.fromDJVM(schemeCodeName), encodedKey) + } catch (e: InvalidKeySpecException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + return DJVMPublicKey(underlying) + } + + @JvmStatic + fun decodePublicKey(signatureScheme: SignatureScheme, encodedKey: ByteArray): PublicKey { + return decodePublicKey(signatureScheme.schemeCodeName, encodedKey) + } + + @JvmStatic + fun deriveKeyPair(signatureScheme: SignatureScheme, privateKey: PrivateKey, seed: ByteArray): KeyPair { + throw sandbox.java.lang.failApi("Crypto.deriveKeyPair(SignatureScheme, PrivateKey, byte[])") + } + + @JvmStatic + fun deriveKeyPair(privateKey: PrivateKey, seed: ByteArray): KeyPair { + throw sandbox.java.lang.failApi("Crypto.deriveKeyPair(PrivateKey, byte[])") + } + + @JvmStatic + fun deriveKeyPairFromEntropy(signatureScheme: SignatureScheme, entropy: BigInteger): KeyPair { + throw sandbox.java.lang.failApi("Crypto.deriveKeyPairFromEntropy(SignatureScheme, BigInteger)") + } + + @JvmStatic + fun deriveKeyPairFromEntropy(entropy: BigInteger): KeyPair { + throw sandbox.java.lang.failApi("Crypto.deriveKeyPairFromEntropy(BigInteger)") + } + + @JvmStatic + fun doVerify(schemeCodeName: String, publicKey: PublicKey, signatureData: ByteArray, clearData: ByteArray): Boolean { + val underlyingKey = publicKey.toUnderlyingKey() + return try { + net.corda.core.crypto.Crypto.doVerify(String.fromDJVM(schemeCodeName), underlyingKey, signatureData, clearData) + } catch (e: GeneralSecurityException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + } + + @JvmStatic + fun doVerify(publicKey: PublicKey, signatureData: ByteArray, clearData: ByteArray): Boolean { + val underlyingKey = publicKey.toUnderlyingKey() + return try { + net.corda.core.crypto.Crypto.doVerify(underlyingKey, signatureData, clearData) + } catch (e: GeneralSecurityException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + } + + @JvmStatic + fun doVerify(signatureScheme: SignatureScheme, publicKey: PublicKey, signatureData: ByteArray, clearData: ByteArray): Boolean { + val underlyingScheme = findUnderlyingSignatureScheme(signatureScheme) + val underlyingKey = publicKey.toUnderlyingKey() + return try { + net.corda.core.crypto.Crypto.doVerify(underlyingScheme, underlyingKey, signatureData, clearData) + } catch (e: GeneralSecurityException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + } + + @JvmStatic + fun doVerify(txId: SecureHash, transactionSignature: TransactionSignature): Boolean { + throw sandbox.java.lang.failApi("Crypto.doVerify(SecureHash, TransactionSignature)") + } + + @JvmStatic + fun isValid(txId: SecureHash, transactionSignature: TransactionSignature): Boolean { + throw sandbox.java.lang.failApi("Crypto.isValid(SecureHash, TransactionSignature)") + } + + @JvmStatic + fun isValid(publicKey: PublicKey, signatureData: ByteArray, clearData: ByteArray): Boolean { + val underlyingKey = publicKey.toUnderlyingKey() + return try { + net.corda.core.crypto.Crypto.isValid(underlyingKey, signatureData, clearData) + } catch (e: SignatureException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + } + + @JvmStatic + fun isValid(signatureScheme: SignatureScheme, publicKey: PublicKey, signatureData: ByteArray, clearData: ByteArray): Boolean { + val underlyingScheme = findUnderlyingSignatureScheme(signatureScheme) + val underlyingKey = publicKey.toUnderlyingKey() + return try { + net.corda.core.crypto.Crypto.isValid(underlyingScheme, underlyingKey, signatureData, clearData) + } catch (e: SignatureException) { + throw sandbox.java.lang.fromDJVM(doCatch(e)) + } + } + + @JvmStatic + fun publicKeyOnCurve(signatureScheme: SignatureScheme, publicKey: PublicKey): Boolean { + val underlyingScheme = findUnderlyingSignatureScheme(signatureScheme) + val underlyingKey = publicKey.toUnderlyingKey() + return net.corda.core.crypto.Crypto.publicKeyOnCurve(underlyingScheme, underlyingKey) + } + + @JvmStatic + fun validatePublicKey(key: PublicKey): Boolean { + return net.corda.core.crypto.Crypto.validatePublicKey(key.toUnderlyingKey()) + } + + @JvmStatic + fun toSupportedPublicKey(key: SubjectPublicKeyInfo): PublicKey { + return decodePublicKey(key.encoded) + } + + @JvmStatic + fun toSupportedPublicKey(key: PublicKey): PublicKey { + val underlyingKey = key.toUnderlyingKey() + val supportedKey = net.corda.core.crypto.Crypto.toSupportedPublicKey(underlyingKey) + return if (supportedKey === underlyingKey) { + key + } else { + DJVMPublicKey(supportedKey) + } + } +} diff --git a/node/src/main/kotlin/sandbox/net/corda/core/crypto/SecureHash.kt b/node/src/main/kotlin/sandbox/net/corda/core/crypto/SecureHash.kt new file mode 100644 index 0000000000..ac8f733511 --- /dev/null +++ b/node/src/main/kotlin/sandbox/net/corda/core/crypto/SecureHash.kt @@ -0,0 +1,8 @@ +package sandbox.net.corda.core.crypto + +/** + * This is a dummy class that implements just enough of [net.corda.core.crypto.SecureHash] + * to allow us to compile [sandbox.net.corda.core.crypto.Crypto]. + */ +@Suppress("unused_parameter") +sealed class SecureHash(bytes: ByteArray) : sandbox.java.lang.Object() diff --git a/node/src/main/kotlin/sandbox/net/corda/core/crypto/SignatureScheme.kt b/node/src/main/kotlin/sandbox/net/corda/core/crypto/SignatureScheme.kt new file mode 100644 index 0000000000..dd315252df --- /dev/null +++ b/node/src/main/kotlin/sandbox/net/corda/core/crypto/SignatureScheme.kt @@ -0,0 +1,26 @@ +package sandbox.net.corda.core.crypto + +import sandbox.java.lang.String +import sandbox.java.lang.Integer +import sandbox.java.lang.Object +import sandbox.java.security.spec.AlgorithmParameterSpec +import sandbox.java.util.List +import sandbox.org.bouncycastle.asn1.x509.AlgorithmIdentifier + +/** + * This is a dummy class that implements just enough of [net.corda.core.crypto.SignatureScheme] + * to allow us to compile [sandbox.net.corda.core.crypto.Crypto]. + */ +@Suppress("unused") +class SignatureScheme( + val schemeNumberID: Int, + val schemeCodeName: String, + val signatureOID: AlgorithmIdentifier, + val alternativeOIDs: List, + val providerName: String, + val algorithmName: String, + val signatureName: String, + val algSpec: AlgorithmParameterSpec?, + val keySize: Integer?, + val desc: String +) : Object() diff --git a/node/src/main/kotlin/sandbox/net/corda/core/crypto/TransactionSignature.kt b/node/src/main/kotlin/sandbox/net/corda/core/crypto/TransactionSignature.kt new file mode 100644 index 0000000000..2d3c3e8b90 --- /dev/null +++ b/node/src/main/kotlin/sandbox/net/corda/core/crypto/TransactionSignature.kt @@ -0,0 +1,7 @@ +package sandbox.net.corda.core.crypto + +/** + * This is a dummy class that implements just enough of [net.corda.core.crypto.TransactionSignature] + * to allow us to compile [sandbox.net.corda.core.crypto.Crypto]. + */ +class TransactionSignature : sandbox.java.lang.Object() diff --git a/node/src/main/kotlin/sandbox/net/corda/core/crypto/internal/ProviderMap.kt b/node/src/main/kotlin/sandbox/net/corda/core/crypto/internal/ProviderMap.kt new file mode 100644 index 0000000000..5a0ebb4065 --- /dev/null +++ b/node/src/main/kotlin/sandbox/net/corda/core/crypto/internal/ProviderMap.kt @@ -0,0 +1,8 @@ +package sandbox.net.corda.core.crypto.internal + +/** + * THIS FILE IS DELIBERATELY EMPTY, APART FROM A SINGLE DUMMY VALUE. + * KOTLIN WILL NOT CREATE A CLASS IF THIS FILE IS COMPLETELY EMPTY. + */ +@Suppress("unused") +private const val DUMMY_VALUE = 0 diff --git a/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/serializers/SandboxPublicKeySerializer.kt b/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/serializers/SandboxPublicKeySerializer.kt index ec528667ee..6a22e05da6 100644 --- a/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/serializers/SandboxPublicKeySerializer.kt +++ b/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/serializers/SandboxPublicKeySerializer.kt @@ -19,8 +19,7 @@ class SandboxPublicKeySerializer( taskFactory: Function>, out Function> ) : CustomSerializer.Implements(classLoader.toSandboxAnyClass(PublicKey::class.java)) { @Suppress("unchecked_cast") - private val decoder: Function - = taskFactory.apply(PublicKeyDecoder::class.java) as Function + private val decoder = taskFactory.apply(PublicKeyDecoder::class.java) as Function override val schemaForDocumentation: Schema = Schema(emptyList()) diff --git a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializePublicKeyTest.kt b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializePublicKeyTest.kt index d952ff10d6..dc982a8569 100644 --- a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializePublicKeyTest.kt +++ b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializePublicKeyTest.kt @@ -8,6 +8,7 @@ import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.internal._contextSerializationEnv import net.corda.core.serialization.serialize import net.corda.serialization.djvm.SandboxType.KOTLIN +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test import org.junit.jupiter.api.extension.ExtendWith @@ -69,13 +70,18 @@ class DeserializePublicKeyTest : TestBase(KOTLIN) { sandbox { _contextSerializationEnv.set(createSandboxSerializationEnv(classLoader)) - val sandboxKey = data.deserializeFor(classLoader) + val sandboxData = data.deserializeFor(classLoader) - val taskFactory = classLoader.createRawTaskFactory() - val showPublicKey = taskFactory.compose(classLoader.createSandboxFunction()).apply(ShowPublicKey::class.java) - val result = showPublicKey.apply(sandboxKey) ?: fail("Result cannot be null") + val taskFactory = classLoader.createRawTaskFactory().compose(classLoader.createSandboxFunction()) + val showPublicKey = taskFactory.apply(ShowPublicKey::class.java) + val result = showPublicKey.apply(sandboxData) ?: fail("Result cannot be null") assertEquals(ShowPublicKey().apply(compositeData), result.toString()) + + val sandboxKey = taskFactory.apply(GetPublicKey::class.java) + .apply(sandboxData) ?: fail("PublicKey cannot be null") + assertThat(sandboxKey::class.java.name) + .isEqualTo("sandbox." + CompositeKey::class.java.name) } } @@ -86,6 +92,12 @@ class DeserializePublicKeyTest : TestBase(KOTLIN) { } } } + + class GetPublicKey : Function { + override fun apply(data: PublicKeyData): PublicKey { + return data.key + } + } } @CordaSerializable From 5afdb63c94373239b9b63cce580e77723f4a6149 Mon Sep 17 00:00:00 2001 From: Denis Rekalov Date: Tue, 26 May 2020 16:10:14 +0100 Subject: [PATCH 46/49] CORDA-3818: Synchronize OS implementation of PublicKeyToOwningIdentityCache with CE --- .../PublicKeyToOwningIdentityCache.kt | 4 +- .../identity/PersistentIdentityService.kt | 5 +- .../PublicKeyToOwningIdentityCacheImpl.kt | 61 +++---------------- .../PublicKeyToOwningIdentityCacheImplTest.kt | 4 +- .../MockPublicKeyToOwningIdentityCache.kt | 4 +- 5 files changed, 19 insertions(+), 59 deletions(-) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/PublicKeyToOwningIdentityCache.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/PublicKeyToOwningIdentityCache.kt index e7f0e3ec3c..4c66b14396 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/PublicKeyToOwningIdentityCache.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/PublicKeyToOwningIdentityCache.kt @@ -11,7 +11,7 @@ interface PublicKeyToOwningIdentityCache { /** * Obtain the owning identity for a public key. * - * If the key is unknown to the node, then this will return null. + * If the key is unknown to the node, then this will return [KeyOwningIdentity.UnmappedIdentity]. */ - operator fun get(key: PublicKey): KeyOwningIdentity? + operator fun get(key: PublicKey): KeyOwningIdentity } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt index 6e896a6d22..ac91bdec68 100644 --- a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt +++ b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt @@ -413,6 +413,9 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri return database.transaction { log.info("Linking: ${publicKey.hash} to ${party.name}") keyToName[publicKey.toStringShort()] = party.name + if (party == wellKnownPartyFromX500Name(ourNames.first())) { + _pkToIdCache[publicKey] = KeyOwningIdentity.UnmappedIdentity + } } } @@ -422,7 +425,7 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri } override fun externalIdForPublicKey(publicKey: PublicKey): UUID? { - return _pkToIdCache[publicKey]?.uuid + return _pkToIdCache[publicKey].uuid } private fun publicKeysForExternalId(externalId: UUID, table: Class<*>): List { diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImpl.kt b/node/src/main/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImpl.kt index 7dcae65237..fd3d01431d 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImpl.kt @@ -5,12 +5,9 @@ import net.corda.core.crypto.toStringShort import net.corda.core.internal.NamedCacheFactory import net.corda.core.utilities.contextLogger import net.corda.core.utilities.debug -import net.corda.node.services.identity.PersistentIdentityService -import net.corda.node.services.keys.BasicHSMKeyManagementService import net.corda.nodeapi.internal.KeyOwningIdentity import net.corda.nodeapi.internal.persistence.CordaPersistence import java.security.PublicKey -import java.util.* /** * The [PublicKeyToOwningIdentityCacheImpl] provides a caching layer over the pk_hash_to_external_id table. Gets will attempt to read an @@ -22,42 +19,10 @@ class PublicKeyToOwningIdentityCacheImpl(private val database: CordaPersistence, val log = contextLogger() } - private val cache = cacheFactory.buildNamed(Caffeine.newBuilder(), "PublicKeyToOwningIdentityCache_cache") - - /** - * Establish whether a public key is one of the node's identity keys, by looking in the node's identity database table. - */ - private fun isKeyIdentityKey(key: PublicKey): Boolean { - return database.transaction { - val criteriaBuilder = session.criteriaBuilder - val criteriaQuery = criteriaBuilder.createQuery(Long::class.java) - val queryRoot = criteriaQuery.from(PersistentIdentityService.PersistentPublicKeyHashToCertificate::class.java) - criteriaQuery.select(criteriaBuilder.count(queryRoot)) - criteriaQuery.where( - criteriaBuilder.equal(queryRoot.get(PersistentIdentityService.PersistentPublicKeyHashToCertificate::publicKeyHash.name), key.toStringShort()) - ) - val query = session.createQuery(criteriaQuery) - query.uniqueResult() > 0 - } - } - - /** - * Check to see if the key belongs to one of the key pairs in the node_our_key_pairs table. These keys may relate to confidential - * identities. - */ - private fun isKeyPartOfNodeKeyPairs(key: PublicKey): Boolean { - return database.transaction { - val criteriaBuilder = session.criteriaBuilder - val criteriaQuery = criteriaBuilder.createQuery(Long::class.java) - val queryRoot = criteriaQuery.from(BasicHSMKeyManagementService.PersistentKey::class.java) - criteriaQuery.select(criteriaBuilder.count(queryRoot)) - criteriaQuery.where( - criteriaBuilder.equal(queryRoot.get(BasicHSMKeyManagementService.PersistentKey::publicKeyHash.name), key.toStringShort()) - ) - val query = session.createQuery(criteriaQuery) - query.uniqueResult() > 0 - } - } + private val cache = cacheFactory.buildNamed( + Caffeine.newBuilder(), + "PublicKeyToOwningIdentityCache_cache" + ) /** * Return the owning identity associated with a given key. @@ -65,25 +30,17 @@ class PublicKeyToOwningIdentityCacheImpl(private val database: CordaPersistence, * This method caches the result of a database lookup to prevent multiple database accesses for the same key. This assumes that once a * key is generated, the UUID assigned to it is never changed. */ - override operator fun get(key: PublicKey): KeyOwningIdentity? { + override operator fun get(key: PublicKey): KeyOwningIdentity { return cache.asMap().computeIfAbsent(key) { database.transaction { - val criteriaBuilder = session.criteriaBuilder - val criteriaQuery = criteriaBuilder.createQuery(UUID::class.java) - val queryRoot = criteriaQuery.from(PublicKeyHashToExternalId::class.java) - criteriaQuery.select(queryRoot.get(PublicKeyHashToExternalId::externalId.name)) - criteriaQuery.where( - criteriaBuilder.equal(queryRoot.get(PublicKeyHashToExternalId::publicKeyHash.name), key.toStringShort()) - ) - val query = session.createQuery(criteriaQuery) - val uuid = query.uniqueResult() - if (uuid != null || isKeyPartOfNodeKeyPairs(key) || isKeyIdentityKey(key)) { + val uuid = session.find(PublicKeyHashToExternalId::class.java, key.toStringShort())?.externalId + if (uuid != null) { val signingEntity = KeyOwningIdentity.fromUUID(uuid) log.debug { "Database lookup for public key ${key.toStringShort()}, found signing entity $signingEntity" } signingEntity } else { - log.debug { "Attempted to find owning identity for public key ${key.toStringShort()}, but key is unknown to node" } - null + log.debug { "Database lookup for public key ${key.toStringShort()}, using ${KeyOwningIdentity.UnmappedIdentity}" } + KeyOwningIdentity.UnmappedIdentity } } } diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImplTest.kt b/node/src/test/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImplTest.kt index af31369209..5f71169ef4 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/PublicKeyToOwningIdentityCacheImplTest.kt @@ -114,9 +114,9 @@ class PublicKeyToOwningIdentityCacheImplTest { } @Test(timeout=300_000) - fun `requesting a key unknown to the node returns null`() { + fun `requesting a key unknown to the node returns unmapped identity`() { val keys = generateKeyPair() - assertEquals(null, testCache[keys.public]) + assertEquals(KeyOwningIdentity.UnmappedIdentity, testCache[keys.public]) } @Test(timeout=300_000) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockPublicKeyToOwningIdentityCache.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockPublicKeyToOwningIdentityCache.kt index c8acf07faf..06abf4a1cc 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockPublicKeyToOwningIdentityCache.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockPublicKeyToOwningIdentityCache.kt @@ -13,8 +13,8 @@ class MockPublicKeyToOwningIdentityCache : WritablePublicKeyToOwningIdentityCach private val cache: MutableMap = mutableMapOf().toSynchronised() - override fun get(key: PublicKey): KeyOwningIdentity? { - return cache[key] + override fun get(key: PublicKey): KeyOwningIdentity { + return cache[key] ?: KeyOwningIdentity.UnmappedIdentity } override fun set(key: PublicKey, value: KeyOwningIdentity) { From 598228634f9d7d739a232cd9bde768408dc29370 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Wed, 27 May 2020 16:46:56 +0100 Subject: [PATCH 47/49] CORDA-3608 Performance tune the new checkpoint schema (#6277) Performance tuning of the new checkpoint schema; - Checkpoint tables are now using `flowId` as join keys. - Indexes consist of a PK's index on `node_checkpoints(flow_id)` and then unique indexes on `node_checkpoint_blobs(flow_id)` and `node_flow_metadata(flow_id)`. - Serialization of `checkpointState` is being done with `CHECKPOINT_CONTEXT` so that we can have compression. This is needed when messages get passed into `checkpointState.sessions` therefore `checkpointState` grows in size upon serialized and saved into the database. * Deserialize checkpointState with CHECKPOINT_CONTEXT * Align tests with schema update; We cannot add and update a checkpoint in the same session now, ends up with hibernate complaining: two different objects with same identifier * Fix indentation and format * Ignore tests that assert DBFlowResult or DBFlowException * Set DBFlowCheckpoint.blob to null whenever the flow errors or hospitalizes; this way we save an extra SELECT in such cases; * Fix test; cleared Hibernate session, it would fail at checkpoint with 'org.hibernate.NonUniqueObjectException' * Changing VARCHAR to NVARCHAR * Rename v17 liquibase scripts to v19 to resolve collision with ENT v17 scripts --- .../CordaPersistenceServiceTests.kt | 3 +- .../node/services/api/CheckpointStorage.kt | 7 +- .../persistence/DBCheckpointStorage.kt | 324 +++++++++--------- .../statemachine/ActionExecutorImpl.kt | 8 +- .../statemachine/StateMachineState.kt | 2 +- .../migration/node-core.changelog-master.xml | 6 +- .../node-core.changelog-v17-keys.xml | 35 -- .../node-core.changelog-v19-keys.xml | 26 ++ ...l => node-core.changelog-v19-postgres.xml} | 35 +- ...og-v17.xml => node-core.changelog-v19.xml} | 25 +- .../persistence/DBCheckpointStorageTests.kt | 164 +++++---- .../services/rpc/CheckpointDumperImplTest.kt | 12 +- .../statemachine/FlowFrameworkTests.kt | 6 + 13 files changed, 344 insertions(+), 309 deletions(-) delete mode 100644 node/src/main/resources/migration/node-core.changelog-v17-keys.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v19-keys.xml rename node/src/main/resources/migration/{node-core.changelog-v17-postgres.xml => node-core.changelog-v19-postgres.xml} (85%) rename node/src/main/resources/migration/{node-core.changelog-v17.xml => node-core.changelog-v19.xml} (87%) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt index 25e7f3af95..307e67d12b 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/persistence/CordaPersistenceServiceTests.kt @@ -65,8 +65,9 @@ class CordaPersistenceServiceTests { val flowId = it.toString() session.save( DBCheckpointStorage.DBFlowCheckpoint( - id = flowId, + flowId = flowId, blob = DBCheckpointStorage.DBFlowCheckpointBlob( + flowId = flowId, checkpoint = ByteArray(8192), flowStack = ByteArray(8192), hmac = ByteArray(16), diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index b9f67d8bd7..0bac15c171 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -3,6 +3,7 @@ package net.corda.node.services.api import net.corda.core.flows.StateMachineRunId import net.corda.core.serialization.SerializedBytes import net.corda.node.services.statemachine.Checkpoint +import net.corda.node.services.statemachine.CheckpointState import net.corda.node.services.statemachine.FlowState import java.util.stream.Stream @@ -13,12 +14,14 @@ interface CheckpointStorage { /** * Add a checkpoint for a new id to the store. Will throw if there is already a checkpoint for this id */ - fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) + fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes, + serializedCheckpointState: SerializedBytes) /** * Update an existing checkpoint. Will throw if there is not checkpoint for this id. */ - fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?) + fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?, + serializedCheckpointState: SerializedBytes) /** * Update all persisted checkpoints with status [Checkpoint.FlowStatus.RUNNABLE] or [Checkpoint.FlowStatus.HOSPITALIZED], diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 473e824dc4..2dc5a0d3e9 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -28,15 +28,12 @@ import java.time.Clock import java.time.Instant import java.util.* import java.util.stream.Stream -import javax.persistence.CascadeType import javax.persistence.Column import javax.persistence.Entity import javax.persistence.FetchType -import javax.persistence.GeneratedValue -import javax.persistence.GenerationType import javax.persistence.Id -import javax.persistence.JoinColumn import javax.persistence.OneToOne +import javax.persistence.PrimaryKeyJoinColumn /** * Simple checkpoint key value storage in DB. @@ -94,22 +91,22 @@ class DBCheckpointStorage( class DBFlowCheckpoint( @Id @Column(name = "flow_id", length = 64, nullable = false) - var id: String, - - @OneToOne(fetch = FetchType.LAZY, cascade = [CascadeType.ALL], optional = true) - @JoinColumn(name = "checkpoint_blob_id", referencedColumnName = "id") - var blob: DBFlowCheckpointBlob, + var flowId: String, @OneToOne(fetch = FetchType.LAZY, optional = true) - @JoinColumn(name = "result_id", referencedColumnName = "id") + @PrimaryKeyJoinColumn + var blob: DBFlowCheckpointBlob?, + + @OneToOne(fetch = FetchType.LAZY, optional = true) + @PrimaryKeyJoinColumn var result: DBFlowResult?, @OneToOne(fetch = FetchType.LAZY, optional = true) - @JoinColumn(name = "error_id", referencedColumnName = "id") + @PrimaryKeyJoinColumn var exceptionDetails: DBFlowException?, @OneToOne(fetch = FetchType.LAZY) - @JoinColumn(name = "flow_id", referencedColumnName = "flow_id") + @PrimaryKeyJoinColumn var flowMetadata: DBFlowMetadata, @Column(name = "status", nullable = false) @@ -132,9 +129,8 @@ class DBCheckpointStorage( @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoint_blobs") class DBFlowCheckpointBlob( @Id - @GeneratedValue(strategy = GenerationType.SEQUENCE) - @Column(name = "id", nullable = false) - var id: Long = 0, + @Column(name = "flow_id", length = 64, nullable = false) + var flowId: String, @Type(type = "corda-blob") @Column(name = "checkpoint_value", nullable = false) @@ -144,6 +140,7 @@ class DBCheckpointStorage( @Column(name = "flow_state") var flowStack: ByteArray?, + @Type(type = "corda-wrapper-binary") @Column(name = "hmac") var hmac: ByteArray, @@ -155,9 +152,8 @@ class DBCheckpointStorage( @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_results") class DBFlowResult( @Id - @Column(name = "id", nullable = false) - @GeneratedValue(strategy = GenerationType.SEQUENCE) - var id: Long = 0, + @Column(name = "flow_id", length = 64, nullable = false) + var flow_id: String, @Type(type = "corda-blob") @Column(name = "result_value", nullable = false) @@ -171,9 +167,8 @@ class DBCheckpointStorage( @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_exceptions") class DBFlowException( @Id - @Column(name = "id", nullable = false) - @GeneratedValue(strategy = GenerationType.SEQUENCE) - var id: Long = 0, + @Column(name = "flow_id", length = 64, nullable = false) + var flow_id: String, @Column(name = "type", nullable = false) var type: String, @@ -195,9 +190,8 @@ class DBCheckpointStorage( @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}flow_metadata") class DBFlowMetadata( - @Id - @Column(name = "flow_id", nullable = false) + @Column(name = "flow_id", length = 64, nullable = false) var flowId: String, @Column(name = "invocation_id", nullable = false) @@ -273,36 +267,121 @@ class DBCheckpointStorage( } } - override fun addCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes) { - currentDBSession().save(createDBCheckpoint(id, checkpoint, serializedFlowState)) + override fun addCheckpoint( + id: StateMachineRunId, + checkpoint: Checkpoint, + serializedFlowState: SerializedBytes, + serializedCheckpointState: SerializedBytes + ) { + val now = clock.instant() + val flowId = id.uuid.toString() + + checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + + val blob = createDBCheckpointBlob( + flowId, + serializedCheckpointState, + serializedFlowState, + now + ) + + val metadata = createDBFlowMetadata(flowId, checkpoint) + + // Most fields are null as they cannot have been set when creating the initial checkpoint + val dbFlowCheckpoint = DBFlowCheckpoint( + flowId = flowId, + blob = blob, + result = null, + exceptionDetails = null, + flowMetadata = metadata, + status = checkpoint.status, + compatible = checkpoint.compatible, + progressStep = null, + ioRequestType = null, + checkpointInstant = now + ) + + currentDBSession().save(dbFlowCheckpoint) + currentDBSession().save(blob) + currentDBSession().save(metadata) } - override fun updateCheckpoint(id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?) { - currentDBSession().update(updateDBCheckpoint(id, checkpoint, serializedFlowState)) + override fun updateCheckpoint( + id: StateMachineRunId, checkpoint: Checkpoint, serializedFlowState: SerializedBytes?, + serializedCheckpointState: SerializedBytes + ) { + val now = clock.instant() + val flowId = id.uuid.toString() + + // Do not update in DB [Checkpoint.checkpointState] or [Checkpoint.flowState] if flow failed or got hospitalized + val blob = if (checkpoint.status == FlowStatus.FAILED || checkpoint.status == FlowStatus.HOSPITALIZED) { + null + } else { + checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) + createDBCheckpointBlob( + flowId, + serializedCheckpointState, + serializedFlowState, + now + ) + } + + //This code needs to be added back in when we want to persist the result. For now this requires the result to be @CordaSerializable. + //val result = updateDBFlowResult(entity, checkpoint, now) + val exceptionDetails = updateDBFlowException(flowId, checkpoint, now) + + val metadata = createDBFlowMetadata(flowId, checkpoint) + + val dbFlowCheckpoint = DBFlowCheckpoint( + flowId = flowId, + blob = blob, + result = null, + exceptionDetails = exceptionDetails, + flowMetadata = metadata, + status = checkpoint.status, + compatible = checkpoint.compatible, + progressStep = checkpoint.progressStep?.take(MAX_PROGRESS_STEP_LENGTH), + ioRequestType = checkpoint.flowIoRequest, + checkpointInstant = now + ) + + currentDBSession().update(dbFlowCheckpoint) + blob?.let { currentDBSession().update(it) } + if (checkpoint.isFinished()) { + metadata.finishInstant = now + currentDBSession().update(metadata) + } } override fun markAllPaused() { val session = currentDBSession() - val runnableOrdinals = RUNNABLE_CHECKPOINTS.map{ "${it.ordinal}"}.joinToString { it } + val runnableOrdinals = RUNNABLE_CHECKPOINTS.map { "${it.ordinal}" }.joinToString { it } val sqlQuery = "Update ${NODE_DATABASE_PREFIX}checkpoints set status = ${FlowStatus.PAUSED.ordinal} " + "where status in ($runnableOrdinals)" val query = session.createNativeQuery(sqlQuery) query.executeUpdate() } + // DBFlowResult and DBFlowException to be integrated with rest of schema + @Suppress("MagicNumber") override fun removeCheckpoint(id: StateMachineRunId): Boolean { - // This will be changed after performance tuning - return currentDBSession().let { session -> - session.find(DBFlowCheckpoint::class.java, id.uuid.toString())?.run { - result?.let { session.delete(result) } - exceptionDetails?.let { session.delete(exceptionDetails) } - session.delete(blob) - session.delete(this) - // The metadata foreign key might be the wrong way around - session.delete(flowMetadata) - true - } - } ?: false + var deletedRows = 0 + val flowId = id.uuid.toString() + deletedRows += deleteRow(DBFlowMetadata::class.java, DBFlowMetadata::flowId.name, flowId) + deletedRows += deleteRow(DBFlowCheckpointBlob::class.java, DBFlowCheckpointBlob::flowId.name, flowId) + deletedRows += deleteRow(DBFlowCheckpoint::class.java, DBFlowCheckpoint::flowId.name, flowId) +// resultId?.let { deletedRows += deleteRow(DBFlowResult::class.java, DBFlowResult::flow_id.name, it.toString()) } +// exceptionId?.let { deletedRows += deleteRow(DBFlowException::class.java, DBFlowException::flow_id.name, it.toString()) } + return deletedRows == 3 + } + + private fun deleteRow(clazz: Class, pk: String, value: String): Int { + val session = currentDBSession() + val criteriaBuilder = session.criteriaBuilder + val delete = criteriaBuilder.createCriteriaDelete(clazz) + val root = delete.from(clazz) + delete.where(criteriaBuilder.equal(root.get(pk), value)) + return session.createQuery(delete).executeUpdate() } override fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? { @@ -315,14 +394,14 @@ class DBCheckpointStorage( val criteriaQuery = criteriaBuilder.createQuery(DBFlowCheckpoint::class.java) val root = criteriaQuery.from(DBFlowCheckpoint::class.java) criteriaQuery.select(root) - .where(criteriaBuilder.isTrue(root.get(DBFlowCheckpoint::status.name).`in`(statuses))) + .where(criteriaBuilder.isTrue(root.get(DBFlowCheckpoint::status.name).`in`(statuses))) return session.createQuery(criteriaQuery).stream().map { - StateMachineRunId(UUID.fromString(it.id)) to it.toSerializedCheckpoint() + StateMachineRunId(UUID.fromString(it.flowId)) to it.toSerializedCheckpoint() } } override fun getCheckpointsToRun(): Stream> { - return getCheckpoints(RUNNABLE_CHECKPOINTS) + return getCheckpoints(RUNNABLE_CHECKPOINTS) } @VisibleForTesting @@ -342,36 +421,7 @@ class DBCheckpointStorage( } } - private fun createDBCheckpoint( - id: StateMachineRunId, - checkpoint: Checkpoint, - serializedFlowState: SerializedBytes - ): DBFlowCheckpoint { - val flowId = id.uuid.toString() - val now = clock.instant() - - val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() - checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) - - val blob = createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) - - val metadata = createMetadata(flowId, checkpoint) - // Most fields are null as they cannot have been set when creating the initial checkpoint - return DBFlowCheckpoint( - id = flowId, - blob = blob, - result = null, - exceptionDetails = null, - flowMetadata = metadata, - status = checkpoint.status, - compatible = checkpoint.compatible, - progressStep = null, - ioRequestType = null, - checkpointInstant = now - ) - } - - private fun createMetadata(flowId: String, checkpoint: Checkpoint): DBFlowMetadata { + private fun createDBFlowMetadata(flowId: String, checkpoint: Checkpoint): DBFlowMetadata { val context = checkpoint.checkpointState.invocationContext val flowInfo = checkpoint.checkpointState.subFlowStack.first() return DBFlowMetadata( @@ -390,63 +440,17 @@ class DBCheckpointStorage( invocationInstant = context.trace.invocationId.timestamp, startInstant = clock.instant(), finishInstant = null - ).apply { - currentDBSession().save(this) - } - } - - private fun updateDBCheckpoint( - id: StateMachineRunId, - checkpoint: Checkpoint, - serializedFlowState: SerializedBytes? - ): DBFlowCheckpoint { - val flowId = id.uuid.toString() - val now = clock.instant() - - // Load the previous entity from the hibernate cache so the meta data join does not get updated - val entity = currentDBSession().find(DBFlowCheckpoint::class.java, flowId) - - // Do not update in DB [Checkpoint.checkpointState] or [Checkpoint.flowState] if flow failed or got hospitalized - val blob = if (checkpoint.status == FlowStatus.FAILED || checkpoint.status == FlowStatus.HOSPITALIZED) { - entity.blob - } else { - val serializedCheckpointState = checkpoint.checkpointState.storageSerialize() - checkpointPerformanceRecorder.record(serializedCheckpointState, serializedFlowState) - createDBCheckpointBlob(serializedCheckpointState, serializedFlowState, now) - } - - //This code needs to be added back in when we want to persist the result. For now this requires the result to be @CordaSerializable. - //val result = updateDBFlowResult(entity, checkpoint, now) - val exceptionDetails = updateDBFlowException(entity, checkpoint, now) - - val metadata = entity.flowMetadata.apply { - if (checkpoint.isFinished() && finishInstant == null) { - finishInstant = now - currentDBSession().update(this) - } - } - - return entity.apply { - this.blob = blob - //Set the result to null for now. - this.result = null - this.exceptionDetails = exceptionDetails - // Do not update the meta data relationship on updates - this.flowMetadata = metadata - this.status = checkpoint.status - this.compatible = checkpoint.compatible - this.progressStep = checkpoint.progressStep?.take(MAX_PROGRESS_STEP_LENGTH) - this.ioRequestType = checkpoint.flowIoRequest - this.checkpointInstant = now - } + ) } private fun createDBCheckpointBlob( + flowId: String, serializedCheckpointState: SerializedBytes, serializedFlowState: SerializedBytes?, now: Instant ): DBFlowCheckpointBlob { return DBFlowCheckpointBlob( + flowId = flowId, checkpoint = serializedCheckpointState.bytes, flowStack = serializedFlowState?.bytes, hmac = ByteArray(HMAC_SIZE_BYTES), @@ -464,11 +468,11 @@ class DBCheckpointStorage( * The existing [DBFlowResult] is deleted if [DBFlowCheckpoint.result] exists and the [Checkpoint] has no result. * Nothing happens if both [DBFlowCheckpoint] and [Checkpoint] do not have a result. */ - private fun updateDBFlowResult(entity: DBFlowCheckpoint, checkpoint: Checkpoint, now: Instant): DBFlowResult? { - val result = checkpoint.result?.let { createDBFlowResult(it, now) } + private fun updateDBFlowResult(flowId: String, entity: DBFlowCheckpoint, checkpoint: Checkpoint, now: Instant): DBFlowResult? { + val result = checkpoint.result?.let { createDBFlowResult(flowId, it, now) } if (entity.result != null) { if (result != null) { - result.id = entity.result!!.id + result.flow_id = entity.result!!.flow_id currentDBSession().update(result) } else { currentDBSession().delete(entity.result) @@ -479,8 +483,9 @@ class DBCheckpointStorage( return result } - private fun createDBFlowResult(result: Any, now: Instant): DBFlowResult { + private fun createDBFlowResult(flowId: String, result: Any, now: Instant): DBFlowResult { return DBFlowResult( + flow_id = flowId, value = result.storageSerialize().bytes, persistedInstant = now ) @@ -496,24 +501,31 @@ class DBCheckpointStorage( * The existing [DBFlowException] is deleted if [DBFlowCheckpoint.exceptionDetails] exists and the [Checkpoint] has no error. * Nothing happens if both [DBFlowCheckpoint] and [Checkpoint] are related to no errors. */ - private fun updateDBFlowException(entity: DBFlowCheckpoint, checkpoint: Checkpoint, now: Instant): DBFlowException? { - val exceptionDetails = (checkpoint.errorState as? ErrorState.Errored)?.let { createDBFlowException(it, now) } - if (entity.exceptionDetails != null) { - if (exceptionDetails != null) { - exceptionDetails.id = entity.exceptionDetails!!.id - currentDBSession().update(exceptionDetails) - } else { - currentDBSession().delete(entity.exceptionDetails) - } - } else if (exceptionDetails != null) { - currentDBSession().save(exceptionDetails) - } + // DBFlowException to be integrated with rest of schema + // Add a flag notifying if an exception is already saved in the database for below logic (are we going to do this after all?) + private fun updateDBFlowException(flowId: String, checkpoint: Checkpoint, now: Instant): DBFlowException? { + val exceptionDetails = (checkpoint.errorState as? ErrorState.Errored)?.let { createDBFlowException(flowId, it, now) } +// if (checkpoint.dbExoSkeleton.dbFlowExceptionId != null) { +// if (exceptionDetails != null) { +// exceptionDetails.flow_id = checkpoint.dbExoSkeleton.dbFlowExceptionId!! +// currentDBSession().update(exceptionDetails) +// } else { +// val session = currentDBSession() +// val entity = session.get(DBFlowException::class.java, checkpoint.dbExoSkeleton.dbFlowExceptionId) +// session.delete(entity) +// return null +// } +// } else if (exceptionDetails != null) { +// currentDBSession().save(exceptionDetails) +// checkpoint.dbExoSkeleton.dbFlowExceptionId = exceptionDetails.flow_id +// } return exceptionDetails } - private fun createDBFlowException(errorState: ErrorState.Errored, now: Instant): DBFlowException { + private fun createDBFlowException(flowId: String, errorState: ErrorState.Errored, now: Instant): DBFlowException { return errorState.errors.last().exception.let { DBFlowException( + flow_id = flowId, type = it::class.java.name.truncate(MAX_EXC_TYPE_LENGTH, true), message = it.message?.truncate(MAX_EXC_MSG_LENGTH, false), stackTrace = it.stackTraceToString(), @@ -542,9 +554,9 @@ class DBCheckpointStorage( } private fun DBFlowCheckpoint.toSerializedCheckpoint(): Checkpoint.Serialized { - val serialisedFlowState = blob.flowStack?.let { SerializedBytes(it) } + val serialisedFlowState = blob!!.flowStack?.let { SerializedBytes(it) } return Checkpoint.Serialized( - serializedCheckpointState = SerializedBytes(blob.checkpoint), + serializedCheckpointState = SerializedBytes(blob!!.checkpoint), serializedFlowState = serialisedFlowState, // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint errorState = ErrorState.Clean, @@ -558,24 +570,24 @@ class DBCheckpointStorage( } private class DBPausedFields( - val id: String, - val checkpoint: ByteArray = EMPTY_BYTE_ARRAY, - val status: FlowStatus, - val progressStep: String?, - val ioRequestType: String?, - val compatible: Boolean + val id: String, + val checkpoint: ByteArray = EMPTY_BYTE_ARRAY, + val status: FlowStatus, + val progressStep: String?, + val ioRequestType: String?, + val compatible: Boolean ) { fun toSerializedCheckpoint(): Checkpoint.Serialized { return Checkpoint.Serialized( - serializedCheckpointState = SerializedBytes(checkpoint), - serializedFlowState = null, - // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint - errorState = ErrorState.Clean, - result = null, - status = status, - progressStep = progressStep, - flowIoRequest = ioRequestType, - compatible = compatible + serializedCheckpointState = SerializedBytes(checkpoint), + serializedFlowState = null, + // Always load as a [Clean] checkpoint to represent that the checkpoint is the last _good_ checkpoint + errorState = ErrorState.Clean, + result = null, + status = status, + progressStep = progressStep, + flowIoRequest = ioRequestType, + compatible = compatible ) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index 925fdc99a7..c3ddadd716 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -4,6 +4,7 @@ import co.paralleluniverse.fibers.Suspendable import com.codahale.metrics.Gauge import com.codahale.metrics.Reservoir import net.corda.core.internal.concurrent.thenMatch +import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.checkpointSerialize import net.corda.core.utilities.contextLogger @@ -90,15 +91,18 @@ class ActionExecutorImpl( val flowState = checkpoint.flowState val serializedFlowState = when(flowState) { FlowState.Completed -> null + // upon implementing CORDA-3816: If we have errored or hospitalized then we don't need to serialize the flowState as it will not get saved in the DB else -> flowState.checkpointSerialize(checkpointSerializationContext) } + // upon implementing CORDA-3816: If we have errored or hospitalized then we don't need to serialize the serializedCheckpointState as it will not get saved in the DB + val serializedCheckpointState: SerializedBytes = checkpoint.checkpointState.checkpointSerialize(checkpointSerializationContext) if (action.isCheckpointUpdate) { - checkpointStorage.updateCheckpoint(action.id, checkpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(action.id, checkpoint, serializedFlowState, serializedCheckpointState) } else { if (flowState is FlowState.Completed) { throw IllegalStateException("A new checkpoint cannot be created with a Completed FlowState.") } - checkpointStorage.addCheckpoint(action.id, checkpoint, serializedFlowState!!) + checkpointStorage.addCheckpoint(action.id, checkpoint, serializedFlowState!!, serializedCheckpointState) } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt index 6c88e97ea1..58a072fc99 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineState.kt @@ -176,7 +176,7 @@ data class Checkpoint( else -> serializedFlowState!!.checkpointDeserialize(checkpointSerializationContext) } return Checkpoint( - checkpointState = serializedCheckpointState.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), + checkpointState = serializedCheckpointState.checkpointDeserialize(checkpointSerializationContext), flowState = flowState, errorState = errorState, result = result?.deserialize(context = SerializationDefaults.STORAGE_CONTEXT), diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index 8c5a7916e1..9e96e93d01 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -31,8 +31,8 @@ - - - + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml b/node/src/main/resources/migration/node-core.changelog-v17-keys.xml deleted file mode 100644 index a8e82b3966..0000000000 --- a/node/src/main/resources/migration/node-core.changelog-v17-keys.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/node/src/main/resources/migration/node-core.changelog-v19-keys.xml b/node/src/main/resources/migration/node-core.changelog-v19-keys.xml new file mode 100644 index 0000000000..26359ecd2f --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v19-keys.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml similarity index 85% rename from node/src/main/resources/migration/node-core.changelog-v17-postgres.xml rename to node/src/main/resources/migration/node-core.changelog-v19-postgres.xml index 723cfb1951..3f9ed5cab1 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml @@ -4,22 +4,13 @@ xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd" logicalFilePath="migration/node-services.changelog-init.xml"> - + - - - - - - - - - - + @@ -38,9 +29,9 @@ - + - + @@ -52,16 +43,16 @@ - + - + - + @@ -73,12 +64,12 @@ - + - + - + @@ -96,7 +87,7 @@ - + @@ -110,7 +101,7 @@ - + @@ -119,7 +110,7 @@ - + diff --git a/node/src/main/resources/migration/node-core.changelog-v17.xml b/node/src/main/resources/migration/node-core.changelog-v19.xml similarity index 87% rename from node/src/main/resources/migration/node-core.changelog-v17.xml rename to node/src/main/resources/migration/node-core.changelog-v19.xml index 02a8d16f7b..cba014503c 100644 --- a/node/src/main/resources/migration/node-core.changelog-v17.xml +++ b/node/src/main/resources/migration/node-core.changelog-v19.xml @@ -10,16 +10,7 @@ - - - - - - - - - - + @@ -40,7 +31,7 @@ - + @@ -52,7 +43,7 @@ - + @@ -61,7 +52,7 @@ - + @@ -75,10 +66,10 @@ - + - + @@ -110,7 +101,7 @@ - + @@ -119,7 +110,7 @@ - + diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index f47a363c16..a75960523b 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -86,7 +86,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { assertEquals(serializedFlowState, checkpointStorage.checkpoints().single().serializedFlowState) @@ -113,7 +113,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val logic: FlowLogic<*> = object : FlowLogic() { override fun call(): String { @@ -131,7 +131,7 @@ class DBCheckpointStorageTests { ) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals( @@ -146,12 +146,12 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val completedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.COMPLETED) database.transaction { - checkpointStorage.updateCheckpoint(id, completedCheckpoint, null) + checkpointStorage.updateCheckpoint(id, completedCheckpoint, null, completedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals( @@ -166,12 +166,12 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val pausedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) database.transaction { - checkpointStorage.updateCheckpoint(id, pausedCheckpoint, null) + checkpointStorage.updateCheckpoint(id, pausedCheckpoint, null, pausedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals( @@ -181,17 +181,18 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `removing a checkpoint deletes from all checkpoint tables`() { val exception = IllegalStateException("I am a naughty exception") val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.addError(exception).copy(result = "The result") val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals(1, findRecordsFromDatabase().size) @@ -224,17 +225,18 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `removing a checkpoint when there is no result does not fail`() { val exception = IllegalStateException("I am a naughty exception") val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.addError(exception) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals(1, findRecordsFromDatabase().size) @@ -272,19 +274,18 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.copy(result = "The result") val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals(0, findRecordsFromDatabase().size) // The result not stored yet assertEquals(0, findRecordsFromDatabase().size) assertEquals(1, findRecordsFromDatabase().size) - // The saving of checkpoint blobs needs to be fixed - assertEquals(2, findRecordsFromDatabase().size) + assertEquals(1, findRecordsFromDatabase().size) assertEquals(1, findRecordsFromDatabase().size) } @@ -303,8 +304,7 @@ class DBCheckpointStorageTests { assertEquals(0, findRecordsFromDatabase().size) assertEquals(0, findRecordsFromDatabase().size) assertEquals(0, findRecordsFromDatabase().size) - // The saving of checkpoint blobs needs to be fixed - assertEquals(1, findRecordsFromDatabase().size) + assertEquals(0, findRecordsFromDatabase().size) assertEquals(0, findRecordsFromDatabase().size) } } @@ -316,8 +316,8 @@ class DBCheckpointStorageTests { val (id2, checkpoint2) = newCheckpoint() val serializedFlowState2 = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(id2, checkpoint2, serializedFlowState2) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(id2, checkpoint2, serializedFlowState2, checkpoint2.serializeCheckpointState()) checkpointStorage.removeCheckpoint(id) } database.transaction { @@ -341,12 +341,12 @@ class DBCheckpointStorageTests { val serializedFirstFlowState = firstCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, firstCheckpoint, serializedFirstFlowState) + checkpointStorage.addCheckpoint(id, firstCheckpoint, serializedFirstFlowState, firstCheckpoint.serializeCheckpointState()) } val (id2, secondCheckpoint) = newCheckpoint() val serializedSecondFlowState = secondCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id2, secondCheckpoint, serializedSecondFlowState) + checkpointStorage.addCheckpoint(id2, secondCheckpoint, serializedSecondFlowState, secondCheckpoint.serializeCheckpointState()) } database.transaction { checkpointStorage.removeCheckpoint(id) @@ -371,7 +371,7 @@ class DBCheckpointStorageTests { val (id, originalCheckpoint) = newCheckpoint() val serializedOriginalFlowState = originalCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, originalCheckpoint, serializedOriginalFlowState) + checkpointStorage.addCheckpoint(id, originalCheckpoint, serializedOriginalFlowState, originalCheckpoint.serializeCheckpointState()) } newCheckpointStorage() val reconstructedCheckpoint = database.transaction { @@ -395,7 +395,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()).also { @@ -409,7 +409,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val metadata = database.transaction { session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()).also { @@ -422,7 +422,7 @@ class DBCheckpointStorageTests { ) ) database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, serializedFlowState, updatedCheckpoint.serializeCheckpointState()) } val potentiallyUpdatedMetadata = database.transaction { session.get(DBCheckpointStorage.DBFlowMetadata::class.java, id.uuid.toString()) @@ -436,7 +436,7 @@ class DBCheckpointStorageTests { database.transaction { val (id, checkpoint) = newCheckpoint(1) val serializedFlowState = checkpoint.serializeFlowState() - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { @@ -446,7 +446,7 @@ class DBCheckpointStorageTests { database.transaction { val (id1, checkpoint1) = newCheckpoint(2) val serializedFlowState1 = checkpoint1.serializeFlowState() - checkpointStorage.addCheckpoint(id1, checkpoint1, serializedFlowState1) + checkpointStorage.addCheckpoint(id1, checkpoint1, serializedFlowState1, checkpoint1.serializeCheckpointState()) } assertThatThrownBy { @@ -464,12 +464,12 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.copy(result = result) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { assertEquals( @@ -490,17 +490,17 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.copy(result = result) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } val updatedCheckpoint2 = checkpoint.copy(result = somehowThereIsANewResult) val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2, updatedCheckpoint2.serializeCheckpointState()) } database.transaction { assertEquals( @@ -519,17 +519,17 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.copy(result = result) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } val updatedCheckpoint2 = checkpoint.copy(result = null) val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() database.transaction { - checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) + checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2, updatedCheckpoint2.serializeCheckpointState()) } database.transaction { assertNull(checkpointStorage.getCheckpoint(id)!!.deserialize().result) @@ -538,17 +538,18 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `update checkpoint with error information creates a new error database record`() { val exception = IllegalStateException("I am a naughty exception") val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.addError(exception) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { // Checkpoint always returns clean error state when retrieved via [getCheckpoint] assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) @@ -560,6 +561,7 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `update checkpoint with new error information updates the existing error database record`() { val illegalStateException = IllegalStateException("I am a naughty exception") @@ -567,15 +569,15 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint1 = checkpoint.addError(illegalStateException) val updatedSerializedFlowState1 = updatedCheckpoint1.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint1, updatedSerializedFlowState1) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint1, updatedSerializedFlowState1, updatedCheckpoint1.serializeCheckpointState()) } // Set back to clean val updatedCheckpoint2 = checkpoint.addError(illegalArgumentException) val updatedSerializedFlowState2 = updatedCheckpoint2.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint2, updatedSerializedFlowState2, updatedCheckpoint2.serializeCheckpointState()) } database.transaction { assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) val exceptionDetails = session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails @@ -592,17 +594,17 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint() val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } val updatedCheckpoint = checkpoint.addError(exception) val updatedSerializedFlowState = updatedCheckpoint.serializeFlowState() - database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, updatedCheckpoint, updatedSerializedFlowState, updatedCheckpoint.serializeCheckpointState()) } database.transaction { // Checkpoint always returns clean error state when retrieved via [getCheckpoint] assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) } // Set back to clean - database.transaction { checkpointStorage.updateCheckpoint(id, checkpoint, serializedFlowState) } + database.transaction { checkpointStorage.updateCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { assertTrue(checkpointStorage.getCheckpoint(id)!!.deserialize().errorState is ErrorState.Clean) assertNull(session.get(DBCheckpointStorage.DBFlowCheckpoint::class.java, id.uuid.toString()).exceptionDetails) @@ -615,7 +617,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint(1) database.transaction { val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) val checkpointFromStorage = checkpointStorage.getCheckpoint(id) assertNull(checkpointFromStorage!!.flowIoRequest) } @@ -624,7 +626,7 @@ class DBCheckpointStorageTests { val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT ) - checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState, newCheckpoint.serializeCheckpointState()) } database.transaction { val checkpointFromStorage = checkpointStorage.getCheckpoint(id) @@ -640,7 +642,7 @@ class DBCheckpointStorageTests { val (id, checkpoint) = newCheckpoint(1) database.transaction { val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) val checkpointFromStorage = checkpointStorage.getCheckpoint(id) assertNull(checkpointFromStorage!!.progressStep) } @@ -653,7 +655,7 @@ class DBCheckpointStorageTests { val serializedFlowState = newCheckpoint.flowState.checkpointSerialize( context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT ) - checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState) + checkpointStorage.updateCheckpoint(id, newCheckpoint, serializedFlowState, newCheckpoint.serializeCheckpointState()) } database.transaction { val checkpointFromStorage = checkpointStorage.getCheckpoint(id) @@ -678,12 +680,12 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), runnable, serializedFlowState) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), hospitalized, serializedFlowState) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), completed, serializedFlowState) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), failed, serializedFlowState) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), killed, serializedFlowState) - checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), paused, serializedFlowState) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), runnable, serializedFlowState, runnable.serializeCheckpointState()) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), hospitalized, serializedFlowState, hospitalized.serializeCheckpointState()) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), completed, serializedFlowState, completed.serializeCheckpointState()) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), failed, serializedFlowState, failed.serializeCheckpointState()) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), killed, serializedFlowState, killed.serializeCheckpointState()) + checkpointStorage.addCheckpoint(StateMachineRunId.createRandom(), paused, serializedFlowState, paused.serializeCheckpointState()) } database.transaction { @@ -699,6 +701,7 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `-not greater than DBCheckpointStorage_MAX_STACKTRACE_LENGTH- stackTrace gets persisted as a whole`() { val smallerDummyStackTrace = ArrayList() @@ -716,10 +719,12 @@ class DBCheckpointStorageTests { assertTrue(smallerStackTraceSize < DBCheckpointStorage.MAX_STACKTRACE_LENGTH) val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - val serializedFlowState = checkpoint.serializeFlowState() - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) - checkpointStorage.updateCheckpoint(id, checkpoint.addError(smallerStackTraceException), serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) + } + database.transaction { + checkpointStorage.updateCheckpoint(id, checkpoint.addError(smallerStackTraceException), serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { val persistedCheckpoint = checkpointStorage.getDBCheckpoint(id) @@ -729,6 +734,7 @@ class DBCheckpointStorageTests { } } + @Ignore @Test(timeout = 300_000) fun `-greater than DBCheckpointStorage_MAX_STACKTRACE_LENGTH- stackTrace gets truncated to MAX_LENGTH_VARCHAR, and persisted`() { val smallerDummyStackTrace = ArrayList() @@ -755,10 +761,12 @@ class DBCheckpointStorageTests { assertTrue(biggerStackTraceSize > DBCheckpointStorage.MAX_STACKTRACE_LENGTH) val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() database.transaction { - val serializedFlowState = checkpoint.serializeFlowState() - checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState) - checkpointStorage.updateCheckpoint(id, checkpoint.addError(biggerStackTraceException), serializedFlowState) + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) + } + database.transaction { + checkpointStorage.updateCheckpoint(id, checkpoint.addError(biggerStackTraceException), serializedFlowState, checkpoint.serializeCheckpointState()) } database.transaction { val persistedCheckpoint = checkpointStorage.getDBCheckpoint(id) @@ -784,7 +792,7 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.serializeFlowState() val pausedCheckpoint = checkpoint.copy(status = Checkpoint.FlowStatus.PAUSED) database.transaction { - checkpointStorage.addCheckpoint(id, pausedCheckpoint, serializedFlowState) + checkpointStorage.addCheckpoint(id, pausedCheckpoint, serializedFlowState, pausedCheckpoint.serializeCheckpointState()) } database.transaction { @@ -820,12 +828,12 @@ class DBCheckpointStorageTests { val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) - checkpointStorage.addCheckpoint(runnable.id, runnable.checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(hospitalized.id, hospitalized.checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(completed.id, completed.checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(failed.id, failed.checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(killed.id, killed.checkpoint, serializedFlowState) - checkpointStorage.addCheckpoint(paused.id, paused.checkpoint, serializedFlowState) + checkpointStorage.addCheckpoint(runnable.id, runnable.checkpoint, serializedFlowState, runnable.checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(hospitalized.id, hospitalized.checkpoint, serializedFlowState, hospitalized.checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(completed.id, completed.checkpoint, serializedFlowState, completed.checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(failed.id, failed.checkpoint, serializedFlowState, failed.checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(killed.id, killed.checkpoint, serializedFlowState, killed.checkpoint.serializeCheckpointState()) + checkpointStorage.addCheckpoint(paused.id, paused.checkpoint, serializedFlowState, paused.checkpoint.serializeCheckpointState()) } database.transaction { @@ -844,6 +852,26 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `updateCheckpoint setting DBFlowCheckpoint_blob to null whenever flow fails or gets hospitalized doesn't break ORM relationship`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) + } + + database.transaction { + val paused = changeStatus(checkpoint, Checkpoint.FlowStatus.FAILED) // the exact same behaviour applies for 'HOSPITALIZED' as well + checkpointStorage.updateCheckpoint(id, paused.checkpoint, serializedFlowState, paused.checkpoint.serializeCheckpointState()) + } + + database.transaction { + val dbFlowCheckpoint= checkpointStorage.getDBCheckpoint(id) + assert(dbFlowCheckpoint!!.blob != null) + } + } + data class IdAndCheckpoint(val id: StateMachineRunId, val checkpoint: Checkpoint) private fun changeStatus(oldCheckpoint: Checkpoint, status: Checkpoint.FlowStatus): IdAndCheckpoint { @@ -889,6 +917,10 @@ class DBCheckpointStorageTests { return flowState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) } + private fun Checkpoint.serializeCheckpointState(): SerializedBytes { + return checkpointState.checkpointSerialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + } + private fun Checkpoint.Serialized.deserialize(): Checkpoint { return deserialize(CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) } diff --git a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt index 4eb5bd5a98..4037bd80f0 100644 --- a/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/rpc/CheckpointDumperImplTest.kt @@ -108,7 +108,7 @@ class CheckpointDumperImplTest { // add a checkpoint val (id, checkpoint) = newCheckpoint() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint), serializeCheckpointState(checkpoint)) } dumper.dumpCheckpoints() @@ -123,14 +123,14 @@ class CheckpointDumperImplTest { // add a checkpoint val (id, checkpoint) = newCheckpoint() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint), serializeCheckpointState(checkpoint)) } val newCheckpoint = checkpoint.copy( flowState = FlowState.Completed, status = Checkpoint.FlowStatus.COMPLETED ) database.transaction { - checkpointStorage.updateCheckpoint(id, newCheckpoint, null) + checkpointStorage.updateCheckpoint(id, newCheckpoint, null, serializeCheckpointState(newCheckpoint)) } dumper.dumpCheckpoints() @@ -163,7 +163,7 @@ class CheckpointDumperImplTest { // add a checkpoint val (id, checkpoint) = newCheckpoint() database.transaction { - checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint)) + checkpointStorage.addCheckpoint(id, checkpoint, serializeFlowState(checkpoint), serializeCheckpointState(checkpoint)) } dumper.dumpCheckpoints() @@ -201,4 +201,8 @@ class CheckpointDumperImplTest { private fun serializeFlowState(checkpoint: Checkpoint): SerializedBytes { return checkpoint.flowState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) } + + private fun serializeCheckpointState(checkpoint: Checkpoint): SerializedBytes { + return checkpoint.checkpointState.checkpointSerialize(context = CheckpointSerializationDefaults.CHECKPOINT_CONTEXT) + } } diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 2b84537bdc..c7e94086c2 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -44,6 +44,7 @@ import net.corda.node.services.persistence.CheckpointPerformanceRecorder import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.checkpoints import net.corda.nodeapi.internal.persistence.DatabaseTransaction +import net.corda.nodeapi.internal.persistence.currentDBSession import net.corda.testing.contracts.DummyContract import net.corda.testing.contracts.DummyState import net.corda.testing.core.ALICE_NAME @@ -683,6 +684,7 @@ class FlowFrameworkTests { throw HospitalizeFlowException() } else { dbCheckpointStatusBeforeSuspension = aliceNode.internals.checkpointStorage.getCheckpoints().toList().single().second.status + currentDBSession().clear() // clear session as Hibernate with fails with 'org.hibernate.NonUniqueObjectException' once it tries to save a DBFlowCheckpoint upon checkpoint inMemoryCheckpointStatusBeforeSuspension = flowFiber.transientState!!.value.checkpoint.status futureFiber.complete(flowFiber) @@ -754,6 +756,8 @@ class FlowFrameworkTests { assertEquals(Checkpoint.FlowStatus.RUNNABLE, inMemoryCheckpointStatus) } + // Upon implementing CORDA-3681 unignore this test; DBFlowException is not currently integrated + @Ignore @Test(timeout=300_000) fun `Checkpoint is updated in DB with FAILED status and the error when flow fails`() { var flowId: StateMachineRunId? = null @@ -777,6 +781,8 @@ class FlowFrameworkTests { } } + // Upon implementing CORDA-3681 unignore this test; DBFlowException is not currently integrated + @Ignore @Test(timeout=300_000) fun `Checkpoint is updated in DB with HOSPITALIZED status and the error when flow is kept for overnight observation` () { var flowId: StateMachineRunId? = null From 4507b55857deb3ffe549339a0d025982c0149339 Mon Sep 17 00:00:00 2001 From: Kyriakos Tharrouniatis Date: Fri, 29 May 2020 12:35:05 +0100 Subject: [PATCH 48/49] CORDA-3725 Fix SQL deadlocks coming from soft locking states (#6287) Adding to the query -explicitly- the flow's locked states resolved the SQL Deadlocks in SQL server. The SQL Deadlocks would come up from `softLockRelease` when only the `lockId` was passed in as argument. In that case the query optimizer would use `lock_id_idx(lock_id, state_status)` to search and update entries in `VAULT_STATES` table. However, all rest of the queries would follow the opposite direction meaning they would use PK's `index(output_index, transaction_id)` but they would also update the `lock_id` column and therefore the `lock_id_idx` as well, because `lock_id` is a part of it. That was causing a circular locking among the different transactions (SQL processes) within the database. To resolve this, whenever a flow attempts to reserve soft locks using their flow id (remember the flow id is always the flow id for the very first flow in the flow stack), we then save these states to the fiber. Then, upon releasing soft locks the fiber passes that set to the release soft locks query. That way the database query optimizer will use the primary key index of VAULT_STATES table, instead of lock_id_idx in order to search rows to update. That way the query will be aligned with the rest of the queries that are following that route as well (i.e. making use of the primary key), and therefore its locking order of resources within the database will be aligned with the rest queries' locking orders (solving SQL deadlocks). * Fixed SQL deadlocks caused from softLockRelease, by saving locked states per fiber; NodeVaultService.softLockRelease query then uses VAULT_STATES PK index instead of lock_id_idx Speed up SQL server by breaking down queries with > 16 elements in their IN clause, into sub queries with 16 elements max in their IN clauses * Allow softLockedStates to remove states * Add to softLockedStates only states soft locked under our flow id * Fix softLockRelease not to take into account flowStateMachineImpl.softLockedStates when using lockId != ourFlowId * Moved CriteriaBuilder.executeUpdate at the bottom of the file --- .../statemachine/FlowStateMachineImpl.kt | 8 +- .../node/services/vault/NodeVaultService.kt | 93 ++++- .../statemachine/FlowFrameworkTests.kt | 3 +- .../statemachine/FlowSoftLocksTests.kt | 330 ++++++++++++++++++ .../services/vault/NodeVaultServiceTest.kt | 41 +++ 5 files changed, 452 insertions(+), 23 deletions(-) create mode 100644 node/src/test/kotlin/net/corda/node/services/statemachine/FlowSoftLocksTests.kt diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 3bc833dfdb..1c85348981 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -8,6 +8,7 @@ import co.paralleluniverse.strands.Strand import co.paralleluniverse.strands.channels.Channel import net.corda.core.concurrent.CordaFuture import net.corda.core.context.InvocationContext +import net.corda.core.contracts.StateRef import net.corda.core.cordapp.Cordapp import net.corda.core.flows.Destination import net.corda.core.flows.FlowException @@ -132,10 +133,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, override val ourIdentity: Party get() = transientState!!.value.checkpoint.checkpointState.ourIdentity override val isKilled: Boolean get() = transientState!!.value.isKilled - internal var hasSoftLockedStates: Boolean = false - set(value) { - if (value) field = value else throw IllegalArgumentException("Can only set to true") - } + internal val softLockedStates = mutableSetOf() /** * Processes an event by creating the associated transition and executing it using the given executor. @@ -306,7 +304,7 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, logger.info("Flow raised an error: ${t.message}. Sending it to flow hospital to be triaged.") Try.Failure(t) } - val softLocksId = if (hasSoftLockedStates) logic.runId.uuid else null + val softLocksId = if (softLockedStates.isNotEmpty()) logic.runId.uuid else null val finalEvent = when (resultOrError) { is Try.Success -> { Event.FlowFinish(resultOrError.value, softLocksId) diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index 438e2c8dec..6dfd3b6e04 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -42,11 +42,6 @@ import javax.persistence.criteria.CriteriaUpdate import javax.persistence.criteria.Predicate import javax.persistence.criteria.Root -private fun CriteriaBuilder.executeUpdate(session: Session, configure: Root<*>.(CriteriaUpdate<*>) -> Any?) = createCriteriaUpdate(VaultSchemaV1.VaultStates::class.java).let { update -> - update.from(VaultSchemaV1.VaultStates::class.java).run { configure(update) } - session.createQuery(update).executeUpdate() -} - /** * The vault service handles storage, retrieval and querying of states. * @@ -67,6 +62,8 @@ class NodeVaultService( companion object { private val log = contextLogger() + val MAX_SQL_IN_CLAUSE_SET = 16 + /** * Establish whether a given state is relevant to a node, given the node's public keys. * @@ -462,13 +459,20 @@ class NodeVaultService( } } + /** + * Whenever executed inside a [FlowStateMachineImpl], if [lockId] refers to the currently running [FlowStateMachineImpl], + * then in that case the [FlowStateMachineImpl] instance is locking states with its [FlowStateMachineImpl.id]'s [UUID]. + * In this case alone, we keep the reserved set of [StateRef] in [FlowStateMachineImpl.softLockedStates]. This set will be then + * used by default in [softLockRelease]. + */ + @Suppress("NestedBlockDepth", "ComplexMethod") @Throws(StatesNotAvailableException::class) override fun softLockReserve(lockId: UUID, stateRefs: NonEmptySet) { val softLockTimestamp = clock.instant() try { val session = currentDBSession() val criteriaBuilder = session.criteriaBuilder - fun execute(configure: Root<*>.(CriteriaUpdate<*>, Array) -> Any?) = criteriaBuilder.executeUpdate(session) { update -> + fun execute(configure: Root<*>.(CriteriaUpdate<*>, Array) -> Any?) = criteriaBuilder.executeUpdate(session, null) { update, _ -> val persistentStateRefs = stateRefs.map { PersistentStateRef(it.txhash.bytes.toHexString(), it.index) } val compositeKey = get(VaultSchemaV1.VaultStates::stateRef.name) val stateRefsPredicate = criteriaBuilder.and(compositeKey.`in`(persistentStateRefs)) @@ -485,7 +489,11 @@ class NodeVaultService( } if (updatedRows > 0 && updatedRows == stateRefs.size) { log.trace { "Reserving soft lock states for $lockId: $stateRefs" } - FlowStateMachineImpl.currentStateMachine()?.hasSoftLockedStates = true + FlowStateMachineImpl.currentStateMachine()?.let { + if (lockId == it.id.uuid) { + it.softLockedStates.addAll(stateRefs) + } + } } else { // revert partial soft locks val revertUpdatedRows = execute { update, commonPredicates -> @@ -508,19 +516,44 @@ class NodeVaultService( } } + /** + * Whenever executed inside a [FlowStateMachineImpl], if [lockId] refers to the currently running [FlowStateMachineImpl] and [stateRefs] is null, + * then in that case the [FlowStateMachineImpl] instance will, by default, retrieve its set of [StateRef] + * from [FlowStateMachineImpl.softLockedStates] (previously reserved from [softLockReserve]). This set will be then explicitly provided + * to the below query which then leads to the database query optimizer use the primary key index in VAULT_STATES table, instead of lock_id_idx + * in order to search rows to be updated. That way the query will be aligned with the rest of the queries that are following that route as well + * (i.e. making use of the primary key), and therefore its locking order of resources within the database will be aligned + * with the rest queries' locking orders (solving SQL deadlocks). + * + * If [lockId] does not refer to the currently running [FlowStateMachineImpl] and [stateRefs] is null, then it will be using only [lockId] in + * the below query. + */ + @Suppress("NestedBlockDepth", "ComplexMethod") override fun softLockRelease(lockId: UUID, stateRefs: NonEmptySet?) { val softLockTimestamp = clock.instant() val session = currentDBSession() val criteriaBuilder = session.criteriaBuilder - fun execute(configure: Root<*>.(CriteriaUpdate<*>, Array) -> Any?) = criteriaBuilder.executeUpdate(session) { update -> + fun execute(stateRefs: NonEmptySet?, configure: Root<*>.(CriteriaUpdate<*>, Array, List?) -> Any?) = + criteriaBuilder.executeUpdate(session, stateRefs) { update, persistentStateRefs -> val stateStatusPredication = criteriaBuilder.equal(get(VaultSchemaV1.VaultStates::stateStatus.name), Vault.StateStatus.UNCONSUMED) val lockIdPredicate = criteriaBuilder.equal(get(VaultSchemaV1.VaultStates::lockId.name), lockId.toString()) update.set(get(VaultSchemaV1.VaultStates::lockId.name), criteriaBuilder.nullLiteral(String::class.java)) update.set(get(VaultSchemaV1.VaultStates::lockUpdateTime.name), softLockTimestamp) - configure(update, arrayOf(stateStatusPredication, lockIdPredicate)) + configure(update, arrayOf(stateStatusPredication, lockIdPredicate), persistentStateRefs) } - if (stateRefs == null) { - val update = execute { update, commonPredicates -> + + val stateRefsToBeReleased = + stateRefs ?: FlowStateMachineImpl.currentStateMachine()?.let { + // We only hold states under our flowId. For all other lockId fall back to old query mechanism, i.e. stateRefsToBeReleased = null + if (lockId == it.id.uuid && it.softLockedStates.isNotEmpty()) { + NonEmptySet.copyOf(it.softLockedStates) + } else { + null + } + } + + if (stateRefsToBeReleased == null) { + val update = execute(null) { update, commonPredicates, _ -> update.where(*commonPredicates) } if (update > 0) { @@ -528,19 +561,21 @@ class NodeVaultService( } } else { try { - val updatedRows = execute { update, commonPredicates -> - val persistentStateRefs = stateRefs.map { PersistentStateRef(it.txhash.bytes.toHexString(), it.index) } + val updatedRows = execute(stateRefsToBeReleased) { update, commonPredicates, persistentStateRefs -> val compositeKey = get(VaultSchemaV1.VaultStates::stateRef.name) val stateRefsPredicate = criteriaBuilder.and(compositeKey.`in`(persistentStateRefs)) update.where(*commonPredicates, stateRefsPredicate) } if (updatedRows > 0) { - log.trace { "Releasing $updatedRows soft locked states for $lockId and stateRefs $stateRefs" } + FlowStateMachineImpl.currentStateMachine()?.let { + if (lockId == it.id.uuid) { + it.softLockedStates.removeAll(stateRefsToBeReleased) + } + } + log.trace { "Releasing $updatedRows soft locked states for $lockId and stateRefs $stateRefsToBeReleased" } } } catch (e: Exception) { - log.error("""soft lock update error attempting to release states for $lockId and $stateRefs") - $e. - """) + log.error("Soft lock update error attempting to release states for $lockId and $stateRefsToBeReleased", e) throw e } } @@ -819,5 +854,29 @@ class NodeVaultService( } } +private fun CriteriaBuilder.executeUpdate( + session: Session, + stateRefs: NonEmptySet?, + configure: Root<*>.(CriteriaUpdate<*>, List?) -> Any? +): Int { + fun doUpdate(persistentStateRefs: List?): Int { + createCriteriaUpdate(VaultSchemaV1.VaultStates::class.java).let { update -> + update.from(VaultSchemaV1.VaultStates::class.java).run { configure(update, persistentStateRefs) } + return session.createQuery(update).executeUpdate() + } + } + return stateRefs?.let { + // Increase SQL server performance by, processing updates in chunks allowing the database's optimizer to make use of the index. + var updatedRows = 0 + it.asSequence() + .map { stateRef -> PersistentStateRef(stateRef.txhash.bytes.toHexString(), stateRef.index) } + .chunked(NodeVaultService.MAX_SQL_IN_CLAUSE_SET) + .forEach { persistentStateRefs -> + updatedRows += doUpdate(persistentStateRefs) + } + updatedRows + } ?: doUpdate(null) +} + /** The Observable returned allows subscribing with custom SafeSubscribers to source [Observable]. */ internal fun Observable.resilientOnError(): Observable = Observable.unsafeCreate(OnResilientSubscribe(this, false)) \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index c7e94086c2..feafb34279 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -56,6 +56,7 @@ import net.corda.testing.internal.LogHelper import net.corda.testing.node.InMemoryMessagingNetwork.MessageTransfer import net.corda.testing.node.InMemoryMessagingNetwork.ServicePeerAllocationStrategy.RoundRobin import net.corda.testing.node.internal.DUMMY_CONTRACTS_CORDAPP +import net.corda.testing.node.internal.FINANCE_CONTRACTS_CORDAPP import net.corda.testing.node.internal.InternalMockNetwork import net.corda.testing.node.internal.InternalMockNodeParameters import net.corda.testing.node.internal.TestStartedNode @@ -118,7 +119,7 @@ class FlowFrameworkTests { @Before fun setUpMockNet() { mockNet = InternalMockNetwork( - cordappsForAllNodes = listOf(DUMMY_CONTRACTS_CORDAPP), + cordappsForAllNodes = listOf(DUMMY_CONTRACTS_CORDAPP, FINANCE_CONTRACTS_CORDAPP), servicePeerAllocationStrategy = RoundRobin() ) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowSoftLocksTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowSoftLocksTests.kt new file mode 100644 index 0000000000..6f0fa3278c --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowSoftLocksTests.kt @@ -0,0 +1,330 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.contracts.Command +import net.corda.core.contracts.StateRef +import net.corda.core.flows.FlowLogic +import net.corda.core.identity.Party +import net.corda.core.internal.FlowIORequest +import net.corda.core.node.services.Vault +import net.corda.core.node.services.VaultService +import net.corda.core.node.services.queryBy +import net.corda.core.node.services.vault.QueryCriteria +import net.corda.core.transactions.TransactionBuilder +import net.corda.core.utilities.NonEmptySet +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.finance.DOLLARS +import net.corda.finance.contracts.asset.Cash +import net.corda.node.services.statemachine.FlowSoftLocksTests.Companion.queryCashStates +import net.corda.node.services.vault.NodeVaultServiceTest +import net.corda.testing.contracts.DummyContract +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOC_NAME +import net.corda.testing.core.TestIdentity +import net.corda.testing.core.singleIdentity +import net.corda.testing.internal.vault.VaultFiller +import net.corda.testing.node.internal.DUMMY_CONTRACTS_CORDAPP +import net.corda.testing.node.internal.FINANCE_CONTRACTS_CORDAPP +import net.corda.testing.node.internal.InternalMockNetwork +import net.corda.testing.node.internal.InternalMockNodeParameters +import net.corda.testing.node.internal.TestStartedNode +import net.corda.testing.node.internal.startFlow +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Before +import org.junit.Test +import java.lang.IllegalStateException +import java.sql.SQLTransientConnectionException +import java.util.UUID +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +class FlowSoftLocksTests { + + companion object { + fun queryCashStates(softLockingType: QueryCriteria.SoftLockingType, vaultService: VaultService) = + vaultService.queryBy( + QueryCriteria.VaultQueryCriteria( + softLockingCondition = QueryCriteria.SoftLockingCondition( + softLockingType + ) + ) + ).states.map { it.ref }.toSet() + + val EMPTY_SET = emptySet() + } + + private lateinit var mockNet: InternalMockNetwork + private lateinit var aliceNode: TestStartedNode + private lateinit var notaryIdentity: Party + + @Before + fun setUpMockNet() { + mockNet = InternalMockNetwork( + cordappsForAllNodes = listOf(DUMMY_CONTRACTS_CORDAPP, FINANCE_CONTRACTS_CORDAPP) + ) + aliceNode = mockNet.createNode(InternalMockNodeParameters(legalName = ALICE_NAME)) + notaryIdentity = mockNet.defaultNotaryIdentity + } + + @After + fun cleanUp() { + mockNet.stopNodes() + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with its own flow id and then manually releases them`() { + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = vaultStates), + SoftLockAction(SoftLockingAction.UNLOCK, null, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.UNLOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(vaultStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with its own flow id and by default releases them when completing`() { + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = vaultStates) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(vaultStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with its own flow id and by default releases them when errors`() { + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toSet() + val softLockActions = arrayOf( + SoftLockAction( + SoftLockingAction.LOCK, + null, + vaultStates, + ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), + expectedSoftLockedStates = vaultStates, + exception = IllegalStateException("Throwing error after flow has soft locked states") + ) + ) + assertFailsWith { + aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + } + assertEquals(vaultStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + LockingUnlockingFlow.throwOnlyOnce = true + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with random id and then manually releases them`() { + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, randomId, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET), + SoftLockAction(SoftLockingAction.UNLOCK, randomId, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.UNLOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(vaultStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with random id and does not release them upon completing`() { + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, randomId, vaultStates, ExpectedSoftLocks(vaultStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(vaultStates, queryCashStates(QueryCriteria.SoftLockingType.LOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow only releases by default reserved states with flow id upon completing`() { + // lock with flow id and random id, dont manually release any. At the end, check that only flow id ones got unlocked. + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toList() + val flowIdStates = vaultStates.subList(0, vaultStates.size / 2).toSet() + val randomIdStates = vaultStates.subList(vaultStates.size / 2, vaultStates.size).toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, flowIdStates, ExpectedSoftLocks(flowIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction(SoftLockingAction.LOCK, randomId, randomIdStates, ExpectedSoftLocks(flowIdStates + randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(flowIdStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + assertEquals(randomIdStates, queryCashStates(QueryCriteria.SoftLockingType.LOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with flow id and random id, then releases the flow id ones - assert the random id ones are still locked`() { + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toList() + val flowIdStates = vaultStates.subList(0, vaultStates.size / 2).toSet() + val randomIdStates = vaultStates.subList(vaultStates.size / 2, vaultStates.size).toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, flowIdStates, ExpectedSoftLocks(flowIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction(SoftLockingAction.LOCK, randomId, randomIdStates, ExpectedSoftLocks(flowIdStates + randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction(SoftLockingAction.UNLOCK, null, flowIdStates, ExpectedSoftLocks(randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(flowIdStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + assertEquals(randomIdStates, queryCashStates(QueryCriteria.SoftLockingType.LOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow reserves fungible states with flow id and random id, then releases the random id ones - assert the flow id ones are still locked inside the flow`() { + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toList() + val flowIdStates = vaultStates.subList(0, vaultStates.size / 2).toSet() + val randomIdStates = vaultStates.subList(vaultStates.size / 2, vaultStates.size).toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, flowIdStates, ExpectedSoftLocks(flowIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction(SoftLockingAction.LOCK, randomId, randomIdStates, ExpectedSoftLocks(flowIdStates + randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction(SoftLockingAction.UNLOCK, randomId, randomIdStates, ExpectedSoftLocks(flowIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(flowIdStates + randomIdStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + } + + @Test(timeout=300_000) + fun `flow soft locks fungible state upon creation`() { + var lockedStates = 0 + CreateFungibleStateFLow.hook = { vaultService -> + lockedStates = vaultService.queryBy( + QueryCriteria.VaultQueryCriteria(softLockingCondition = QueryCriteria.SoftLockingCondition(QueryCriteria.SoftLockingType.LOCKED_ONLY)) + ).states.size + } + aliceNode.services.startFlow(CreateFungibleStateFLow()).resultFuture.getOrThrow(30.seconds) + assertEquals(1, lockedStates) + } + + @Test(timeout=300_000) + fun `when flow soft locks, then errors and retries from previous checkpoint, softLockedStates are reverted back correctly`() { + val randomId = UUID.randomUUID() + val vaultStates = fillVault(aliceNode, 10)!!.states.map { it.ref }.toList() + val flowIdStates = vaultStates.subList(0, vaultStates.size / 2).toSet() + val randomIdStates = vaultStates.subList(vaultStates.size / 2, vaultStates.size).toSet() + val softLockActions = arrayOf( + SoftLockAction(SoftLockingAction.LOCK, null, flowIdStates, ExpectedSoftLocks(flowIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = flowIdStates), + SoftLockAction( + SoftLockingAction.LOCK, + randomId, + randomIdStates, + ExpectedSoftLocks(flowIdStates + randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), + expectedSoftLockedStates = flowIdStates, + doCheckpoint = true + ), + SoftLockAction(SoftLockingAction.UNLOCK, null, flowIdStates, ExpectedSoftLocks(randomIdStates, QueryCriteria.SoftLockingType.LOCKED_ONLY), expectedSoftLockedStates = EMPTY_SET), + SoftLockAction( + SoftLockingAction.UNLOCK, + randomId, + randomIdStates, + ExpectedSoftLocks(EMPTY_SET, QueryCriteria.SoftLockingType.LOCKED_ONLY), + expectedSoftLockedStates = EMPTY_SET, + exception = SQLTransientConnectionException("connection is not available") + ) + ) + val flowCompleted = aliceNode.services.startFlow(LockingUnlockingFlow(softLockActions)).resultFuture.getOrThrow(30.seconds) + assertTrue(flowCompleted) + assertEquals(flowIdStates + randomIdStates, queryCashStates(QueryCriteria.SoftLockingType.UNLOCKED_ONLY, aliceNode.services.vaultService)) + LockingUnlockingFlow.throwOnlyOnce = true + } + + private fun fillVault(node: TestStartedNode, thisManyStates: Int): Vault? { + val bankNode = mockNet.createPartyNode(BOC_NAME) + val bank = bankNode.info.singleIdentity() + val cashIssuer = bank.ref(1) + return node.database.transaction { + VaultFiller(node.services, TestIdentity(notaryIdentity.name, 20), notaryIdentity).fillWithSomeTestCash( + 100.DOLLARS, + bankNode.services, + thisManyStates, + thisManyStates, + cashIssuer + ) + } + } +} + +enum class SoftLockingAction { + LOCK, + UNLOCK +} + +data class ExpectedSoftLocks(val states: Set, val queryCriteria: QueryCriteria.SoftLockingType) + +/** + * If [lockId] is set to null, it will be populated with the flowId within the flow. + */ +data class SoftLockAction(val action: SoftLockingAction, + var lockId: UUID?, + val states: Set, + val expectedSoftLocks: ExpectedSoftLocks, + val expectedSoftLockedStates: Set, + val exception: Exception? = null, + val doCheckpoint: Boolean = false) + +internal class LockingUnlockingFlow(private val softLockActions: Array): FlowLogic() { + + companion object { + var throwOnlyOnce = true + } + + @Suspendable + override fun call(): Boolean { + for (softLockAction in softLockActions) { + if (softLockAction.lockId == null) { softLockAction.lockId = stateMachine.id.uuid } + + when (softLockAction.action) { + SoftLockingAction.LOCK -> { + serviceHub.vaultService.softLockReserve(softLockAction.lockId!!, NonEmptySet.copyOf(softLockAction.states)) + // We checkpoint here so that, upon retrying to assert state after reserving + if (softLockAction.doCheckpoint) { + stateMachine.suspend(FlowIORequest.ForceCheckpoint, false) + } + assertEquals(softLockAction.expectedSoftLocks.states, queryCashStates(softLockAction.expectedSoftLocks.queryCriteria, serviceHub.vaultService)) + assertEquals(softLockAction.expectedSoftLockedStates, (stateMachine as? FlowStateMachineImpl<*>)!!.softLockedStates) + } + SoftLockingAction.UNLOCK -> { + serviceHub.vaultService.softLockRelease(softLockAction.lockId!!, NonEmptySet.copyOf(softLockAction.states)) + assertEquals(softLockAction.expectedSoftLocks.states, queryCashStates(softLockAction.expectedSoftLocks.queryCriteria, serviceHub.vaultService)) + assertEquals(softLockAction.expectedSoftLockedStates, (stateMachine as? FlowStateMachineImpl<*>)!!.softLockedStates) + } + } + + softLockAction.exception?.let { + if (throwOnlyOnce) { + throwOnlyOnce = false + throw it + } + } + } + return true + } +} + +internal class CreateFungibleStateFLow : FlowLogic() { + + companion object { + var hook: ((VaultService) -> Unit)? = null + } + + @Suspendable + override fun call() { + val issuer = serviceHub.myInfo.legalIdentities.first() + val notary = serviceHub.networkMapCache.notaryIdentities[0] + val fungibleState = NodeVaultServiceTest.FungibleFoo(100.DOLLARS, listOf(issuer)) + val txCommand = Command(DummyContract.Commands.Create(), issuer.owningKey) + val txBuilder = TransactionBuilder(notary) + .addOutputState(fungibleState, DummyContract.PROGRAM_ID) + .addCommand(txCommand) + val signedTx = serviceHub.signInitialTransaction(txBuilder) + serviceHub.recordTransactions(signedTx) + hook?.invoke(serviceHub.vaultService) + } +} \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt b/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt index 7b7a051491..320137e8b4 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt @@ -406,6 +406,47 @@ class NodeVaultServiceTest { } } + @Test(timeout=300_000) + fun `softLockRelease - correctly releases n locked states`() { + fun queryStates(softLockingType: SoftLockingType) = + vaultService.queryBy(VaultQueryCriteria(softLockingCondition = SoftLockingCondition(softLockingType))).states + + database.transaction { + vaultFiller.fillWithSomeTestCash(100.DOLLARS, issuerServices, 100, DUMMY_CASH_ISSUER) + } + + val softLockId = UUID.randomUUID() + val lockCount = NodeVaultService.MAX_SQL_IN_CLAUSE_SET * 2 + database.transaction { + assertEquals(100, queryStates(SoftLockingType.UNLOCKED_ONLY).size) + val unconsumedStates = vaultService.queryBy().states + + val lockSet = mutableListOf() + for (i in 0 until lockCount) { + lockSet.add(unconsumedStates[i].ref) + } + vaultService.softLockReserve(softLockId, NonEmptySet.copyOf(lockSet)) + assertEquals(lockCount, queryStates(SoftLockingType.LOCKED_ONLY).size) + + val unlockSet0 = mutableSetOf() + for (i in 0 until NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 1) { + unlockSet0.add(lockSet[i]) + } + vaultService.softLockRelease(softLockId, NonEmptySet.copyOf(unlockSet0)) + assertEquals(NodeVaultService.MAX_SQL_IN_CLAUSE_SET - 1, queryStates(SoftLockingType.LOCKED_ONLY).size) + + val unlockSet1 = mutableSetOf() + for (i in NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 1 until NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 3) { + unlockSet1.add(lockSet[i]) + } + vaultService.softLockRelease(softLockId, NonEmptySet.copyOf(unlockSet1)) + assertEquals(NodeVaultService.MAX_SQL_IN_CLAUSE_SET - 1 - 2, queryStates(SoftLockingType.LOCKED_ONLY).size) + + vaultService.softLockRelease(softLockId) // release the rest + assertEquals(100, queryStates(SoftLockingType.UNLOCKED_ONLY).size) + } + } + @Test(timeout=300_000) fun `unconsumedStatesForSpending exact amount`() { database.transaction { From 14b9bc2c5319295b16e1a3b8ec621242f90f867c Mon Sep 17 00:00:00 2001 From: nikinagy <61757742+nikinagy@users.noreply.github.com> Date: Mon, 1 Jun 2020 11:55:17 +0100 Subject: [PATCH 49/49] throwing clearer error message when not supported 301 response code is used (#6296) --- .../main/kotlin/net/corda/core/internal/InternalUtils.kt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt index a97951cffa..cf847679e9 100644 --- a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt @@ -33,6 +33,7 @@ import java.lang.reflect.Member import java.lang.reflect.Modifier import java.math.BigDecimal import java.net.HttpURLConnection +import java.net.HttpURLConnection.HTTP_MOVED_PERM import java.net.HttpURLConnection.HTTP_OK import java.net.Proxy import java.net.URI @@ -478,7 +479,11 @@ fun URL.post(serializedData: OpaqueBytes, vararg properties: Pair