Merge pull request #6511 from corda/christians/ENT-5273-update-from-os-4.6

ENT-5273 update from os 4.6

.ci/api-current.txt

@@ -5398,6 +5398,10 @@ public interface net.corda.core.schemas.QueryableState extends net.corda.core.co
 ##
 public interface net.corda.core.schemas.StatePersistable
 ##
+public interface net.corda.core.serialization.CheckpointCustomSerializer
+  public abstract OBJ fromProxy(PROXY)
+  public abstract PROXY toProxy(OBJ)
+##
 public interface net.corda.core.serialization.ClassWhitelist
   public abstract boolean hasListed(Class<?>)
 ##

.ci/dev/compatibility/JenkinsfileJDK11Azul

@@ -1,14 +1,45 @@
+#!groovy
+/**
+ * Jenkins pipeline to build Corda OS release with JDK11
+ */
+
+/**
+ * Kill already started job.
+ * Assume new commit takes precendence and results from previous
+ * unfinished builds are not required.
+ * This feature doesn't play well with disableConcurrentBuilds() option
+ */
 @Library('corda-shared-build-pipeline-steps')
 import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
+/**
+ * Sense environment
+ */
+boolean isReleaseTag = (env.TAG_NAME =~ /^release.*JDK11$/)
+
+/*
+** calculate the stage for NexusIQ evaluation
+**  * build for snapshots
+**  * stage-release:  for release candidates and for health checks
+**  * operate: for final release
+*/
+def nexusIqStage = "build"
+if (isReleaseTag) {
+    switch (env.TAG_NAME) {
+        case ~/.*-RC\d+(-.*)?/: nexusIqStage = "stage-release"; break;
+        case ~/.*-HC\d+(-.*)?/: nexusIqStage = "stage-release"; break;
+        default: nexusIqStage = "release"
+    }
+}
+
 pipeline {
     agent { label 'k8s' }
     options {
         timestamps()
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     environment {
@@ -16,10 +47,34 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
-        stage('Corda Pull Request - Generate Build Image') {
+        stage('Sonatype Check') {
+            steps {
+                sh "./gradlew --no-daemon clean jar"
+                script {
+                    sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
+                    /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
+                    def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim()
+                    def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim()
+                    def artifactId = 'corda'
+                    nexusAppId = "jenkins-${groupId}-${artifactId}-jdk11-${version}"
+                }
+                nexusPolicyEvaluation (
+                        failBuildOnNetworkError: false,
+                        iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts!
+                        iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']],
+                        iqStage: nexusIqStage
+                )
+            }
+        }
+
+        stage('Generate Build Image') {
             steps {
                 withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
                     sh "./gradlew " +
@@ -28,8 +83,11 @@ pipeline {
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " +
                             "-Ddocker.buildbase.tag=11latest " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.dockerfile=DockerfileJDK11Azul" +
-                            " clean pushBuildImage --stacktrace"
+                            " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace"
                 }
                 sh "kubectl auth can-i get pods"
             }
@@ -65,12 +123,49 @@ pipeline {
                 }
             }
         }
+
+        stage('Publish to Artifactory') {
+            agent {
+                dockerfile {
+                    reuseNode true
+                    additionalBuildArgs "--build-arg USER=stresstester"
+                    filename '.ci/dev/compatibility/DockerfileJDK11'
+                }
+            }
+            when {
+                expression { isReleaseTag }
+            }
+            steps {
+                rtServer(
+                        id: 'R3-Artifactory',
+                        url: 'https://software.r3.com/artifactory',
+                        credentialsId: 'artifactory-credentials'
+                )
+                rtGradleDeployer(
+                        id: 'deployer',
+                        serverId: 'R3-Artifactory',
+                        repo: 'corda-releases'
+                )
+                rtGradleRun(
+                        usesPlugin: true,
+                        useWrapper: true,
+                        switches: '-s --info',
+                        tasks: 'artifactoryPublish',
+                        deployerId: 'deployer',
+                        buildName: env.ARTIFACTORY_BUILD_NAME
+                )
+                rtPublishBuildInfo(
+                        serverId: 'R3-Artifactory',
+                        buildName: env.ARTIFACTORY_BUILD_NAME
+                )
+            }
+        }
     }
 
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

.ci/dev/compatibility/JenkinsfileJDK11Compile

@@ -8,12 +8,13 @@ pipeline {
         dockerfile {
             label 'k8s'
             additionalBuildArgs "--build-arg USER=stresstester"
-            filename '.ci/dev/compatibility/DockerfileJDK11Compile'
+            filename '.ci/dev/compatibility/DockerfileJDK11'
         }
     }
     options {
         timestamps()
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     stages {
@@ -35,4 +36,4 @@ pipeline {
             deleteDir() /* clean up our workspace */
         }
     }
-}
+}

.ci/dev/integration/Jenkinsfile

@@ -1,62 +0,0 @@
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-@Library('corda-shared-build-pipeline-steps')
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-
-killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
-
-pipeline {
-    agent { label 'k8s' }
-    options {
-        timestamps()
-        timeout(time: 3, unit: 'HOURS')
-    }
-
-    environment {
-        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
-    }
-
-    stages {
-        stage('Corda - Generate Build Image') {
-            steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
-                    sh "./gradlew " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
-                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean pushBuildImage"
-                }
-                sh "kubectl auth can-i get pods"
-            }
-        }
-
-        stage('Corda - Run Tests') {
-            stage('Integration Tests') {
-                steps {
-                    sh "./gradlew " +
-                            "-DbuildId=\"\${BUILD_ID}\" " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " allParallelIntegrationTest"
-                    if (env.CHANGE_ID) {
-                        pullRequest.createStatus(status: 'success',
-                                context: 'continuous-integration/jenkins/pr-merge/integrationTest',
-                                description: 'Integration Tests Passed',
-                                targetUrl: "${env.JOB_URL}/testResults")
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            junit '**/build/test-results-xml/**/*.xml'
-        }
-        cleanup {
-            deleteDir() /* clean up our workspace */
-        }
-    }
-}

.ci/dev/mswin/Jenkinsfile

@@ -0,0 +1,101 @@
+#!groovy
+/**
+ * Jenkins pipeline to build Corda on MS Windows server.
+ * Because it takes a long time to run tests sequentially, unit tests and
+ * integration tests are started in parallel on separate agents.
+ *
+ * Additionally, pull requests by default run only unit tests.
+ */
+
+/**
+ * Kill already started job.
+ * Assume new commit takes precendence and results from previous
+ * unfinished builds are not required.
+ * This feature doesn't play well with disableConcurrentBuilds() option
+ */
+@Library('corda-shared-build-pipeline-steps')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+/**
+ * Sense environment
+ */
+boolean isReleaseBranch = (env.BRANCH_NAME =~ /^release\/os\/.*/)
+
+pipeline {
+    agent none
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
+
+        /*
+         * a bit awkward to read
+         * is parameter is true  -> push events are *not* ignored
+         * if parameter is false -> push events *are* ignored
+         */
+        overrideIndexTriggers (!isReleaseBranch)
+    }
+
+    parameters {
+        booleanParam defaultValue: (isReleaseBranch), description: 'Run integration tests?', name: 'DO_INTEGRATION_TESTS'
+    }
+
+    /*
+     * Do no receive Github's push events for release branches -> suitable for nightly builds
+     * but projects for pull requests will receive them as normal, and PR builds are started ASAP
+     */
+    triggers {
+        pollSCM ignorePostCommitHooks: isReleaseBranch, scmpoll_spec: '@midnight'
+    }
+
+    stages {
+        stage('Tests') {
+            parallel {
+                stage('Unit Tests') {
+                    agent { label 'mswin' }
+                    steps {
+                        bat "./gradlew --no-daemon " +
+                                "--stacktrace " +
+                                "-Pcompilation.warningsAsErrors=false " +
+                                "-Ptests.failFast=true " +
+                                "clean test"
+                    }
+                    post {
+                        always {
+                            archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log'
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            bat '.ci/kill_corda_procs.cmd'
+                        }
+                        cleanup {
+                            deleteDir() /* clean up our workspace */
+                        }
+                    }
+
+                }
+                stage('Integration Tests') {
+                    when {
+                        expression { params.DO_INTEGRATION_TESTS }
+                        beforeAgent true
+                    }
+                    agent { label 'mswin' }
+                    steps {
+                        bat "./gradlew --no-daemon " +
+                                "clean integrationTest"
+                    }
+                    post {
+                        always {
+                            archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log'
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            bat '.ci/kill_corda_procs.cmd'
+                        }
+                        cleanup {
+                            deleteDir() /* clean up our workspace */
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

.ci/dev/nightly-regression/Jenkinsfile

@@ -8,8 +8,8 @@ pipeline {
     options {
         timestamps()
         overrideIndexTriggers(false)
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
     triggers {
         pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight'
@@ -20,64 +20,74 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
-            stage('Corda Pull Request - Generate Build Image') {
-                steps {
-                    withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
-                        sh "./gradlew " +
-                                "-Dkubenetize=true " +
-                                "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
-                                "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
-                                "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                                " clean jar deployNodes install pushBuildImage --stacktrace"
-                    }
-                    sh "kubectl auth can-i get pods"
-                }
+        stage('Deploy Nodes') {
+            steps {
+                sh "./gradlew --no-daemon jar deployNodes"
             }
+        }
 
-            stage('Testing phase') {
-                parallel {
-                    stage('Regression Test') {
-                        steps {
-                            sh "./gradlew " +
-                                    "-DbuildId=\"\${BUILD_ID}\" " +
-                                    "-Dkubenetize=true " +
-                                    "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
-                                    "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
-                                    "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
-                                    "-Dgit.branch=\"\${GIT_BRANCH}\" " +
-                                    "-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
-                                    " parallelRegressionTest --stacktrace"
-                        }
+        stage('Generate Build Image') {
+            steps {
+                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                    sh "./gradlew " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
+                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " clean pushBuildImage --stacktrace"
+                }
+                sh "kubectl auth can-i get pods"
+            }
+        }
+
+        stage('Testing phase') {
+            parallel {
+                stage('Regression Test') {
+                    steps {
+                        sh "./gradlew " +
+                                "-DbuildId=\"\${BUILD_ID}\" " +
+                                "-Dkubenetize=true " +
+                                "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
+                                "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                                "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
+                                "-Dgit.branch=\"\${GIT_BRANCH}\" " +
+                                "-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
+                                " parallelRegressionTest --stacktrace"
                     }
-                    stage('Slow Integration Test') {
-                        steps {
-                            sh "./gradlew " +
-                                    "-DbuildId=\"\${BUILD_ID}\" " +
-                                    "-Dkubenetize=true " +
-                                    "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
-                                    "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
-                                    "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
-                                    "-Dgit.branch=\"\${GIT_BRANCH}\" " +
-                                    "-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
-                                    " allParallelSlowIntegrationTest --stacktrace"
-                        }
+                }
+                stage('Slow Integration Test') {
+                    steps {
+                        sh "./gradlew " +
+                                "-DbuildId=\"\${BUILD_ID}\" " +
+                                "-Dkubenetize=true " +
+                                "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
+                                "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                                "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
+                                "-Dgit.branch=\"\${GIT_BRANCH}\" " +
+                                "-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
+                                " allParallelSlowIntegrationTest --stacktrace"
                     }
                 }
             }
         }
-
+    }
 
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */
         }
     }
 }
-

.ci/dev/pr-code-checks/Jenkinsfile

@@ -4,10 +4,11 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
 pipeline {
-    agent { label 'k8s' }
+    agent { label 'standard' }
     options {
         timestamps()
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     environment {
@@ -40,6 +41,12 @@ pipeline {
                 sh ".ci/check-api-changes.sh"
             }
         }
+
+        stage('Deploy Nodes') {
+            steps {
+                sh "./gradlew --no-daemon jar deployNodes"
+            }
+        }
     }
 
     post {

.ci/dev/publish-api-docs/Jenkinsfile

@@ -0,0 +1,35 @@
+@Library('corda-shared-build-pipeline-steps')
+
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'standard' }
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 3, unit: 'HOURS')
+    }
+
+    environment {
+        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
+    }
+
+    stages {
+        stage('Publish Archived API Docs to Artifactory') {
+            when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' }
+            steps {
+                sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs"
+            }
+        }
+    }
+
+    post {
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

.ci/dev/publish-branch/Jenkinsfile.nightly

@@ -1,18 +1,34 @@
 #!groovy
+/**
+ * Jenkins pipeline to build Corda OS nightly snapshots
+ */
+
+/**
+ * Kill already started job.
+ * Assume new commit takes precendence and results from previous
+ * unfinished builds are not required.
+ * This feature doesn't play well with disableConcurrentBuilds() option
+ */
 @Library('corda-shared-build-pipeline-steps')
 import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
+/*
+** calculate the stage for NexusIQ evaluation
+**  * build for snapshots
+*/
+def nexusIqStage = "build"
+
 pipeline {
-    agent { label 'k8s' }
+    agent { label 'standard' }
 
     options {
         timestamps()
         ansiColor('xterm')
         overrideIndexTriggers(false)
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     triggers {
@@ -24,9 +40,30 @@ pipeline {
         // in the name
         ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Nightly to Artifactory"
                 .replaceAll("/", " :: ")
+        DOCKER_URL = "https://index.docker.io/v1/"
     }
 
     stages {
+        stage('Sonatype Check') {
+            steps {
+                sh "./gradlew --no-daemon clean jar"
+                script {
+                    sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
+                    /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
+                    def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim()
+                    def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim()
+                    def artifactId = 'corda'
+                    nexusAppId = "jenkins-${groupId}-${artifactId}-${version}"
+                }
+                nexusPolicyEvaluation (
+                        failBuildOnNetworkError: false,
+                        iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts!
+                        iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']],
+                        iqStage: nexusIqStage
+                )
+            }
+        }
+
         stage('Publish to Artifactory') {
             steps {
                 rtServer (
@@ -58,6 +95,17 @@ pipeline {
                 )
             }
         }
+
+        stage('Publish Nightly to Docker Hub') {
+            steps {
+                withCredentials([
+                        usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials',
+                                usernameVariable: 'DOCKER_USERNAME',
+                                passwordVariable: 'DOCKER_PASSWORD')]) {
+                    sh "./gradlew pushOfficialImages"
+                }
+            }
+        }
     }
 
 

.ci/dev/publish-branch/Jenkinsfile.preview

@@ -11,8 +11,8 @@ pipeline {
         timestamps()
         ansiColor('xterm')
         overrideIndexTriggers(false)
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     environment {

.ci/dev/regression/Jenkinsfile

@@ -1,29 +1,99 @@
+#!groovy
+/**
+ * Jenkins pipeline to build Corda OS release branches and tags
+ */
+
+/**
+ * Kill already started job.
+ * Assume new commit takes precendence and results from previous
+ * unfinished builds are not required.
+ * This feature doesn't play well with disableConcurrentBuilds() option
+ */
+@Library('corda-shared-build-pipeline-steps')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+/**
+ * Sense environment
+ */
+boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?<!_JDK11)$/)
+boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
+/*
+** calculate the stage for NexusIQ evaluation
+**  * build for snapshots
+**  * stage-release:  for release candidates and for health checks
+**  * operate: for final release
+*/
+def nexusIqStage = "build"
+if (isReleaseTag) {
+    switch (env.TAG_NAME) {
+        case ~/.*-RC\d+(-.*)?/: nexusIqStage = "stage-release"; break;
+        case ~/.*-HC\d+(-.*)?/: nexusIqStage = "stage-release"; break;
+        default: nexusIqStage = "release"
+    }
+}
+
 pipeline {
     agent { label 'k8s' }
     options {
         timestamps()
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
         disableConcurrentBuilds()
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     environment {
         DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}"
+        DOCKER_URL = "https://index.docker.io/v1/"
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
-        stage('Corda Pull Request - Generate Build Image') {
+        stage('Sonatype Check') {
+            steps {
+                sh "./gradlew --no-daemon clean jar"
+                script {
+                    sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
+                    /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
+                    def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim()
+                    def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim()
+                    def artifactId = 'corda'
+                    nexusAppId = "jenkins-${groupId}-${artifactId}-${version}"
+                }
+                nexusPolicyEvaluation (
+                        failBuildOnNetworkError: false,
+                        iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts!
+                        iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']],
+                        iqStage: nexusIqStage
+                )
+            }
+        }
+
+        stage('Deploy Nodes') {
+            steps {
+                sh "./gradlew --no-daemon jar deployNodes"
+            }
+        }
+
+        stage('Generate Build Image') {
             steps {
                 withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
                     sh "./gradlew " +
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean jar deployNodes install pushBuildImage --stacktrace"
+                            " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace"
                 }
                 sh "kubectl auth can-i get pods"
             }
@@ -59,13 +129,56 @@ pipeline {
                 }
             }
         }
-    }
 
+        stage('Publish to Artifactory') {
+            when {
+                expression { isReleaseTag }
+            }
+            steps {
+                rtServer(
+                        id: 'R3-Artifactory',
+                        url: 'https://software.r3.com/artifactory',
+                        credentialsId: 'artifactory-credentials'
+                )
+                rtGradleDeployer(
+                        id: 'deployer',
+                        serverId: 'R3-Artifactory',
+                        repo: 'corda-releases'
+                )
+                rtGradleRun(
+                        usesPlugin: true,
+                        useWrapper: true,
+                        switches: '-s --info',
+                        tasks: 'artifactoryPublish',
+                        deployerId: 'deployer',
+                        buildName: env.ARTIFACTORY_BUILD_NAME
+                )
+                rtPublishBuildInfo(
+                        serverId: 'R3-Artifactory',
+                        buildName: env.ARTIFACTORY_BUILD_NAME
+                )
+            }
+        }
+
+        stage('Publish Release to Docker Hub') {
+            when {
+                expression { !isInternalRelease && isReleaseTag }
+            }
+            steps {
+                withCredentials([
+                        usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials',
+                                usernameVariable: 'DOCKER_USERNAME',
+                                passwordVariable: 'DOCKER_PASSWORD')]) {
+                    sh "./gradlew pushOfficialImages"
+                }
+            }
+        }
+    }
 
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
 
             script {
                 try {
@@ -97,38 +210,40 @@ pipeline {
                     }
                 }
             }
-            
+
             script
             {
-                // We want to send a summary email, but want to limit to once per day.
-                // Comparing the dates of the previous and current builds achieves this,
-                // i.e. we will only send an email for the first build on a given day.
-                def prevBuildDate = new Date(
-                        currentBuild?.previousBuild.timeInMillis ?: 0).clearTime()
-                def currentBuildDate = new Date(
-                        currentBuild.timeInMillis).clearTime()
+                if (!isReleaseTag) {
+                    // We want to send a summary email, but want to limit to once per day.
+                    // Comparing the dates of the previous and current builds achieves this,
+                    // i.e. we will only send an email for the first build on a given day.
+                    def prevBuildDate = new Date(
+                            currentBuild?.previousBuild.timeInMillis ?: 0).clearTime()
+                    def currentBuildDate = new Date(
+                            currentBuild.timeInMillis).clearTime()
 
-                if (prevBuildDate != currentBuildDate) {
-                    def statusSymbol = '\u2753'
-                    switch(currentBuild.result) {
-                        case 'SUCCESS':
-                            statusSymbol = '\u2705'
-                            break;
-                        case 'UNSTABLE':
-                        case 'FAILURE':
-                            statusSymbol = '\u274c'
-                            break;
-                        default:
-                            break;
+                    if (prevBuildDate != currentBuildDate) {
+                        def statusSymbol = '\u2753'
+                        switch(currentBuild.result) {
+                            case 'SUCCESS':
+                                statusSymbol = '\u2705'
+                                break;
+                            case 'UNSTABLE':
+                            case 'FAILURE':
+                                statusSymbol = '\u274c'
+                                break;
+                            default:
+                                break;
+                        }
+
+                        echo('First build for this date, sending summary email')
+                        emailext to: '$DEFAULT_RECIPIENTS',
+                            subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS',
+                            mimeType: 'text/html',
+                            body: '${SCRIPT, template="groovy-html.template"}'
+                    } else {
+                        echo('Already sent summary email today, suppressing')
                     }
-
-                    echo('First build for this date, sending summary email')
-                    emailext to: '$DEFAULT_RECIPIENTS',
-                        subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS',
-                        mimeType: 'text/html',
-                        body: '${SCRIPT, template="groovy-html.template"}'
-                } else {
-                    echo('Already sent summary email today, suppressing')
                 }
             }
         }

.ci/dev/unit/Jenkinsfile

@@ -1,60 +0,0 @@
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-@Library('corda-shared-build-pipeline-steps')
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-
-killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
-
-pipeline {
-    agent { label 'k8s' }
-    options {
-        timestamps()
-        timeout(time: 3, unit: 'HOURS')
-    }
-
-    environment {
-        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
-    }
-
-    stages {
-        stage('Corda Pull Request - Generate Build Image') {
-            steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
-                    sh "./gradlew " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
-                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean pushBuildImage"
-                }
-                sh "kubectl auth can-i get pods"
-            }
-        }
-
-        stage('Unit Tests') {
-            steps {
-                sh "./gradlew " +
-                        "-DbuildId=\"\${BUILD_ID}\" " +
-                        "-Dkubenetize=true " +
-                        "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                        " allParallelUnitTest"
-                if (env.CHANGE_ID) {
-                    pullRequest.createStatus(status: 'success',
-                            context: 'continuous-integration/jenkins/pr-merge/unitTest',
-                            description: 'Unit Tests Passed',
-                            targetUrl: "${env.JOB_URL}/testResults")
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            junit '**/build/test-results-xml/**/*.xml'
-        }
-        cleanup {
-            deleteDir() /* clean up our workspace */
-        }
-    }
-}

.github/CODEOWNERS

@@ -0,0 +1,62 @@
+# All documentation should be reviewed by the technical writers
+*.md	@corda/technical-writers
+
+# By default anything under core or node-api is the Kernel team
+core    @corda/kernel
+node-api        @corda/kernel
+node/src/main/kotlin/net/corda/node/internal    @corda/kernel
+node/src/main/kotlin/net/corda/node/services    @corda/kernel
+
+# Determinstic components
+core-deterministic      @chrisr3
+jdk8u-deterministic     @chrisr3
+node/djvm       @chrisr3
+serialization-deterministic     @chrisr3
+serialization-djvm      @chrisr3
+serialization-tests     @chrisr3
+
+# Demobench defaults to Chris, but Viktor for the main code
+tools/demobench @chrisr3
+tools/demobench/src/main/kotlin/net/corda/demobench @vkolomeyko
+
+# General Corda code
+
+client/rpc	@vkolomeyko
+
+core/src/main/kotlin/net/corda/core/flows	@dimosr
+core/src/main/kotlin/net/corda/core/internal/notary	@thschroeter
+core/src/main/kotlin/net/corda/core/messaging	@vkolomeyko
+
+node/src/integration-test/kotlin/net/corda/node/persistence	@blsemo
+node/src/integration-test/kotlin/net/corda/node/services/persistence	@blsemo
+node/src/main/kotlin/net/corda/node/internal/artemis @rekalov
+node/src/main/kotlin/net/corda/node/services/identity	@rekalov
+node/src/main/kotlin/net/corda/node/services/keys	@rekalov
+node/src/main/kotlin/net/corda/node/services/messaging	@dimosr
+node/src/main/kotlin/net/corda/node/services/network	@rekalov
+node/src/main/kotlin/net/corda/node/services/persistence	@blsemo
+node/src/main/kotlin/net/corda/node/services/rpc	@vkolomeyko
+node/src/main/kotlin/net/corda/node/services/statemachine	@lankydan
+node/src/main/kotlin/net/corda/node/utilities/registration	@rekalov
+node/src/main/kotlin/net/corda/notary	@thschroeter
+
+node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging	@vkolomeyko
+node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto	@rekalov
+node-api/src/main/kotlin/net/corda/nodeapi/internal/cryptoservice @rekalov
+node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle	@vkolomeyko
+node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence	@blsemo
+node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper	@vkolomeyko
+node-api/src/test/kotlin/net/corda/nodeapi/internal/bridging	@rekalov
+
+common/logging/src/main/kotlin/net/corda/common/logging/errorReporting @JamesHR3
+common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting @JamesHR3
+
+# Single file ownerships go at the end, as they are most specific and take precedence over other ownerships
+
+core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt      @adelel1
+core/src/main/kotlin/net/corda/core/internal/AttachmentTrustCalculator.kt       @adelel1
+core/src/main/kotlin/net/corda/core/internal/AttachmentWithContext.kt   @adelel1
+core/src/main/kotlin/net/corda/core/internal/CertRole.kt        @rekalov
+core/src/main/kotlin/net/corda/core/node/services/AttachmentStorage.kt  @adelel1
+core/src/main/kotlin/net/corda/core/node/services/IdentityService.kt    @rekalov
+core/src/main/kotlin/net/corda/core/node/services/NetworkMapCache.kt    @rekalov

Jenkinsfile

@@ -9,6 +9,7 @@ pipeline {
     options {
         timestamps()
         timeout(time: 3, unit: 'HOURS')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
     environment {
@@ -16,6 +17,9 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -26,8 +30,11 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean jar deployNodes pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelIntegrationTest --stacktrace"
+                            " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace"
                 }
                 sh "kubectl auth can-i get pods"
             }
@@ -72,7 +79,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

build.gradle

@@ -105,7 +105,7 @@ buildscript {
     ext.eddsa_version = '0.3.0'
     ext.dependency_checker_version = '5.2.0'
     ext.commons_collections_version = '4.3'
-    ext.beanutils_version = '1.9.3'
+    ext.beanutils_version = '1.9.4'
     ext.crash_version = '1.7.4'
     ext.jsr305_version = constants.getProperty("jsr305Version")
     ext.shiro_version = '1.4.1'
@@ -155,22 +155,39 @@ buildscript {
     ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion"
     repositories {
         mavenLocal()
-        mavenCentral()
-        jcenter()
-        maven {
-            url 'https://kotlin.bintray.com/kotlinx'
-        }
-        maven {
-            url "$artifactory_contextUrl/corda-dependencies-dev"
-        }
-        maven {
-            url "$artifactory_contextUrl/corda-releases"
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven {
+                url 'https://kotlin.bintray.com/kotlinx'
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-dependencies-dev"
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-releases"
+            }
         }
     }
     dependencies {
         classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
         classpath "org.jetbrains.kotlin:kotlin-allopen:$kotlin_version"
-        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.4'
         classpath "net.corda.plugins:publish-utils:$gradle_plugins_version"
         classpath "net.corda.plugins:quasar-utils:$gradle_plugins_version"
         classpath "net.corda.plugins:cordformation:$gradle_plugins_version"
@@ -204,7 +221,6 @@ plugins {
 apply plugin: 'project-report'
 apply plugin: 'com.github.ben-manes.versions'
 apply plugin: 'net.corda.plugins.publish-utils'
-apply plugin: 'maven-publish'
 apply plugin: 'com.jfrog.artifactory'
 apply plugin: "com.bmuschko.docker-remote-api"
 apply plugin: "com.r3.dependx.dependxies"
@@ -275,7 +291,7 @@ allprojects {
         toolVersion = "0.8.3"
     }
 
-    tasks.withType(JavaCompile) {
+    tasks.withType(JavaCompile).configureEach {
         options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" << "-Xlint:-options" << "-parameters"
         options.compilerArgs << '-XDenableSunApiLintControl'
         if (warnings_as_errors) {
@@ -287,7 +303,7 @@ allprojects {
         options.encoding = 'UTF-8'
     }
 
-    tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) {
+    tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
         kotlinOptions {
             languageVersion = "1.2"
             apiVersion = "1.2"
@@ -302,7 +318,7 @@ allprojects {
         task.dependsOn tasks.withType(AbstractCompile)
     }
 
-    tasks.withType(Jar) { task ->
+    tasks.withType(Jar).configureEach { task ->
         // Includes War and Ear
         manifest {
             attributes('Corda-Release-Version': corda_release_version)
@@ -314,7 +330,7 @@ allprojects {
         }
     }
 
-    tasks.withType(Test) {
+    tasks.withType(Test).configureEach {
         forkEvery = 10
         ignoreFailures = project.hasProperty('tests.ignoreFailures') ? project.property('tests.ignoreFailures').toBoolean() : false
         failFast = project.hasProperty('tests.failFast') ? project.property('tests.failFast').toBoolean() : false
@@ -339,7 +355,7 @@ allprojects {
         systemProperty 'java.security.egd', 'file:/dev/./urandom'
     }
 
-    tasks.withType(Test) {
+    tasks.withType(Test).configureEach {
         if (name.contains("integrationTest")) {
             maxParallelForks = (System.env.CORDA_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_INT_TESTING_FORKS".toInteger()
         }
@@ -357,11 +373,29 @@ allprojects {
 
     repositories {
         mavenLocal()
-        mavenCentral()
-        jcenter()
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
-        maven { url 'https://repo.gradle.org/gradle/libs-releases' }
-        maven { url "$artifactory_contextUrl/corda-dev" }
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+            maven { url 'https://repo.gradle.org/gradle/libs-releases' }
+            maven { url "${artifactory_contextUrl}/corda-dev" }
+        }
     }
 
     configurations {
@@ -520,7 +554,7 @@ tasks.register('detektBaseline', JavaExec) {
     args(params)
 }
 
-tasks.withType(Test) {
+tasks.withType(Test).configureEach {
     reports.html.destination = file("${reporting.baseDir}/${name}")
 }
 
@@ -626,7 +660,7 @@ dependxiesModule {
     skipTasks = "test,integrationTest,smokeTest,slowIntegrationTest"
 }
 
-task generateApi(type: net.corda.plugins.GenerateApi) {
+tasks.register('generateApi', net.corda.plugins.apiscanner.GenerateApi) {
     baseName = "api-corda"
 }
 
@@ -662,7 +696,7 @@ if (file('corda-docs-only-build').exists() || (System.getenv('CORDA_DOCS_ONLY_BU
 }
 
 wrapper {
-    gradleVersion = "5.4.1"
+    gradleVersion = '5.6.4'
     distributionType = Wrapper.DistributionType.ALL
 }
 

client/rpc/src/integration-test/java/net/corda/client/rpc/CordaRPCJavaClientTest.java

@@ -13,6 +13,7 @@ import net.corda.node.internal.NodeWithInfo;
 import net.corda.testing.internal.InternalTestUtilsKt;
 import net.corda.testing.node.User;
 import net.corda.testing.node.internal.NodeBasedTest;
+import net.corda.testing.node.internal.TestCordappInternal;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -30,10 +31,18 @@ import static net.corda.node.services.Permissions.invokeRpc;
 import static net.corda.node.services.Permissions.startFlow;
 import static net.corda.testing.core.TestConstants.ALICE_NAME;
 import static net.corda.testing.core.TestConstants.DUMMY_NOTARY_NAME;
+import static net.corda.testing.node.internal.InternalTestUtilsKt.FINANCE_CORDAPPS;
+import static net.corda.testing.node.internal.InternalTestUtilsKt.cordappWithPackages;
 
 public class CordaRPCJavaClientTest extends NodeBasedTest {
     public CordaRPCJavaClientTest() {
-        super(Arrays.asList("net.corda.finance.contracts", CashSchemaV1.class.getPackage().getName()), Collections.singletonList(DUMMY_NOTARY_NAME));
+        super(cordapps(), Collections.singletonList(DUMMY_NOTARY_NAME));
+    }
+
+    private static Set<TestCordappInternal> cordapps() {
+        Set<TestCordappInternal> cordapps = new HashSet<>(FINANCE_CORDAPPS);
+        cordapps.add(cordappWithPackages(CashSchemaV1.class.getPackage().getName()));
+        return cordapps;
     }
 
     private List<String> perms = Arrays.asList(

client/rpc/src/integration-test/kotlin/net/corda/client/rpc/CordaRPCClientTest.kt

@@ -40,6 +40,7 @@ import net.corda.testing.core.expect
 import net.corda.testing.core.expectEvents
 import net.corda.testing.core.sequence
 import net.corda.testing.node.User
+import net.corda.testing.node.internal.FINANCE_CORDAPPS
 import net.corda.testing.node.internal.NodeBasedTest
 import net.corda.testing.node.internal.ProcessUtilities
 import net.corda.testing.node.internal.poll
@@ -62,7 +63,7 @@ import kotlin.test.assertEquals
 import kotlin.test.assertFalse
 import kotlin.test.assertTrue
 
-class CordaRPCClientTest : NodeBasedTest(listOf("net.corda.finance"), notaries = listOf(DUMMY_NOTARY_NAME)) {
+class CordaRPCClientTest : NodeBasedTest(FINANCE_CORDAPPS, notaries = listOf(DUMMY_NOTARY_NAME)) {
     companion object {
         val rpcUser = User("user1", "test", permissions = setOf(all()))
         val log = contextLogger()

client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt

@@ -2,9 +2,7 @@ package net.corda.client.rpc
 
 import net.corda.core.context.Actor
 import net.corda.core.context.Trace
-import net.corda.core.internal.packageName
 import net.corda.core.messaging.CordaRPCOps
-import net.corda.finance.schemas.CashSchemaV1
 import net.corda.node.internal.NodeWithInfo
 import net.corda.node.services.Permissions
 import net.corda.testing.core.ALICE_NAME
@@ -14,7 +12,7 @@ import org.assertj.core.api.Assertions.assertThat
 import org.junit.Before
 import org.junit.Test
 
-class FlowsExecutionModeTests : NodeBasedTest(emptyList()) {
+class FlowsExecutionModeTests : NodeBasedTest() {
 
     private val rpcUser = User("user1", "test", permissions = setOf(Permissions.all()))
     private lateinit var node: NodeWithInfo

client/rpc/src/integration-test/kotlin/net/corda/client/rpcreconnect/CordaRPCClientReconnectionTest.kt

@@ -15,6 +15,7 @@ import net.corda.finance.DOLLARS
 import net.corda.finance.contracts.asset.Cash
 import net.corda.finance.flows.CashIssueFlow
 import net.corda.node.services.Permissions
+import net.corda.nodeapi.exceptions.RejectedCommandException
 import net.corda.testing.core.CHARLIE_NAME
 import net.corda.testing.driver.DriverParameters
 import net.corda.testing.driver.NodeHandle
@@ -49,6 +50,38 @@ class CordaRPCClientReconnectionTest {
         val rpcUser = User("user1", "test", permissions = setOf(Permissions.all()))
     }
 
+
+
+
+
+    @Test(timeout=300_000)
+    fun `rpc node start when FlowsDrainingModeEnabled throws RejectedCommandException and won't attempt to reconnect`() {
+        driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS)) {
+            val address = NetworkHostAndPort("localhost", portAllocator.nextPort())
+
+            fun startNode(): NodeHandle {
+                return startNode(
+                        providedName = CHARLIE_NAME,
+                        rpcUsers = listOf(CordaRPCClientTest.rpcUser),
+                        customOverrides = mapOf("rpcSettings.address" to address.toString())
+                ).getOrThrow()
+            }
+
+            val node = startNode()
+            val client = CordaRPCClient(node.rpcAddress,
+                    config.copy(maxReconnectAttempts = 1))
+
+            (client.start(rpcUser.username, rpcUser.password, gracefulReconnect = gracefulReconnect)).use {
+                val rpcOps = it.proxy as ReconnectingCordaRPCOps
+                rpcOps.setFlowsDrainingModeEnabled(true)
+
+                assertThatThrownBy { rpcOps.startTrackedFlow(::CashIssueFlow, 10.DOLLARS, OpaqueBytes.of(0), defaultNotaryIdentity).returnValue.get() }
+                        .isInstanceOf(RejectedCommandException::class.java).hasMessage("Node is draining before shutdown. Cannot start new flows through RPC.")
+            }
+        }
+    }
+
+
     @Test(timeout=300_000)
     fun `rpc client calls and returned observables continue working when the server crashes and restarts`() {
         driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS)) {

client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt

@@ -158,7 +158,8 @@ open class CordaRPCClientConfiguration @JvmOverloads constructor(
         open val connectionRetryIntervalMultiplier: Double = 1.5,
 
         /**
-         * Maximum reconnect attempts on failover or disconnection. The default is -1 which means unlimited.
+         * Maximum reconnect attempts on failover or disconnection.
+         * Any negative value would mean that there will be an infinite number of reconnect attempts.
          */
         open val maxReconnectAttempts: Int = unlimitedReconnectAttempts,
 

client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt

@@ -76,10 +76,10 @@ import kotlin.reflect.jvm.javaMethod
  * forwarded to the [UnicastSubject]. Note that the observations themselves may contain further [Observable]s, which are
  * handled in the same way.
  *
- * To do the above we take advantage of Kryo's datastructure traversal. When the client is deserialising a message from
- * the server that may contain Observables it is supplied with an [ObservableContext] that exposes the map used to demux
- * the observations. When an [Observable] is encountered during traversal a new [UnicastSubject] is added to the map and
- * we carry on. Each observation later contains the corresponding Observable ID, and we just forward that to the
+ * To do the above we take advantage of serialisation data structure traversal. When the client is deserialising a message from
+ * the server that may contain [Observable]s, it is supplied with an [ObservableContext] that exposes the map used to demux
+ * the observations. When a new [Observable] is encountered during traversal a new [UnicastSubject] is added to the map and
+ * we carry on. Each observation later contains the corresponding [Observable] ID, and we just forward that to the
  * associated [UnicastSubject].
  *
  * The client may signal that it no longer consumes a particular [Observable]. This may be done explicitly by
@@ -88,12 +88,12 @@ import kotlin.reflect.jvm.javaMethod
  * The cleanup happens in batches using a dedicated reaper, scheduled on [reaperExecutor].
  *
  * The client will attempt to failover in case the server become unreachable. Depending on the [ServerLocator] instance
- * passed in the constructor, failover is either handle at Artemis level or client level. If only one transport
+ * passed in the constructor, failover is either handled at Artemis level or client level. If only one transport
  * was used to create the [ServerLocator], failover is handled by Artemis (retrying based on [CordaRPCClientConfiguration].
  * If a list of transport configurations was used, failover is handled locally. Artemis is able to do it, however the
  * brokers on server side need to be configured in HA mode and the [ServerLocator] needs to be created with HA as well.
  */
-class RPCClientProxyHandler(
+internal class RPCClientProxyHandler(
         private val rpcConfiguration: CordaRPCClientConfiguration,
         private val rpcUsername: String,
         private val rpcPassword: String,
@@ -247,7 +247,7 @@ class RPCClientProxyHandler(
         try {
             sessionFactory = serverLocator.createSessionFactory()
         } catch (e: ActiveMQNotConnectedException) {
-            throw (RPCException("Cannot connect to server(s). Tried with all available servers.", e))
+            throw RPCException("Cannot connect to server(s). Tried with all available servers.", e)
         }
         // Depending on how the client is constructed, connection failure is treated differently
         if (serverLocator.staticTransportConfigurations.size == 1) {
@@ -380,9 +380,11 @@ class RPCClientProxyHandler(
                 is RPCApi.ServerToClient.Observation -> {
                     val observable: UnicastSubject<Notification<*>>? = observableContext.observableMap.getIfPresent(serverToClient.id)
                     if (observable == null) {
-                        log.debug("Observation ${serverToClient.content} arrived to unknown Observable with ID ${serverToClient.id}. " +
-                                "This may be due to an observation arriving before the server was " +
-                                "notified of observable shutdown")
+                        log.debug {
+                            "Observation ${serverToClient.content} arrived to unknown Observable with ID ${serverToClient.id}. " +
+                                    "This may be due to an observation arriving before the server was " +
+                                    "notified of observable shutdown"
+                        }
                     } else {
                         // We schedule the onNext() on an executor sticky-pooled based on the Observable ID.
                         observationExecutorPool.run(serverToClient.id) { executor ->
@@ -461,7 +463,7 @@ class RPCClientProxyHandler(
             }
         }
         observableContext.observableMap.invalidateAll()
-        rpcReplyMap.forEach { _, replyFuture ->
+        rpcReplyMap.forEach { (_, replyFuture) ->
             replyFuture.setException(ConnectionFailureException())
         }
 
@@ -528,23 +530,26 @@ class RPCClientProxyHandler(
     }
 
     private fun attemptReconnect() {
-        var reconnectAttempts = rpcConfiguration.maxReconnectAttempts.times(serverLocator.staticTransportConfigurations.size)
+        // This can be a negative number as `rpcConfiguration.maxReconnectAttempts = -1` means infinite number of re-connects
+        val maxReconnectCount = rpcConfiguration.maxReconnectAttempts.times(serverLocator.staticTransportConfigurations.size)
+        log.debug { "maxReconnectCount = $maxReconnectCount" }
+        var reconnectAttempt = 1
         var retryInterval = rpcConfiguration.connectionRetryInterval
         val maxRetryInterval = rpcConfiguration.connectionMaxRetryInterval
 
-        var transportIterator = serverLocator.staticTransportConfigurations.iterator()
-        while (transportIterator.hasNext() && reconnectAttempts != 0) {
-            val transport = transportIterator.next()
-            if (!transportIterator.hasNext())
-                transportIterator = serverLocator.staticTransportConfigurations.iterator()
+        fun shouldRetry(reconnectAttempt: Int) =
+                if (maxReconnectCount < 0) true else reconnectAttempt <= maxReconnectCount
 
-            log.debug("Trying to connect using ${transport.params}")
+        while (shouldRetry(reconnectAttempt)) {
+            val transport = serverLocator.staticTransportConfigurations.let { it[(reconnectAttempt - 1) % it.size] }
+
+            log.debug { "Trying to connect using ${transport.params}" }
             try {
                 if (!serverLocator.isClosed) {
                     sessionFactory = serverLocator.createSessionFactory(transport)
                 } else {
                     log.warn("Stopping reconnect attempts.")
-                    log.debug("Server locator is closed or garbage collected. Proxy may have been closed during reconnect.")
+                    log.debug { "Server locator is closed or garbage collected. Proxy may have been closed during reconnect." }
                     break
                 }
             } catch (e: ActiveMQException) {
@@ -552,12 +557,12 @@ class RPCClientProxyHandler(
                     Thread.sleep(retryInterval.toMillis())
                 } catch (e: InterruptedException) {}
                 // Could not connect, try with next server transport.
-                reconnectAttempts--
+                reconnectAttempt++
                 retryInterval = minOf(maxRetryInterval, retryInterval.times(rpcConfiguration.connectionRetryIntervalMultiplier.toLong()))
                 continue
             }
 
-            log.debug("Connected successfully after $reconnectAttempts attempts using ${transport.params}.")
+            log.debug { "Connected successfully after $reconnectAttempt attempts using ${transport.params}." }
             log.info("RPC server available.")
             sessionFactory!!.addFailoverListener(this::haFailoverHandler)
             initSessions()
@@ -566,8 +571,12 @@ class RPCClientProxyHandler(
             break
         }
 
-        if (reconnectAttempts == 0 || sessionFactory == null)
-            log.error("Could not reconnect to the RPC server.")
+        val maxReconnectReached = !shouldRetry(reconnectAttempt)
+        if (maxReconnectReached || sessionFactory == null) {
+            val errMessage = "Could not reconnect to the RPC server after trying $reconnectAttempt times." +
+                    if (sessionFactory != null) "" else " It was never possible to to establish connection with any of the endpoints."
+            log.error(errMessage)
+        }
     }
 
     private fun initSessions() {
@@ -620,10 +629,11 @@ class RPCClientProxyHandler(
         sendingEnabled.set(false)
         log.warn("Terminating observables.")
         val m = observableContext.observableMap.asMap()
+        val connectionFailureException = ConnectionFailureException()
         m.keys.forEach { k ->
             observationExecutorPool.run(k) {
                 try {
-                    m[k]?.onError(ConnectionFailureException())
+                    m[k]?.onError(connectionFailureException)
                 } catch (e: Exception) {
                     log.error("Unexpected exception when RPC connection failure handling", e)
                 }
@@ -631,8 +641,8 @@ class RPCClientProxyHandler(
         }
         observableContext.observableMap.invalidateAll()
 
-        rpcReplyMap.forEach { _, replyFuture ->
-            replyFuture.setException(ConnectionFailureException())
+        rpcReplyMap.forEach { (_, replyFuture) ->
+            replyFuture.setException(connectionFailureException)
         }
 
         rpcReplyMap.clear()
@@ -666,5 +676,5 @@ class RPCClientProxyHandler(
     }
 }
 
-private typealias RpcReplyMap = ConcurrentHashMap<Trace.InvocationId, SettableFuture<Any?>>
+private typealias RpcReplyMap = ConcurrentHashMap<InvocationId, SettableFuture<Any?>>
 

client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt

@@ -325,8 +325,8 @@ class ReconnectingCordaRPCOps private constructor(
                     }
                     when (e.targetException) {
                         is RejectedCommandException -> {
-                            log.warn("Node is being shutdown. Operation ${method.name} rejected. Retrying when node is up...", e)
-                            reconnectingRPCConnection.reconnectOnError(e)
+                            log.warn("Node is being shutdown. Operation ${method.name} rejected. Shutting down...", e)
+                            throw e.targetException
                         }
                         is ConnectionFailureException -> {
                             log.warn("Failed to perform operation ${method.name}. Connection dropped. Retrying....", e)

common/logging/src/main/kotlin/net/corda/common/logging/ExceptionsErrorCodeFunctions.kt

@@ -32,8 +32,9 @@ fun Message.withErrorCodeFor(error: Throwable?, level: Level): Message {
 
     return when {
         error != null && level.isInRange(Level.FATAL, Level.WARN) -> {
+            val logMessage = this.formattedMessage
             val message = error.walkExceptionCausedByList().asSequence().mapNotNull(Throwable::message).joinToString(" - ")
-            CompositeMessage("$message [errorCode=${error.errorCode()}, moreInformationAt=${error.errorCodeLocationUrl()}]", format, parameters, throwable)
+            CompositeMessage("$logMessage - $message [errorCode=${error.errorCode()}, moreInformationAt=${error.errorCodeLocationUrl()}]", format, parameters, throwable)
         }
         else -> this
     }

common/logging/src/main/kotlin/net/corda/common/logging/errorReporting/ErrorReporterImpl.kt

@@ -1,6 +1,7 @@
 package net.corda.common.logging.errorReporting
 
 import org.slf4j.Logger
+import java.lang.Exception
 import java.text.MessageFormat
 import java.util.*
 
@@ -31,6 +32,10 @@ internal class ErrorReporterImpl(private val resourceLocation: String,
     override fun report(error: ErrorCode<*>, logger: Logger) {
         val errorResource = ErrorResource.fromErrorCode(error, resourceLocation, locale)
         val message = "${errorResource.getErrorMessage(error.parameters.toTypedArray())} ${getErrorInfo(error)}"
-        logger.error(message)
+        if (error is Exception) {
+            logger.error(message, error)
+        } else {
+            logger.error(message)
+        }
     }
 }

common/logging/src/main/resources/error-codes/database-failed-startup.properties

@@ -1,4 +1,4 @@
-errorTemplate = Failed to create the datasource. See the logs for further information and the cause.
+errorTemplate = Failed to create the datasource: {0}. See the logs for further information and the cause.
 shortDescription = The datasource could not be created for unknown reasons.
 actionsToFix = The logs in the logs directory should contain more information on what went wrong.
 aliases =

common/logging/src/main/resources/error-codes/database-failed-startup_en_US.properties

@@ -1,3 +1,3 @@
-errorTemplate = Failed to create the datasource. See the logs for further information and the cause.
+errorTemplate = Failed to create the datasource: {0}. See the logs for further information and the cause.
 shortDescription = The datasource could not be created for unknown reasons.
 actionsToFix = The logs in the logs directory should contain more information on what went wrong.

common/logging/src/test/kotlin/net/corda/commmon/logging/ExceptionsErrorCodeFunctionsTest.kt

@@ -0,0 +1,34 @@
+package net.corda.commmon.logging
+
+import com.natpryce.hamkrest.assertion.assertThat
+import com.natpryce.hamkrest.contains
+import net.corda.common.logging.withErrorCodeFor
+import org.apache.logging.log4j.Level
+import org.apache.logging.log4j.message.SimpleMessage
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class ExceptionsErrorCodeFunctionsTest {
+
+    @Test(timeout=3_000)
+    fun `error code for message prints out message and full stack trace`() {
+        val originalMessage = SimpleMessage("This is a test message")
+        var previous: Exception? = null
+        val throwables = (0..10).map {
+            val current = TestThrowable(it, previous)
+            previous = current
+            current
+        }
+        val exception = throwables.last()
+        val message = originalMessage.withErrorCodeFor(exception, Level.ERROR)
+        assertThat(message.formattedMessage, contains("This is a test message".toRegex()))
+        for (i in (0..10)) {
+            assertThat(message.formattedMessage, contains("This is exception $i".toRegex()))
+        }
+        assertEquals(message.format, originalMessage.format)
+        assertEquals(message.parameters, originalMessage.parameters)
+        assertEquals(message.throwable, originalMessage.throwable)
+    }
+
+    private class TestThrowable(index: Int, cause: Exception?) : Exception("This is exception $index", cause)
+}

common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/DatabaseErrorsTest.kt

@@ -6,7 +6,7 @@ import java.net.InetAddress
 class DatabaseErrorsTest : ErrorCodeTest<NodeDatabaseErrors>(NodeDatabaseErrors::class.java) {
     override val dataForCodes = mapOf(
             NodeDatabaseErrors.COULD_NOT_CONNECT to listOf<Any>(),
-            NodeDatabaseErrors.FAILED_STARTUP to listOf(),
+            NodeDatabaseErrors.FAILED_STARTUP to listOf("This is a test message"),
             NodeDatabaseErrors.MISSING_DRIVER to listOf(),
             NodeDatabaseErrors.PASSWORD_REQUIRED_FOR_H2 to listOf(InetAddress.getLocalHost())
     )

common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/ErrorReporterImplTest.kt

@@ -7,6 +7,7 @@ import net.corda.common.logging.errorReporting.ErrorContextProvider
 import net.corda.common.logging.errorReporting.ErrorReporterImpl
 import org.junit.After
 import org.junit.Test
+import org.mockito.ArgumentMatchers.any
 import org.mockito.ArgumentMatchers.anyString
 import org.mockito.Mockito
 import org.slf4j.Logger
@@ -24,6 +25,7 @@ class ErrorReporterImplTest {
 
     private val loggerMock = Mockito.mock(Logger::class.java).also {
         Mockito.`when`(it.error(anyString())).then { logs.addAll(it.arguments) }
+        Mockito.`when`(it.error(anyString(), any(Exception::class.java))).then { params -> logs.addAll(params.arguments) }
     }
 
     private val contextProvider: ErrorContextProvider  = object : ErrorContextProvider {
@@ -39,7 +41,8 @@ class ErrorReporterImplTest {
     private enum class TestErrors : ErrorCodes {
         CASE1,
         CASE2,
-        CASE_3;
+        CASE_3,
+        CASE4;
 
         override val namespace = TestNamespaces.TEST.toString()
     }
@@ -59,6 +62,11 @@ class ErrorReporterImplTest {
         override val parameters = listOf<Any>()
     }
 
+    private class TestError4(cause: Exception?) : Exception("This is test error 4", cause), ErrorCode<TestErrors> {
+        override val code = TestErrors.CASE4
+        override val parameters = listOf<Any>()
+    }
+
     private fun createReporterImpl(localeTag: String?) : ErrorReporterImpl {
         val locale = if (localeTag != null) Locale.forLanguageTag(localeTag) else Locale.getDefault()
         return ErrorReporterImpl("errorReporting", locale, contextProvider)
@@ -118,4 +126,12 @@ class ErrorReporterImplTest {
         testReporter.report(error, loggerMock)
         assertEquals(listOf("This is the third test message [Code: test-case-3 URL: $TEST_URL/en-US]"), logs)
     }
+
+    @Test(timeout = 3_000)
+    fun `exception based error code logs the stack trace`() {
+        val error = TestError4(Exception("A test exception"))
+        val testReporter = createReporterImpl("en-US")
+        testReporter.report(error, loggerMock)
+        assertEquals(listOf("This is the fourth test message [Code: test-case4 URL: $TEST_URL/en-US]", error), logs)
+    }
 }

common/logging/src/test/resources/errorReporting/test-case4.properties

@@ -0,0 +1,4 @@
+errorTemplate = This is the fourth test message
+shortDescription = Test description
+actionsToFix = Actions
+aliases =

common/logging/src/test/resources/errorReporting/test-case4_en_US.properties

@@ -0,0 +1,4 @@
+errorTemplate = This is the fourth test message
+shortDescription = Test description
+actionsToFix = Actions
+aliases =

constants.properties

@@ -4,14 +4,14 @@
 
 cordaVersion=4.6
 versionSuffix=SNAPSHOT
-gradlePluginsVersion=5.0.10
+gradlePluginsVersion=5.0.11
 kotlinVersion=1.2.71
 java8MinUpdateVersion=171
 # ***************************************************************#
 # When incrementing platformVersion make sure to update          #
 # net.corda.core.internal.CordaUtilsKt.PLATFORM_VERSION as well. #
 # ***************************************************************#
-platformVersion=7
+platformVersion=8
 guavaVersion=28.0-jre
 # Quasar version to use with Java 8:
 quasarVersion=0.7.12_r3
@@ -25,7 +25,7 @@ classgraphVersion=4.8.78
 disruptorVersion=3.4.2
 typesafeConfigVersion=1.3.4
 jsr305Version=3.0.2
-artifactoryPluginVersion=4.7.3
+artifactoryPluginVersion=4.16.1
 snakeYamlVersion=1.19
 caffeineVersion=2.7.0
 metricsVersion=4.1.0

core-deterministic/README.md

@@ -0,0 +1,2 @@
+## corda-core-deterministic.
+This artifact is a deterministic subset of the binary contents of `corda-core`.

core-deterministic/build.gradle

@@ -54,8 +54,8 @@ tasks.named('jar', Jar) {
     enabled = false
 }
 
-def coreJarTask = tasks.getByPath(':core:jar')
-def originalJar = coreJarTask.outputs.files.singleFile
+def coreJarTask = project(':core').tasks.named('jar', Jar)
+def originalJar = coreJarTask.map { it.outputs.files.singleFile }
 
 def patchCore = tasks.register('patchCore', Zip) {
     dependsOn coreJarTask
@@ -132,7 +132,7 @@ def jarFilter = tasks.register('jarFilter', JarFilterTask) {
     }
 }
 
-task determinise(type: ProGuardTask) {
+def determinise = tasks.register('determinise', ProGuardTask) {
     injars jarFilter
     outjars file("$buildDir/proguard/$jarBaseName-${project.version}.jar")
 
@@ -166,17 +166,20 @@ task determinise(type: ProGuardTask) {
     keepclassmembers 'class net.corda.core.** { public synthetic <methods>; }'
 }
 
-task metafix(type: MetaFixerTask) {
+def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask)
+
+def metafix = tasks.register('metafix', MetaFixerTask) {
     outputDir file("$buildDir/libs")
     jars determinise
     suffix ""
 
     // Strip timestamps from the JAR to make it reproducible.
     preserveTimestamps = false
+    finalizedBy checkDeterminism
 }
 
 // DOCSTART 01
-def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) {
+checkDeterminism.configure {
     dependsOn jdkTask
     injars metafix
 
@@ -197,20 +200,31 @@ def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) {
 // DOCEND 01
 
 defaultTasks "determinise"
-determinise.finalizedBy metafix
-metafix.finalizedBy checkDeterminism
-assemble.dependsOn checkDeterminism
+determinise.configure {
+    finalizedBy metafix
+}
+tasks.named('assemble') {
+    dependsOn checkDeterminism
+}
 
-def deterministicJar = metafix.outputs.files.singleFile
+def deterministicJar = metafix.map { it.outputs.files.singleFile }
 artifacts {
-    deterministicArtifacts file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix
-    publish file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix
+    deterministicArtifacts deterministicJar
+    publish deterministicJar
+}
+
+tasks.named('sourceJar', Jar) {
+    from 'README.md'
+    include 'README.md'
+}
+
+tasks.named('javadocJar', Jar) {
+    from 'README.md'
+    include 'README.md'
 }
 
 publish {
     dependenciesFrom configurations.deterministicArtifacts
-    publishSources = false
-    publishJavadoc = false
     name jarBaseName
 }
 

core-tests/build.gradle

@@ -1,7 +1,6 @@
 apply plugin: 'kotlin'
 apply plugin: 'kotlin-jpa'
 apply plugin: 'net.corda.plugins.quasar-utils'
-apply plugin: 'net.corda.plugins.publish-utils'
 
 description 'Corda core tests'
 
@@ -99,7 +98,7 @@ configurations {
     testArtifacts.extendsFrom testRuntimeClasspath
 }
 
-tasks.withType(Test) {
+tasks.withType(Test).configureEach {
     // fork a new test process for every test class
     forkEvery = 10
 }

core-tests/src/test/kotlin/net/corda/coretests/flows/FlowExternalAsyncOperationTest.kt

@@ -167,7 +167,7 @@ class FlowExternalAsyncOperationTest : AbstractFlowExternalOperationTest() {
 
         @Suspendable
         override fun testCode(): Any =
-            await(ExternalAsyncOperation(serviceHub) { _, _ ->
+            await(ExternalAsyncOperation(serviceHub) { serviceHub, _ ->
                 serviceHub.cordaService(FutureService::class.java).createFuture()
             })
     }

core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt

@@ -53,7 +53,7 @@ class ReceiveFinalityFlowTest {
 
         val paymentReceiverId = paymentReceiverFuture.getOrThrow()
         assertThat(bob.services.vaultService.queryBy<FungibleAsset<*>>().states).isEmpty()
-        bob.assertFlowSentForObservationDueToConstraintError(paymentReceiverId)
+        bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(paymentReceiverId)
 
         // Restart Bob with the contracts CorDapp so that it can recover from the error
         bob = mockNet.restartNode(bob,
@@ -71,7 +71,7 @@ class ReceiveFinalityFlowTest {
                 .ofType(R::class.java)
     }
 
-    private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) {
+    private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) {
         val observation = medicalRecordsOfType<Flow>()
                 .filter { it.flowId == runId }
                 .toBlocking()
@@ -79,6 +79,6 @@ class ReceiveFinalityFlowTest {
         assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION)
         assertThat(observation.by).contains(FinalityDoctor)
         val error = observation.errors.single()
-        assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java)
+        assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java)
     }
 }

core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt

@@ -55,7 +55,7 @@ class AttachmentsClassLoaderSerializationTests {
                 arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! },
                 testNetworkParameters(),
                 SecureHash.zeroHash,
-                { attachmentTrustCalculator.calculate(it) }) { classLoader ->
+                { attachmentTrustCalculator.calculate(it) }, attachmentsClassLoaderCache = null) { classLoader ->
             val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader)
             val contract = contractClass.getDeclaredConstructor().newInstance() as Contract
             assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)

core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt

@@ -23,6 +23,7 @@ import net.corda.core.internal.inputStream
 import net.corda.core.node.NetworkParameters
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.serialization.internal.AttachmentsClassLoader
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
 import net.corda.testing.common.internal.testNetworkParameters
 import net.corda.node.services.attachments.NodeAttachmentTrustCalculator
 import net.corda.testing.contracts.DummyContract
@@ -521,6 +522,7 @@ class AttachmentsClassLoaderTests {
             val id = SecureHash.randomSHA256()
             val timeWindow: TimeWindow? = null
             val privacySalt = PrivacySalt()
+            val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory)
             val transaction = createLedgerTransaction(
                     inputs,
                     outputs,
@@ -532,7 +534,8 @@ class AttachmentsClassLoaderTests {
                     privacySalt,
                     testNetworkParameters(),
                     emptyList(),
-                    isAttachmentTrusted = { true }
+                    isAttachmentTrusted = { true },
+                    attachmentsClassLoaderCache = attachmentsClassLoaderCache
             )
             transaction.verify()
         }

core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt

@@ -10,6 +10,7 @@ import net.corda.core.internal.AbstractAttachment
 import net.corda.core.internal.TESTDSL_UPLOADER
 import net.corda.core.internal.createLedgerTransaction
 import net.corda.core.node.NotaryInfo
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.WireTransaction
 import net.corda.testing.common.internal.testNetworkParameters
@@ -18,6 +19,7 @@ import net.corda.testing.core.*
 import net.corda.testing.internal.createWireTransaction
 import net.corda.testing.internal.fakeAttachment
 import net.corda.coretesting.internal.rigorousMock
+import net.corda.testing.internal.TestingNamedCacheFactory
 import org.junit.Rule
 import org.junit.Test
 import java.math.BigInteger
@@ -131,6 +133,7 @@ class TransactionTests {
         val id = SecureHash.randomSHA256()
         val timeWindow: TimeWindow? = null
         val privacySalt = PrivacySalt()
+        val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory())
         val transaction = createLedgerTransaction(
                 inputs,
                 outputs,
@@ -142,7 +145,8 @@ class TransactionTests {
                 privacySalt,
                 testNetworkParameters(),
                 emptyList(),
-                isAttachmentTrusted = { true }
+                isAttachmentTrusted = { true },
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache
         )
 
         transaction.verify()
@@ -183,6 +187,7 @@ class TransactionTests {
         val id = SecureHash.randomSHA256()
         val timeWindow: TimeWindow? = null
         val privacySalt = PrivacySalt()
+        val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory())
 
         fun buildTransaction() = createLedgerTransaction(
                 inputs,
@@ -195,7 +200,8 @@ class TransactionTests {
                 privacySalt,
                 testNetworkParameters(notaries = listOf(NotaryInfo(DUMMY_NOTARY, true))),
                 emptyList(),
-                isAttachmentTrusted = { true }
+                isAttachmentTrusted = { true },
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache
         )
 
         assertFailsWith<TransactionVerificationException.NotaryChangeInWrongTransactionType> { buildTransaction().verify() }

core/src/main/kotlin/net/corda/core/contracts/Structures.kt

@@ -89,6 +89,7 @@ interface OwnableState : ContractState {
 // DOCEND 3
 
 /** Something which is scheduled to happen at a point in time. */
+@KeepForDJVM
 interface Scheduled {
     val scheduledAt: Instant
 }
@@ -101,6 +102,7 @@ interface Scheduled {
  * lifecycle processing needs to take place.  e.g. a fixing or a late payment etc.
  */
 @CordaSerializable
+@KeepForDJVM
 data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instant) : Scheduled
 
 /**
@@ -115,7 +117,7 @@ data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instan
  * for a particular [ContractState] have been processed/fired etc.  If the activity is not "on ledger" then the
  * scheduled activity shouldn't be either.
  */
-@DeleteForDJVM
+@KeepForDJVM
 data class ScheduledActivity(val logicRef: FlowLogicRef, override val scheduledAt: Instant) : Scheduled
 
 // DOCSTART 2
@@ -134,7 +136,7 @@ interface LinearState : ContractState {
     val linearId: UniqueIdentifier
 }
 // DOCEND 2
-@DeleteForDJVM
+@KeepForDJVM
 interface SchedulableState : ContractState {
     /**
      * Indicate whether there is some activity to be performed at some future point in time with respect to this

core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt

@@ -7,6 +7,7 @@ import net.corda.core.crypto.SecureHash
 import net.corda.core.flows.FlowLogic
 import net.corda.core.internal.cordapp.CordappImpl.Companion.UNKNOWN_VALUE
 import net.corda.core.schemas.MappedSchema
+import net.corda.core.serialization.CheckpointCustomSerializer
 import net.corda.core.serialization.SerializationCustomSerializer
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
@@ -29,6 +30,7 @@ import java.net.URL
  * @property services List of RPC services
  * @property serializationWhitelists List of Corda plugin registries
  * @property serializationCustomSerializers List of serializers
+ * @property checkpointCustomSerializers List of serializers for checkpoints
  * @property customSchemas List of custom schemas
  * @property allFlows List of all flow classes
  * @property jarPath The path to the JAR for this CorDapp
@@ -49,6 +51,7 @@ interface Cordapp {
     val services: List<Class<out SerializeAsToken>>
     val serializationWhitelists: List<SerializationWhitelist>
     val serializationCustomSerializers: List<SerializationCustomSerializer<*, *>>
+    val checkpointCustomSerializers: List<CheckpointCustomSerializer<*, *>>
     val customSchemas: Set<MappedSchema>
     val allFlows: List<Class<out FlowLogic<*>>>
     val jarPath: URL

core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt

@@ -25,6 +25,7 @@ import net.corda.core.node.NodeInfo
 import net.corda.core.node.ServiceHub
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.transactions.SignedTransaction
+import net.corda.core.utilities.NonEmptySet
 import net.corda.core.utilities.ProgressTracker
 import net.corda.core.utilities.UntrustworthyData
 import net.corda.core.utilities.debug
@@ -378,6 +379,22 @@ abstract class FlowLogic<out T> {
         stateMachine.suspend(request, maySkipCheckpoint)
     }
 
+    /**
+     * Closes the provided sessions and performs cleanup of any resources tied to these sessions.
+     *
+     * Note that sessions are closed automatically when the corresponding top-level flow terminates.
+     * So, it's beneficial to eagerly close them in long-lived flows that might have many open sessions that are not needed anymore and consume resources (e.g. memory, disk etc.).
+     * A closed session cannot be used anymore, e.g. to send or receive messages. So, you have to ensure you are calling this method only when the provided sessions are not going to be used anymore.
+     * As a result, any operations on a closed session will fail with an [UnexpectedFlowEndException].
+     * When a session is closed, the other side is informed and the session is closed there too eventually.
+     * To prevent misuse of the API, if there is an attempt to close an uninitialised session the invocation will fail with an [IllegalStateException].
+     */
+    @Suspendable
+    fun close(sessions: NonEmptySet<FlowSession>) {
+        val request = FlowIORequest.CloseSessions(sessions)
+        stateMachine.suspend(request, false)
+    }
+
     /**
      * Invokes the given subflow. This function returns once the subflow completes successfully with the result
      * returned by that subflow's [call] method. If the subflow has a progress tracker, it is attached to the

core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt

@@ -1,7 +1,9 @@
 package net.corda.core.flows
 
 import net.corda.core.CordaInternal
+import net.corda.core.DeleteForDJVM
 import net.corda.core.DoNotImplement
+import net.corda.core.KeepForDJVM
 import net.corda.core.serialization.CordaSerializable
 
 /**
@@ -11,11 +13,13 @@ import net.corda.core.serialization.CordaSerializable
  * the flow to run at the scheduled time.
  */
 @DoNotImplement
+@KeepForDJVM
 interface FlowLogicRefFactory {
     /**
      * Construct a FlowLogicRef. This is intended for cases where the calling code has the relevant class already
      * and can provide it directly.
      */
+    @DeleteForDJVM
     fun create(flowClass: Class<out FlowLogic<*>>, vararg args: Any?): FlowLogicRef
 
     /**
@@ -30,12 +34,14 @@ interface FlowLogicRefFactory {
      * [SchedulableFlow] annotation.
      */
     @CordaInternal
+    @DeleteForDJVM
     fun createForRPC(flowClass: Class<out FlowLogic<*>>, vararg args: Any?): FlowLogicRef
 
     /**
      * Converts a [FlowLogicRef] object that was obtained from the calls above into a [FlowLogic], after doing some
      * validation to ensure it points to a legitimate flow class.
      */
+    @DeleteForDJVM
     fun toFlowLogic(ref: FlowLogicRef): FlowLogic<*>
 }
 
@@ -59,4 +65,5 @@ class IllegalFlowLogicException(val type: String, msg: String) :
 // TODO: align this with the existing [FlowRef] in the bank-side API (probably replace some of the API classes)
 @CordaSerializable
 @DoNotImplement
+@KeepForDJVM
 interface FlowLogicRef

core/src/main/kotlin/net/corda/core/flows/FlowSession.kt

@@ -191,6 +191,19 @@ abstract class FlowSession {
      */
     @Suspendable
     abstract fun send(payload: Any)
+
+    /**
+     * Closes this session and performs cleanup of any resources tied to this session.
+     *
+     * Note that sessions are closed automatically when the corresponding top-level flow terminates.
+     * So, it's beneficial to eagerly close them in long-lived flows that might have many open sessions that are not needed anymore and consume resources (e.g. memory, disk etc.).
+     * A closed session cannot be used anymore, e.g. to send or receive messages. So, you have to ensure you are calling this method only when the session is not going to be used anymore.
+     * As a result, any operations on a closed session will fail with an [UnexpectedFlowEndException].
+     * When a session is closed, the other side is informed and the session is closed there too eventually.
+     * To prevent misuse of the API, if there is an attempt to close an uninitialised session the invocation will fail with an [IllegalStateException].
+     */
+    @Suspendable
+    abstract fun close()
 }
 
 /**

core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt

@@ -28,7 +28,7 @@ import java.util.jar.JarInputStream
 
 // *Internal* Corda-specific utilities.
 
-const val PLATFORM_VERSION = 7
+const val PLATFORM_VERSION = 8
 
 fun ServicesForResolution.ensureMinimumPlatformVersion(requiredMinPlatformVersion: Int, feature: String) {
     checkMinimumPlatformVersion(networkParameters.minimumPlatformVersion, requiredMinPlatformVersion, feature)

core/src/main/kotlin/net/corda/core/internal/FlowIORequest.kt

@@ -55,6 +55,13 @@ sealed class FlowIORequest<out R : Any> {
         }}, shouldRetrySend=$shouldRetrySend)"
     }
 
+    /**
+     * Closes the specified sessions.
+     *
+     * @property sessions the sessions to be closed.
+     */
+    data class CloseSessions(val sessions: NonEmptySet<FlowSession>): FlowIORequest<Unit>()
+
     /**
      * Wait for a transaction to be committed to the database.
      *

core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt

@@ -5,6 +5,7 @@ import net.corda.core.DeleteForDJVM
 import net.corda.core.internal.notary.NotaryService
 import net.corda.core.node.ServiceHub
 import net.corda.core.node.StatesToRecord
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
 import java.util.concurrent.ExecutorService
 
 // TODO: This should really be called ServiceHubInternal but that name is already taken by net.corda.node.services.api.ServiceHubInternal.
@@ -21,6 +22,8 @@ interface ServiceHubCoreInternal : ServiceHub {
     val notaryService: NotaryService?
 
     fun createTransactionsResolver(flow: ResolveTransactionsFlow): TransactionsResolver
+
+    val attachmentsClassLoaderCache: AttachmentsClassLoaderCache
 }
 
 interface TransactionsResolver {

core/src/main/kotlin/net/corda/core/internal/concurrent/CordaFutureImpl.kt

@@ -27,6 +27,12 @@ fun <V, W, X> CordaFuture<out V>.thenMatch(success: (V) -> W, failure: (Throwabl
 /** When this future is done and the outcome is failure, log the throwable. */
 fun CordaFuture<*>.andForget(log: Logger) = thenMatch({}, { log.error("Background task failed:", it) })
 
+/**
+ * Returns a future that will also apply the passed closure when it completes.
+ *
+ * @param accept A function to execute when completing the original future.
+ * @return A future returning the same result as the original future that this function was executed on.
+ */
 fun <RESULT> CordaFuture<out RESULT>.doOnComplete(accept: (RESULT) -> Unit): CordaFuture<RESULT> {
     return CordaFutureImpl<RESULT>().also { result ->
         thenMatch({

core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt

@@ -9,6 +9,7 @@ import net.corda.core.internal.VisibleForTesting
 import net.corda.core.internal.notary.NotaryService
 import net.corda.core.internal.toPath
 import net.corda.core.schemas.MappedSchema
+import net.corda.core.serialization.CheckpointCustomSerializer
 import net.corda.core.serialization.SerializationCustomSerializer
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
@@ -25,6 +26,7 @@ data class CordappImpl(
         override val services: List<Class<out SerializeAsToken>>,
         override val serializationWhitelists: List<SerializationWhitelist>,
         override val serializationCustomSerializers: List<SerializationCustomSerializer<*, *>>,
+        override val checkpointCustomSerializers: List<CheckpointCustomSerializer<*, *>>,
         override val customSchemas: Set<MappedSchema>,
         override val allFlows: List<Class<out FlowLogic<*>>>,
         override val jarPath: URL,
@@ -47,7 +49,7 @@ data class CordappImpl(
     }
 
     companion object {
-        fun jarName(url: URL): String = url.toPath().fileName.toString().removeSuffix(".jar")
+        fun jarName(url: URL): String = (url.toPath().fileName ?: "").toString().removeSuffix(".jar")
 
         /** CorDapp manifest entries */
         const val CORDAPP_CONTRACT_NAME = "Cordapp-Contract-Name"
@@ -79,9 +81,10 @@ data class CordappImpl(
                 services = emptyList(),
                 serializationWhitelists = emptyList(),
                 serializationCustomSerializers = emptyList(),
+                checkpointCustomSerializers = emptyList(),
                 customSchemas = emptySet(),
                 jarPath = Paths.get("").toUri().toURL(),
-                info = CordappImpl.UNKNOWN_INFO,
+                info = UNKNOWN_INFO,
                 allFlows = emptyList(),
                 jarHash = SecureHash.allOnesHash,
                 minimumPlatformVersion = 1,

core/src/main/kotlin/net/corda/core/messaging/CordaRPCOps.kt

@@ -302,7 +302,12 @@ interface CordaRPCOps : RPCOps {
     /** Checks whether an attachment with the given hash is stored on the node. */
     fun attachmentExists(id: SecureHash): Boolean
 
-    /** Download an attachment JAR by ID. */
+    /**
+     * Download an attachment JAR by ID.
+     * @param id the id of the attachment to open
+     * @return the stream of the JAR
+     * @throws RPCException if the attachment doesn't exist
+     * */
     fun openAttachment(id: SecureHash): InputStream
 
     /** Uploads a jar to the node, returns it's hash. */

core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt

@@ -25,3 +25,26 @@ interface SerializationCustomSerializer<OBJ, PROXY> {
      */
     fun fromProxy(proxy: PROXY): OBJ
 }
+
+/**
+ * Allows CorDapps to provide custom serializers for classes that do not serialize successfully during a checkpoint.
+ * In this case, a proxy serializer can be written that implements this interface whose purpose is to move between
+ * unserializable types and an intermediate representation.
+ *
+ * NOTE: Only implement this interface if you have a class that triggers an error during normal checkpoint
+ * serialization/deserialization.
+ */
+@KeepForDJVM
+interface CheckpointCustomSerializer<OBJ, PROXY> {
+    /**
+     * Should facilitate the conversion of the third party object into the serializable
+     * local class specified by [PROXY]
+     */
+    fun toProxy(obj: OBJ): PROXY
+
+    /**
+     * Should facilitate the conversion of the proxy object into a new instance of the
+     * unserializable type
+     */
+    fun fromProxy(proxy: PROXY): OBJ
+}

core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt

@@ -1,5 +1,8 @@
 package net.corda.core.serialization.internal
 
+import com.github.benmanes.caffeine.cache.Cache
+import com.github.benmanes.caffeine.cache.Caffeine
+import net.corda.core.DeleteForDJVM
 import net.corda.core.contracts.Attachment
 import net.corda.core.contracts.ContractAttachment
 import net.corda.core.contracts.TransactionVerificationException
@@ -21,6 +24,7 @@ import java.lang.ref.WeakReference
 import java.net.*
 import java.security.Permission
 import java.util.*
+import java.util.function.Function
 
 /**
  * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only
@@ -289,31 +293,27 @@ class AttachmentsClassLoader(attachments: List<Attachment>,
  */
 @VisibleForTesting
 object AttachmentsClassLoaderBuilder {
-    private const val CACHE_SIZE = 1000
+    const val CACHE_SIZE = 16
 
-    // We use a set here because the ordering of attachments doesn't affect code execution, due to the no
-    // overlap rule, and attachments don't have any particular ordering enforced by the builders. So we
-    // can just do unordered comparisons here. But the same attachments run with different network parameters
-    // may behave differently, so that has to be a part of the cache key.
-    private data class Key(val hashes: Set<SecureHash>, val params: NetworkParameters)
-
-    // This runs in the DJVM so it can't use caffeine.
-    private val cache: MutableMap<Key, SerializationContext> = createSimpleCache<Key, SerializationContext>(CACHE_SIZE).toSynchronised()
+    private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE)
 
     /**
      * Runs the given block with serialization execution context set up with a (possibly cached) attachments classloader.
      *
      * @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup.
      */
+    @Suppress("LongParameterList")
     fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>,
                                               params: NetworkParameters,
                                               txId: SecureHash,
                                               isAttachmentTrusted: (Attachment) -> Boolean,
                                               parent: ClassLoader = ClassLoader.getSystemClassLoader(),
+                                              attachmentsClassLoaderCache: AttachmentsClassLoaderCache?,
                                               block: (ClassLoader) -> T): T {
         val attachmentIds = attachments.map(Attachment::id).toSet()
 
-        val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) {
+        val cache = attachmentsClassLoaderCache ?: fallBackCache
+        val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function {
             // Create classloader and load serializers, whitelisted classes
             val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, isAttachmentTrusted, parent)
             val serializers = try {
@@ -336,7 +336,7 @@ object AttachmentsClassLoaderBuilder {
                     .withWhitelist(whitelistedClasses)
                     .withCustomSerializers(serializers)
                     .withoutCarpenter()
-        }
+        })
 
         // Deserialize all relevant classes in the transaction classloader.
         return SerializationFactory.defaultFactory.withCurrentContext(serializationContext) {
@@ -420,6 +420,36 @@ private class AttachmentsHolderImpl : AttachmentsHolder {
     }
 }
 
+interface AttachmentsClassLoaderCache {
+    fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext
+}
+
+@DeleteForDJVM
+class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache {
+
+    private val cache: Cache<AttachmentsClassLoaderKey, SerializationContext> = cacheFactory.buildNamed(Caffeine.newBuilder(), "AttachmentsClassLoader_cache")
+
+    override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
+        return cache.get(key, mappingFunction)  ?: throw NullPointerException("null returned from cache mapping function")
+    }
+}
+
+class AttachmentsClassLoaderSimpleCacheImpl(cacheSize: Int) : AttachmentsClassLoaderCache {
+
+    private val cache: MutableMap<AttachmentsClassLoaderKey, SerializationContext>
+            = createSimpleCache<AttachmentsClassLoaderKey, SerializationContext>(cacheSize).toSynchronised()
+
+    override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
+        return cache.computeIfAbsent(key, mappingFunction)
+    }
+}
+
+// We use a set here because the ordering of attachments doesn't affect code execution, due to the no
+// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we
+// can just do unordered comparisons here. But the same attachments run with different network parameters
+// may behave differently, so that has to be a part of the cache key.
+data class AttachmentsClassLoaderKey(val hashes: Set<SecureHash>, val params: NetworkParameters)
+
 private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) {
     override fun getContentLengthLong(): Long = attachment.size.toLong()
     override fun getInputStream(): InputStream = attachment.open()

core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt

@@ -56,6 +56,10 @@ interface CheckpointSerializationContext {
      * otherwise they appear as new copies of the object.
      */
     val objectReferencesEnabled: Boolean
+    /**
+     * User defined custom serializers for use in checkpoint serialization.
+     */
+    val checkpointCustomSerializers: Iterable<CheckpointCustomSerializer<*,*>>
 
     /**
      * Helper method to return a new context based on this context with the property added.
@@ -86,6 +90,11 @@ interface CheckpointSerializationContext {
      * A shallow copy of this context but with the given encoding whitelist.
      */
     fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist): CheckpointSerializationContext
+
+    /**
+     * A shallow copy of this context but with the given custom serializers.
+     */
+    fun withCheckpointCustomSerializers(checkpointCustomSerializers: Iterable<CheckpointCustomSerializer<*, *>>): CheckpointSerializationContext
 }
 
 /*

core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt

@@ -153,7 +153,8 @@ data class ContractUpgradeWireTransaction(
                 listOf(legacyAttachment, upgradedAttachment),
                 params,
                 id,
-                { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }) { transactionClassLoader ->
+                { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) },
+                attachmentsClassLoaderCache = (services as ServiceHubCoreInternal).attachmentsClassLoaderCache) { transactionClassLoader ->
             val resolvedInput = binaryInput.deserialize()
             val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader)
             val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment)

core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt

@@ -26,6 +26,7 @@ import net.corda.core.internal.deserialiseComponentGroup
 import net.corda.core.internal.isUploaderTrusted
 import net.corda.core.internal.uncheckedCast
 import net.corda.core.node.NetworkParameters
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
 import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
 import net.corda.core.utilities.contextLogger
 import java.util.Collections.unmodifiableList
@@ -87,7 +88,8 @@ private constructor(
         private val serializedInputs: List<SerializedStateAndRef>?,
         private val serializedReferences: List<SerializedStateAndRef>?,
         private val isAttachmentTrusted: (Attachment) -> Boolean,
-        private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier
+        private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier,
+        private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
 ) : FullTransaction() {
 
     init {
@@ -124,7 +126,8 @@ private constructor(
                 componentGroups: List<ComponentGroup>? = null,
                 serializedInputs: List<SerializedStateAndRef>? = null,
                 serializedReferences: List<SerializedStateAndRef>? = null,
-                isAttachmentTrusted: (Attachment) -> Boolean
+                isAttachmentTrusted: (Attachment) -> Boolean,
+                attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
         ): LedgerTransaction {
             return LedgerTransaction(
                 inputs = inputs,
@@ -141,7 +144,8 @@ private constructor(
                 serializedInputs = protect(serializedInputs),
                 serializedReferences = protect(serializedReferences),
                 isAttachmentTrusted = isAttachmentTrusted,
-                verifierFactory = ::BasicVerifier
+                verifierFactory = ::BasicVerifier,
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache
             )
         }
 
@@ -176,7 +180,8 @@ private constructor(
                 serializedInputs = null,
                 serializedReferences = null,
                 isAttachmentTrusted = { true },
-                verifierFactory = ::BasicVerifier
+                verifierFactory = ::BasicVerifier,
+                attachmentsClassLoaderCache = null
             )
         }
     }
@@ -218,7 +223,8 @@ private constructor(
                 txAttachments,
                 getParamsWithGoo(),
                 id,
-                isAttachmentTrusted = isAttachmentTrusted) { transactionClassLoader ->
+                isAttachmentTrusted = isAttachmentTrusted,
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache) { transactionClassLoader ->
             // Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader].
             // Only the copy will be used for verification, and the outer shell will be discarded.
             // This artifice is required to preserve backwards compatibility.
@@ -254,7 +260,8 @@ private constructor(
         serializedInputs = serializedInputs,
         serializedReferences = serializedReferences,
         isAttachmentTrusted = isAttachmentTrusted,
-        verifierFactory = alternateVerifier
+        verifierFactory = alternateVerifier,
+        attachmentsClassLoaderCache = attachmentsClassLoaderCache
     )
 
     // Read network parameters with backwards compatibility goo.
@@ -320,7 +327,8 @@ private constructor(
                     serializedInputs = serializedInputs,
                     serializedReferences = serializedReferences,
                     isAttachmentTrusted = isAttachmentTrusted,
-                    verifierFactory = verifierFactory
+                    verifierFactory = verifierFactory,
+                    attachmentsClassLoaderCache = attachmentsClassLoaderCache
             )
         } else {
             // This branch is only present for backwards compatibility.
@@ -704,7 +712,8 @@ private constructor(
             serializedInputs = null,
             serializedReferences = null,
             isAttachmentTrusted = { it.isUploaderTrusted() },
-            verifierFactory = ::BasicVerifier
+            verifierFactory = ::BasicVerifier,
+            attachmentsClassLoaderCache = null
     )
 
     @Deprecated("LedgerTransaction should not be created directly, use WireTransaction.toLedgerTransaction instead.")
@@ -733,7 +742,8 @@ private constructor(
             serializedInputs = null,
             serializedReferences = null,
             isAttachmentTrusted = { it.isUploaderTrusted() },
-            verifierFactory = ::BasicVerifier
+            verifierFactory = ::BasicVerifier,
+            attachmentsClassLoaderCache = null
     )
 
     @Deprecated("LedgerTransactions should not be created directly, use WireTransaction.toLedgerTransaction instead.")
@@ -761,7 +771,8 @@ private constructor(
                 serializedInputs = serializedInputs,
                 serializedReferences = serializedReferences,
                 isAttachmentTrusted = isAttachmentTrusted,
-                verifierFactory = verifierFactory
+                verifierFactory = verifierFactory,
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache
         )
     }
 
@@ -791,7 +802,8 @@ private constructor(
                 serializedInputs = serializedInputs,
                 serializedReferences = serializedReferences,
                 isAttachmentTrusted = isAttachmentTrusted,
-                verifierFactory = verifierFactory
+                verifierFactory = verifierFactory,
+                attachmentsClassLoaderCache = attachmentsClassLoaderCache
         )
     }
 }

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -15,6 +15,7 @@ import net.corda.core.node.ServicesForResolution
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.serialization.SerializedBytes
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.OpaqueBytes
 import java.security.PublicKey
@@ -109,7 +110,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
                     services.networkParametersService.lookup(hashToResolve)
                 },
                 // `as?` is used due to [MockServices] not implementing [ServiceHubCoreInternal]
-                isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true }
+                isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true },
+                attachmentsClassLoaderCache = (services as? ServiceHubCoreInternal)?.attachmentsClassLoaderCache
             )
         )
     }
@@ -145,7 +147,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
                 resolveAttachment,
                 { stateRef -> resolveStateRef(stateRef)?.serialize() },
                 { null },
-                { it.isUploaderTrusted() }
+                { it.isUploaderTrusted() },
+                null
         )
     }
 
@@ -161,16 +164,19 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
                 resolveAttachment,
                 { stateRef -> resolveStateRef(stateRef)?.serialize() },
                 resolveParameters,
-                { true } // Any attachment loaded through the DJVM should be trusted
+                { true }, // Any attachment loaded through the DJVM should be trusted
+                null
         )
     }
 
+    @Suppress("LongParameterList", "ThrowsCount")
     private fun toLedgerTransactionInternal(
             resolveIdentity: (PublicKey) -> Party?,
             resolveAttachment: (SecureHash) -> Attachment?,
             resolveStateRefAsSerialized: (StateRef) -> SerializedBytes<TransactionState<ContractState>>?,
             resolveParameters: (SecureHash?) -> NetworkParameters?,
-            isAttachmentTrusted: (Attachment) -> Boolean
+            isAttachmentTrusted: (Attachment) -> Boolean,
+            attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
     ): LedgerTransaction {
         // Look up public keys to authenticated identities.
         val authenticatedCommands = commands.lazyMapped { cmd, _ ->
@@ -206,7 +212,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
                 componentGroups,
                 serializedResolvedInputs,
                 serializedResolvedReferences,
-                isAttachmentTrusted
+                isAttachmentTrusted,
+                attachmentsClassLoaderCache
         )
 
         checkTransactionSize(ltx, resolvedNetworkParameters.maxTransactionSize, serializedResolvedInputs, serializedResolvedReferences)

core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt

@@ -4,6 +4,7 @@ import net.corda.core.contracts.*
 import net.corda.core.crypto.SecureHash
 import net.corda.core.identity.Party
 import net.corda.core.node.NetworkParameters
+import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
 import net.corda.core.transactions.ComponentGroup
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.transactions.WireTransaction
@@ -17,6 +18,7 @@ fun WireTransaction.accessGroupHashes() = this.groupHashes
 fun WireTransaction.accessGroupMerkleRoots() = this.groupsMerkleRoots
 fun WireTransaction.accessAvailableComponentHashes() = this.availableComponentHashes
 
+@Suppress("LongParameterList")
 fun createLedgerTransaction(
         inputs: List<StateAndRef<ContractState>>,
         outputs: List<TransactionState<ContractState>>,
@@ -31,8 +33,9 @@ fun createLedgerTransaction(
         componentGroups: List<ComponentGroup>? = null,
         serializedInputs: List<SerializedStateAndRef>? = null,
         serializedReferences: List<SerializedStateAndRef>? = null,
-        isAttachmentTrusted: (Attachment) -> Boolean
-): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted)
+        isAttachmentTrusted: (Attachment) -> Boolean,
+        attachmentsClassLoaderCache: AttachmentsClassLoaderCache
+): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted, attachmentsClassLoaderCache)
 
 fun createContractCreationError(txId: SecureHash, contractClass: String, cause: Throwable) = TransactionVerificationException.ContractCreationError(txId, contractClass, cause)
 fun createContractRejection(txId: SecureHash, contract: Contract, cause: Throwable) = TransactionVerificationException.ContractRejection(txId, contract, cause)

detekt-baseline.xml

@@ -1435,7 +1435,7 @@
     <ID>ThrowsCount:JarScanningCordappLoader.kt$JarScanningCordappLoader$private fun parseVersion(versionStr: String?, attributeName: String): Int</ID>
     <ID>ThrowsCount:LedgerDSLInterpreter.kt$Verifies$ fun failsWith(expectedMessage: String?): EnforceVerifyOrFail</ID>
     <ID>ThrowsCount:MockServices.kt$ fun &lt;T : SerializeAsToken&gt; createMockCordaService(serviceHub: MockServices, serviceConstructor: (AppServiceHub) -&gt; T): T</ID>
-    <ID>ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates(registeringPublicKey: PublicKey, certificates: List&lt;X509Certificate&gt;)</ID>
+    <ID>ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates( registeringPublicKey: PublicKey, registeringLegalName: CordaX500Name, expectedCertRole: CertRole, certificates: List&lt;X509Certificate&gt; )</ID>
     <ID>ThrowsCount:NodeInfoFilesCopier.kt$NodeInfoFilesCopier$private fun atomicCopy(source: Path, destination: Path)</ID>
     <ID>ThrowsCount:NodeVaultService.kt$NodeVaultService$@Throws(VaultQueryException::class) private fun &lt;T : ContractState&gt; _queryBy(criteria: QueryCriteria, paging_: PageSpecification, sorting: Sort, contractStateType: Class&lt;out T&gt;, skipPagingChecks: Boolean): Vault.Page&lt;T&gt;</ID>
     <ID>ThrowsCount:NodeVaultService.kt$NodeVaultService$private fun makeUpdates(batch: Iterable&lt;CoreTransaction&gt;, statesToRecord: StatesToRecord, previouslySeen: Boolean): List&lt;Vault.Update&lt;ContractState&gt;&gt;</ID>
@@ -1598,6 +1598,7 @@
     <ID>TooGenericExceptionCaught:ScheduledFlowIntegrationTests.kt$ScheduledFlowIntegrationTests$ex: Exception</ID>
     <ID>TooGenericExceptionCaught:SerializationOutputTests.kt$SerializationOutputTests$t: Throwable</ID>
     <ID>TooGenericExceptionCaught:ShutdownManager.kt$ShutdownManager$t: Throwable</ID>
+    <ID>TooGenericExceptionCaught:SimpleAMQPClient.kt$SimpleAMQPClient$e: Exception</ID>
     <ID>TooGenericExceptionCaught:SimpleMQClient.kt$SimpleMQClient$e: Exception</ID>
     <ID>TooGenericExceptionCaught:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager$e: Exception</ID>
     <ID>TooGenericExceptionCaught:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager$ex: Exception</ID>
@@ -1617,6 +1618,7 @@
     <ID>TooGenericExceptionCaught:TransformTypes.kt$TransformTypes.Companion$e: IndexOutOfBoundsException</ID>
     <ID>TooGenericExceptionCaught:TransitionExecutorImpl.kt$TransitionExecutorImpl$exception: Exception</ID>
     <ID>TooGenericExceptionCaught:Try.kt$Try.Companion$t: Throwable</ID>
+    <ID>TooGenericExceptionCaught:UserValidationPlugin.kt$UserValidationPlugin$e: Throwable</ID>
     <ID>TooGenericExceptionCaught:Utils.kt$e: Exception</ID>
     <ID>TooGenericExceptionCaught:V1NodeConfigurationSpec.kt$V1NodeConfigurationSpec$e: Exception</ID>
     <ID>TooGenericExceptionCaught:ValidatingNotaryFlow.kt$ValidatingNotaryFlow$e: Exception</ID>

deterministic.gradle

@@ -11,12 +11,12 @@ evaluationDependsOn(':jdk8u-deterministic')
 def jdk8uDeterministic = project(':jdk8u-deterministic')
 
 ext {
-    jdkTask = jdk8uDeterministic.assemble
+    jdkTask = jdk8uDeterministic.tasks.named('assemble')
     deterministic_jdk_home = jdk8uDeterministic.jdk_home
     deterministic_rt_jar = jdk8uDeterministic.rt_jar
 }
 
-tasks.withType(AbstractCompile) {
+tasks.withType(AbstractCompile).configureEach {
     dependsOn jdkTask
 
     // This is a bit ugly, but Gradle isn't recognising the KotlinCompile task
@@ -29,7 +29,7 @@ tasks.withType(AbstractCompile) {
     }
 }
 
-tasks.withType(JavaCompile) {
+tasks.withType(JavaCompile).configureEach {
     options.compilerArgs << '-bootclasspath' << deterministic_rt_jar
     sourceCompatibility = VERSION_1_8
     targetCompatibility = VERSION_1_8

docker/src/docker/Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     mkdir -p /opt/corda/cordapps && \
     mkdir -p /opt/corda/persistence && \
+    mkdir -p /opt/corda/artemis && \
     mkdir -p /opt/corda/certificates && \
     mkdir -p /opt/corda/drivers && \
     mkdir -p /opt/corda/logs && \
@@ -20,6 +21,7 @@ RUN apt-get update && \
 
 ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
     PERSISTENCE_FOLDER="/opt/corda/persistence" \
+    ARTEMIS_FOLDER="/opt/corda/artemis" \
     CERTIFICATES_FOLDER="/opt/corda/certificates" \
     DRIVERS_FOLDER="/opt/corda/drivers" \
     CONFIG_FOLDER="/etc/corda" \
@@ -34,6 +36,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
 VOLUME ["/opt/corda/cordapps"]
 ##PERSISTENCE FOLDER
 VOLUME ["/opt/corda/persistence"]
+##ARTEMIS FOLDER
+VOLUME ["/opt/corda/artemis"]
 ##CERTS FOLDER
 VOLUME ["/opt/corda/certificates"]
 ##OPTIONAL JDBC DRIVERS FOLDER

docker/src/docker/Dockerfile11

@@ -19,6 +19,7 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     mkdir -p /opt/corda/cordapps && \
     mkdir -p /opt/corda/persistence && \
+    mkdir -p /opt/corda/artemis && \
     mkdir -p /opt/corda/certificates && \
     mkdir -p /opt/corda/drivers && \
     mkdir -p /opt/corda/logs && \
@@ -36,6 +37,7 @@ RUN apt-get update && \
 
 ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
     PERSISTENCE_FOLDER="/opt/corda/persistence" \
+    ARTEMIS_FOLDER="/opt/corda/artemis" \
     CERTIFICATES_FOLDER="/opt/corda/certificates" \
     DRIVERS_FOLDER="/opt/corda/drivers" \
     CONFIG_FOLDER="/etc/corda" \
@@ -50,6 +52,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
 VOLUME ["/opt/corda/cordapps"]
 ##PERSISTENCE FOLDER
 VOLUME ["/opt/corda/persistence"]
+##ARTEMIS FOLDER
+VOLUME ["/opt/corda/artemis"]
 ##CERTS FOLDER
 VOLUME ["/opt/corda/certificates"]
 ##OPTIONAL JDBC DRIVERS FOLDER

docker/src/docker/DockerfileAL

@@ -10,6 +10,7 @@ RUN amazon-linux-extras enable corretto8 && \
     rm -rf /var/cache/yum && \
     mkdir -p /opt/corda/cordapps && \
     mkdir -p /opt/corda/persistence && \
+    mkdir -p /opt/corda/artemis && \
     mkdir -p /opt/corda/certificates && \
     mkdir -p /opt/corda/drivers && \
     mkdir -p /opt/corda/logs && \
@@ -23,6 +24,7 @@ RUN amazon-linux-extras enable corretto8 && \
 
 ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
     PERSISTENCE_FOLDER="/opt/corda/persistence" \
+    ARTEMIS_FOLDER="/opt/corda/artemis" \
     CERTIFICATES_FOLDER="/opt/corda/certificates" \
     DRIVERS_FOLDER="/opt/corda/drivers" \
     CONFIG_FOLDER="/etc/corda" \
@@ -37,6 +39,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \
 VOLUME ["/opt/corda/cordapps"]
 ##PERSISTENCE FOLDER
 VOLUME ["/opt/corda/persistence"]
+##ARTEMIS FOLDER
+VOLUME ["/opt/corda/artemis"]
 ##CERTS FOLDER
 VOLUME ["/opt/corda/certificates"]
 ##OPTIONAL JDBC DRIVERS FOLDER

docker/src/main/kotlin/net.corda.core/ConfigExporter.kt

@@ -51,7 +51,7 @@ class ConfigExporter {
 }
 
 fun Config.parseAsNodeConfigWithFallback(): Validated<NodeConfiguration, Configuration.Validation.Error> {
-    val referenceConfig = ConfigFactory.parseResources("reference.conf")
+    val referenceConfig = ConfigFactory.parseResources("corda-reference.conf")
     val nodeConfig = this
             .withValue("baseDirectory", ConfigValueFactory.fromAnyRef("/opt/corda"))
             .withFallback(referenceConfig)

docs/build.gradle

@@ -0,0 +1,122 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+apply plugin: 'org.jetbrains.dokka'
+apply plugin: 'net.corda.plugins.publish-utils'
+apply plugin: 'maven-publish'
+apply plugin: 'com.jfrog.artifactory'
+
+def internalPackagePrefixes(sourceDirs) {
+    def prefixes = []
+    // Kotlin allows packages to deviate from the directory structure, but let's assume they don't:
+    sourceDirs.collect { sourceDir ->
+        sourceDir.traverse(type: groovy.io.FileType.DIRECTORIES) {
+            if (it.name == 'internal') {
+                prefixes.add sourceDir.toPath().relativize(it.toPath()).toString().replace(File.separator, '.')
+            }
+        }
+    }
+    prefixes
+}
+
+ext {
+    // TODO: Add '../client/jfx/src/main/kotlin' and '../client/mock/src/main/kotlin' if we decide to make them into public API
+    dokkaSourceDirs = files('../core/src/main/kotlin', '../client/rpc/src/main/kotlin', '../finance/workflows/src/main/kotlin', '../finance/contracts/src/main/kotlin', '../client/jackson/src/main/kotlin',
+            '../testing/test-utils/src/main/kotlin', '../testing/node-driver/src/main/kotlin')
+    internalPackagePrefixes = internalPackagePrefixes(dokkaSourceDirs)
+    archivedApiDocsBaseFilename = 'api-docs'
+}
+
+dokka {
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/kotlin")
+}
+
+task dokkaJavadoc(type: org.jetbrains.dokka.gradle.DokkaTask) {
+    outputFormat = "javadoc"
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/javadoc")
+}
+
+[dokka, dokkaJavadoc].collect {
+    it.configure {
+        moduleName = 'corda'
+        processConfigurations = ['compile']
+        sourceDirs = dokkaSourceDirs
+        includes = ['packages.md']
+        jdkVersion = 8
+        externalDocumentationLink {
+            url = new URL("http://fasterxml.github.io/jackson-core/javadoc/2.9/")
+        }
+        externalDocumentationLink {
+            url = new URL("https://docs.oracle.com/javafx/2/api/")
+        }
+        externalDocumentationLink {
+            url = new URL("http://www.bouncycastle.org/docs/docs1.5on/")
+        }
+        internalPackagePrefixes.collect { packagePrefix ->
+            packageOptions {
+                prefix = packagePrefix
+                suppress = true
+            }
+        }
+    }
+}
+
+task apidocs(dependsOn: ['dokka', 'dokkaJavadoc']) {
+    group "Documentation"
+    description "Build API documentation"
+}
+
+task makeHTMLDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-html.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-html.sh"
+    }
+}
+
+task makePDFDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-pdf.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-pdf.sh"
+    }
+}
+
+task makeDocs(dependsOn: ['makeHTMLDocs', 'makePDFDocs'])
+apidocs.shouldRunAfter makeDocs
+
+task archiveApiDocs(type: Tar) {
+    dependsOn apidocs
+    from buildDir
+    include 'html/**'
+    extension 'tgz'
+    compression Compression.GZIP
+}
+
+publishing {
+    publications {
+        if (System.getProperty('publishApiDocs') != null) {
+            archivedApiDocs(MavenPublication) {
+                artifact archiveApiDocs {
+                    artifactId archivedApiDocsBaseFilename
+                }
+            }
+        }
+    }
+}
+
+artifactoryPublish {
+    publications('archivedApiDocs')
+    version = version.replaceAll('-SNAPSHOT', '')
+    publishPom = false
+}
+
+artifactory {
+    publish {
+        contextUrl = artifactory_contextUrl
+        repository {
+            repoKey = 'corda-dependencies-dev'
+            username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+            password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+        }
+    }
+}

experimental/avalanche/build.gradle

@@ -22,4 +22,7 @@ jar.enabled = false
 shadowJar {
     baseName = "avalanche"
 }
-assemble.dependsOn shadowJar
+
+artifacts {
+    archives shadowJar
+}

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Aug 21 10:48:19 BST 2019
-distributionUrl=https\://gradleproxy:gradleproxy@software.r3.com/artifactory/gradle-proxy/gradle-5.4.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://gradleproxy:gradleproxy@software.r3.com/artifactory/gradle-proxy/gradle-5.6.4-all.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

gradlew

@@ -7,7 +7,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#      https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -125,8 +125,8 @@ if $darwin; then
     GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
 fi
 
-# For Cygwin, switch paths to Windows format before running java
-if $cygwin ; then
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
     JAVACMD=`cygpath --unix "$JAVACMD"`

gradlew.bat

@@ -5,7 +5,7 @@
 @rem you may not use this file except in compliance with the License.
 @rem You may obtain a copy of the License at
 @rem
-@rem      http://www.apache.org/licenses/LICENSE-2.0
+@rem      https://www.apache.org/licenses/LICENSE-2.0
 @rem
 @rem Unless required by applicable law or agreed to in writing, software
 @rem distributed under the License is distributed on an "AS IS" BASIS,

java8.gradle

@@ -6,7 +6,7 @@ import static org.gradle.api.JavaVersion.VERSION_1_8
  */
 apply plugin: 'kotlin'
 
-tasks.withType(AbstractCompile) {
+tasks.withType(AbstractCompile).configureEach {
     // This is a bit ugly, but Gradle isn't recognising the KotlinCompile task
     // as it does the built-in JavaCompile task.
     if (it.class.name.startsWith('org.jetbrains.kotlin.gradle.tasks.KotlinCompile')) {
@@ -16,7 +16,7 @@ tasks.withType(AbstractCompile) {
     }
 }
 
-tasks.withType(JavaCompile) {
+tasks.withType(JavaCompile).configureEach {
     sourceCompatibility = VERSION_1_8
     targetCompatibility = VERSION_1_8
 }

jdk8u-deterministic/build.gradle

@@ -37,7 +37,9 @@ def copyJdk = tasks.register('copyJdk', Copy) {
     }
 }
 
-assemble.dependsOn copyJdk
+tasks.named('assemble') {
+    dependsOn copyJdk
+}
 tasks.named('jar', Jar)  {
     enabled = false
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle/NodeLifecycleEventsDistributor.kt

@@ -7,6 +7,8 @@ import net.corda.core.internal.concurrent.openFuture
 import net.corda.core.node.services.CordaServiceCriticalFailureException
 import net.corda.core.utilities.Try
 import net.corda.core.utilities.contextLogger
+import net.corda.nodeapi.internal.persistence.contextDatabase
+import net.corda.nodeapi.internal.persistence.contextDatabaseOrNull
 import java.io.Closeable
 import java.util.Collections.singleton
 import java.util.LinkedList
@@ -93,7 +95,14 @@ class NodeLifecycleEventsDistributor : Closeable {
             log.warn("Not distributing $event as executor been already shutdown. Double close() case?")
             result.set(null)
         } else {
+
+            val passTheDbToTheThread = contextDatabaseOrNull
+
             executor.execute {
+
+                if (passTheDbToTheThread != null)
+                    contextDatabase = passTheDbToTheThread
+
                 val orderedSnapshot = if (event.reversedPriority) snapshot.reversed() else snapshot
                 orderedSnapshot.forEach {
                     log.debug("Distributing event $event to: $it")

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt

@@ -6,6 +6,7 @@ import org.hibernate.Session
 import org.hibernate.Transaction
 import rx.subjects.PublishSubject
 import java.sql.Connection
+import java.sql.SQLException
 import java.util.UUID
 import javax.persistence.EntityManager
 
@@ -87,6 +88,7 @@ class DatabaseTransaction(
         committed = true
     }
 
+    @Throws(SQLException::class)
     fun rollback() {
         if (sessionDelegate.isInitialized() && session.isOpen) {
             session.clear()
@@ -97,16 +99,20 @@ class DatabaseTransaction(
         clearException()
     }
 
+    @Throws(SQLException::class)
     fun close() {
-        if (sessionDelegate.isInitialized() && session.isOpen) {
-            session.close()
+        try {
+            if (sessionDelegate.isInitialized() && session.isOpen) {
+                session.close()
+            }
+            if (database.closeConnection) {
+                connection.close()
+            }
+        } finally {
+            clearException()
+            contextTransactionOrNull = outerTransaction
         }
-        if (database.closeConnection) {
-            connection.close()
-        }
-        clearException()
 
-        contextTransactionOrNull = outerTransaction
         if (outerTransaction == null) {
             synchronized(this) {
                 closed = true

node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt

@@ -10,6 +10,7 @@ import io.netty.handler.proxy.ProxyConnectionEvent
 import io.netty.handler.ssl.SniCompletionEvent
 import io.netty.handler.ssl.SslHandler
 import io.netty.handler.ssl.SslHandshakeCompletionEvent
+import io.netty.handler.ssl.SslHandshakeTimeoutException
 import io.netty.util.ReferenceCountUtil
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.utilities.contextLogger
@@ -295,8 +296,8 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
         // This happens when the peer node is closed during SSL establishment.
         when {
             cause is ClosedChannelException -> logWarnWithMDC("SSL Handshake closed early.")
+            cause is SslHandshakeTimeoutException -> logWarnWithMDC("SSL Handshake timed out")
             // Sadly the exception thrown by Netty wrapper requires that we check the message.
-            cause is SSLException && cause.message == "handshake timed out" -> logWarnWithMDC("SSL Handshake timed out")
             cause is SSLException && (cause.message?.contains("close_notify") == true)
                                                                            -> logWarnWithMDC("Received close_notify during handshake")
             // io.netty.handler.ssl.SslHandler.setHandshakeFailureTransportFailure()

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CustomSerializerCheckpointAdaptor.kt

@@ -0,0 +1,103 @@
+package net.corda.nodeapi.internal.serialization.kryo
+
+import com.esotericsoftware.kryo.Kryo
+import com.esotericsoftware.kryo.Serializer
+import com.esotericsoftware.kryo.io.Input
+import com.esotericsoftware.kryo.io.Output
+import net.corda.core.serialization.CheckpointCustomSerializer
+import net.corda.serialization.internal.amqp.CORDAPP_TYPE
+import java.lang.reflect.Type
+import kotlin.reflect.jvm.javaType
+import kotlin.reflect.jvm.jvmErasure
+
+/**
+ * Adapts CheckpointCustomSerializer for use in Kryo
+ */
+internal class CustomSerializerCheckpointAdaptor<OBJ, PROXY>(private val userSerializer : CheckpointCustomSerializer<OBJ, PROXY>) : Serializer<OBJ>() {
+
+    /**
+     * The class name of the serializer we are adapting.
+     */
+    val serializerName: String = userSerializer.javaClass.name
+
+    /**
+     * The input type of this custom serializer.
+     */
+    val cordappType: Type
+
+    /**
+     * Check we have access to the types specified on the CheckpointCustomSerializer interface.
+     *
+     * Throws UnableToDetermineSerializerTypesException if the types are missing.
+     */
+    init {
+        val types: List<Type> = userSerializer::class
+                .supertypes
+                .filter { it.jvmErasure == CheckpointCustomSerializer::class }
+                .flatMap { it.arguments }
+                .mapNotNull { it.type?.javaType }
+
+        // We are expecting a cordapp type and a proxy type.
+        // We will only use the cordapp type in this class
+        // but we want to check both are present.
+        val typeParameterCount = 2
+        if (types.size != typeParameterCount) {
+            throw UnableToDetermineSerializerTypesException("Unable to determine serializer parent types")
+        }
+        cordappType = types[CORDAPP_TYPE]
+    }
+
+    /**
+     * Serialize obj to the Kryo stream.
+     */
+    override fun write(kryo: Kryo, output: Output, obj: OBJ) {
+
+        fun <T> writeToKryo(obj: T) = kryo.writeClassAndObject(output, obj)
+
+        // Write serializer type
+        writeToKryo(serializerName)
+
+        // Write proxy object
+        writeToKryo(userSerializer.toProxy(obj))
+    }
+
+    /**
+     * Deserialize an object from the Kryo stream.
+     */
+    override fun read(kryo: Kryo, input: Input, type: Class<OBJ>): OBJ {
+
+        @Suppress("UNCHECKED_CAST")
+        fun <T> readFromKryo() = kryo.readClassAndObject(input) as T
+
+        // Check the serializer type
+        checkSerializerType(readFromKryo())
+
+        // Read the proxy object
+        return userSerializer.fromProxy(readFromKryo())
+    }
+
+    /**
+     * Throws a `CustomCheckpointSerializersHaveChangedException` if the serializer type in the kryo stream does not match the serializer
+     * type for this custom serializer.
+     *
+     * @param checkpointSerializerType Serializer type from the Kryo stream
+     */
+    private fun checkSerializerType(checkpointSerializerType: String) {
+        if (checkpointSerializerType != serializerName)
+            throw CustomCheckpointSerializersHaveChangedException("The custom checkpoint serializers have changed while checkpoints exist. " +
+                "Please restore the CorDapps to when this checkpoint was created.")
+    }
+}
+
+/**
+ * Thrown when the input/output types are missing from the custom serializer.
+ */
+class UnableToDetermineSerializerTypesException(message: String) : RuntimeException(message)
+
+/**
+ * Thrown when the custom serializer is found to be reading data from another type of custom serializer.
+ *
+ * This was expected to happen if the user adds or removes CorDapps while checkpoints exist but it turned out that registering serializers
+ * as default made the system reliable.
+ */
+class CustomCheckpointSerializersHaveChangedException(message: String) : RuntimeException(message)

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoCheckpointSerializer.kt

@@ -10,12 +10,14 @@ import com.esotericsoftware.kryo.io.Output
 import com.esotericsoftware.kryo.pool.KryoPool
 import com.esotericsoftware.kryo.serializers.ClosureSerializer
 import net.corda.core.internal.uncheckedCast
+import net.corda.core.serialization.CheckpointCustomSerializer
 import net.corda.core.serialization.ClassWhitelist
 import net.corda.core.serialization.SerializationDefaults
 import net.corda.core.serialization.SerializedBytes
 import net.corda.core.serialization.internal.CheckpointSerializationContext
 import net.corda.core.serialization.internal.CheckpointSerializer
 import net.corda.core.utilities.ByteSequence
+import net.corda.core.utilities.loggerFor
 import net.corda.serialization.internal.AlwaysAcceptEncodingWhitelist
 import net.corda.serialization.internal.ByteBufferInputStream
 import net.corda.serialization.internal.CheckpointSerializationContextImpl
@@ -40,10 +42,10 @@ private object AutoCloseableSerialisationDetector : Serializer<AutoCloseable>()
 }
 
 object KryoCheckpointSerializer : CheckpointSerializer {
-    private val kryoPoolsForContexts = ConcurrentHashMap<Pair<ClassWhitelist, ClassLoader>, KryoPool>()
+    private val kryoPoolsForContexts = ConcurrentHashMap<Triple<ClassWhitelist, ClassLoader, Iterable<CheckpointCustomSerializer<*,*>>>, KryoPool>()
 
     private fun getPool(context: CheckpointSerializationContext): KryoPool {
-        return kryoPoolsForContexts.computeIfAbsent(Pair(context.whitelist, context.deserializationClassLoader)) {
+        return kryoPoolsForContexts.computeIfAbsent(Triple(context.whitelist, context.deserializationClassLoader, context.checkpointCustomSerializers)) {
             KryoPool.Builder {
                 val serializer = Fiber.getFiberSerializer(false) as KryoSerializer
                 val classResolver = CordaClassResolver(context).apply { setKryo(serializer.kryo) }
@@ -56,12 +58,60 @@ object KryoCheckpointSerializer : CheckpointSerializer {
                     addDefaultSerializer(AutoCloseable::class.java, AutoCloseableSerialisationDetector)
                     register(ClosureSerializer.Closure::class.java, CordaClosureSerializer)
                     classLoader = it.second
+
+                    // Add custom serializers
+                    val customSerializers = buildCustomSerializerAdaptors(context)
+                    warnAboutDuplicateSerializers(customSerializers)
+                    val classToSerializer = mapInputClassToCustomSerializer(context.deserializationClassLoader, customSerializers)
+                    addDefaultCustomSerializers(this, classToSerializer)
                 }
             }.build()
 
         }
     }
 
+    /**
+     * Returns a sorted list of CustomSerializerCheckpointAdaptor based on the custom serializers inside context.
+     *
+     * The adaptors are sorted by serializerName which maps to javaClass.name for the serializer class
+     */
+    private fun buildCustomSerializerAdaptors(context: CheckpointSerializationContext) =
+            context.checkpointCustomSerializers.map { CustomSerializerCheckpointAdaptor(it) }.sortedBy { it.serializerName }
+
+    /**
+     * Returns a list of pairs where the first element is the input class of the custom serializer and the second element is the
+     * custom serializer.
+     */
+    private fun mapInputClassToCustomSerializer(classLoader: ClassLoader, customSerializers: Iterable<CustomSerializerCheckpointAdaptor<*, *>>) =
+            customSerializers.map { getInputClassForCustomSerializer(classLoader, it) to it }
+
+    /**
+     * Returns the Class object for the serializers input type.
+     */
+    private fun getInputClassForCustomSerializer(classLoader: ClassLoader, customSerializer: CustomSerializerCheckpointAdaptor<*, *>): Class<*> {
+        val typeNameWithoutGenerics = customSerializer.cordappType.typeName.substringBefore('<')
+        return classLoader.loadClass(typeNameWithoutGenerics)
+    }
+
+    /**
+     * Emit a warning if two or more custom serializers are found for the same input type.
+     */
+    private fun warnAboutDuplicateSerializers(customSerializers: Iterable<CustomSerializerCheckpointAdaptor<*,*>>) =
+            customSerializers
+                    .groupBy({ it.cordappType }, { it.serializerName })
+                    .filter { (_, serializerNames) -> serializerNames.distinct().size > 1 }
+                    .forEach { (inputType, serializerNames) -> loggerFor<KryoCheckpointSerializer>().warn("Duplicate custom checkpoint serializer for type $inputType. Serializers: ${serializerNames.joinToString(", ")}") }
+
+    /**
+     * Register all custom serializers as default, this class + subclass, registrations.
+     *
+     * Serializers registered before this will take priority. This needs to run after registrations we want to keep otherwise it may
+     * replace them.
+     */
+    private fun addDefaultCustomSerializers(kryo: Kryo, classToSerializer: Iterable<Pair<Class<*>, CustomSerializerCheckpointAdaptor<*, *>>>) =
+            classToSerializer
+                    .forEach { (clazz, customSerializer) -> kryo.addDefaultSerializer(clazz, customSerializer) }
+
     private fun <T : Any> CheckpointSerializationContext.kryo(task: Kryo.() -> T): T {
         return getPool(this).run { kryo ->
             kryo.context.ensureCapacity(properties.size)

node/build.gradle

@@ -20,6 +20,11 @@ ext {
     jolokia_version = constants.getProperty('jolokiaAgentVersion')
 }
 
+evaluationDependsOn(':core-deterministic')
+evaluationDependsOn(':serialization-deterministic')
+evaluationDependsOn(':serialization-djvm:deserializers')
+evaluationDependsOn(':node:djvm')
+
 //noinspection GroovyAssignabilityCheck
 configurations {
     integrationTestCompile.extendsFrom testCompile
@@ -191,7 +196,8 @@ dependencies {
     // Integration test helpers
     integrationTestCompile "junit:junit:$junit_version"
     integrationTestCompile "org.assertj:assertj-core:${assertj_version}"
-    
+    integrationTestCompile "org.apache.qpid:qpid-jms-client:${protonj_version}"
+
     // BFT-Smart dependencies
     compile 'com.github.bft-smart:library:master-v1.1-beta-g6215ec8-87'
     compile 'commons-codec:commons-codec:1.13'
@@ -242,12 +248,12 @@ dependencies {
     testCompile project(':testing:cordapps:dbfailure:dbfworkflows')
 }
 
-tasks.withType(JavaCompile) {
+tasks.withType(JavaCompile).configureEach {
     // Resolves a Gradle warning about not scanning for pre-processors.
     options.compilerArgs << '-proc:none'
 }
 
-tasks.withType(Test) {
+tasks.withType(Test).configureEach {
     if (JavaVersion.current() == JavaVersion.VERSION_11) {
         jvmArgs '-Djdk.attach.allowAttachSelf=true'
     }
@@ -255,13 +261,13 @@ tasks.withType(Test) {
     systemProperty 'deterministic-sources.path', configurations.deterministic.asPath
 }
 
-task integrationTest(type: Test) {
+tasks.register('integrationTest', Test) {
     testClassesDirs = sourceSets.integrationTest.output.classesDirs
     classpath = sourceSets.integrationTest.runtimeClasspath
     maxParallelForks = (System.env.CORDA_NODE_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_NODE_INT_TESTING_FORKS".toInteger()
 }
 
-task slowIntegrationTest(type: Test) {
+tasks.register('slowIntegrationTest', Test) {
     testClassesDirs = sourceSets.slowIntegrationTest.output.classesDirs
     classpath = sourceSets.slowIntegrationTest.runtimeClasspath
     maxParallelForks = 1
@@ -319,7 +325,7 @@ publish {
     name jar.baseName
 }
 
-test {
+tasks.named('test', Test) {
     maxHeapSize = "3g"
     maxParallelForks = (System.env.CORDA_NODE_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_NODE_TESTING_FORKS".toInteger()
 }

node/capsule/build.gradle

@@ -39,9 +39,9 @@ capsule {
 def nodeProject = project(':node')
 
 task buildCordaJAR(type: FatCapsule, dependsOn: [
-        nodeProject.tasks.jar,
-        project(':core-deterministic').tasks.assemble,
-        project(':serialization-deterministic').tasks.assemble
+        nodeProject.tasks.named('jar'),
+        project(':core-deterministic').tasks.named('assemble'),
+        project(':serialization-deterministic').tasks.named('assemble')
     ]) {
     applicationClass 'net.corda.node.Corda'
     archiveBaseName = 'corda'
@@ -51,7 +51,7 @@ task buildCordaJAR(type: FatCapsule, dependsOn: [
     applicationSource = files(
         nodeProject.configurations.runtimeClasspath,
         nodeProject.tasks.jar,
-        nodeProject.buildDir.toString() + '/resources/main/reference.conf',
+        nodeProject.buildDir.toString() + '/resources/main/corda-reference.conf',
         "$rootDir/config/dev/log4j2.xml",
         'NOTICE' // Copy CDDL notice
     )
@@ -119,9 +119,8 @@ task buildCordaJAR(type: FatCapsule, dependsOn: [
     }
 }
 
-assemble.dependsOn buildCordaJAR
-
 artifacts {
+    archives buildCordaJAR
     runtimeArtifacts buildCordaJAR
     publish buildCordaJAR {
         classifier ''

node/capsule/src/main/java/CordaCaplet.java

@@ -37,7 +37,7 @@ public class CordaCaplet extends Capsule {
         File configFile = (config == null) ? new File(baseDir, "node.conf") : new File(config);
         try {
             ConfigParseOptions parseOptions = ConfigParseOptions.defaults().setAllowMissing(false);
-            Config defaultConfig = ConfigFactory.parseResources("reference.conf", parseOptions);
+            Config defaultConfig = ConfigFactory.parseResources("corda-reference.conf", parseOptions);
             Config baseDirectoryConfig = ConfigFactory.parseMap(Collections.singletonMap("baseDirectory", baseDir));
             Config nodeConfig = ConfigFactory.parseFile(configFile, parseOptions);
             return baseDirectoryConfig.withFallback(nodeConfig).withFallback(defaultConfig).resolve();

node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt

@@ -1,6 +1,7 @@
 package net.corda.node.flows
 
 import co.paralleluniverse.fibers.Suspendable
+import net.corda.core.CordaException
 import net.corda.core.flows.*
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
@@ -16,7 +17,6 @@ import net.corda.testing.driver.DriverDSL
 import net.corda.testing.driver.DriverParameters
 import net.corda.testing.driver.NodeParameters
 import net.corda.testing.driver.driver
-import net.corda.testing.node.internal.ListenProcessDeathException
 import net.corda.testing.node.internal.assertUncompletedCheckpoints
 import net.corda.testing.node.internal.enclosedCordapp
 import org.assertj.core.api.Assertions.assertThat
@@ -78,7 +78,7 @@ class FlowCheckpointVersionNodeStartupCheckTest {
     private fun DriverDSL.assertBobFailsToStartWithLogMessage(logMessage: String) {
         assertUncompletedCheckpoints(BOB_NAME, 1)
 
-        assertFailsWith(ListenProcessDeathException::class) {
+        assertFailsWith(CordaException::class) {
             startNode(NodeParameters(
                     providedName = BOB_NAME,
                     customOverrides = mapOf("devMode" to false)

node/src/integration-test-slow/kotlin/net/corda/node/services/distributed/DistributedServiceTests.kt

@@ -29,6 +29,8 @@ import org.junit.Ignore
 import org.junit.Test
 import rx.Observable
 import java.util.*
+import kotlin.test.assertEquals
+import kotlin.test.assertTrue
 
 class DistributedServiceTests {
     private lateinit var alice: NodeHandle
@@ -157,9 +159,9 @@ class DistributedServiceTests {
 
         // The distribution of requests should be very close to sg like 16/17/17 as by default artemis does round robin
         println("Notarisation distribution: $notarisationsPerNotary")
-        require(notarisationsPerNotary.size == 3)
+        assertEquals(3, notarisationsPerNotary.size)
         // We allow some leeway for artemis as it doesn't always produce perfect distribution
-        require(notarisationsPerNotary.values.all { it > 10 })
+        assertTrue { notarisationsPerNotary.values.all { it > 10 } }
     }
 
     private fun issueCash(amount: Amount<Currency>) {

node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt

@@ -1,355 +0,0 @@
-package net.corda.node.services.rpc
-
-import net.corda.client.rpc.CordaRPCClient
-import net.corda.client.rpc.CordaRPCClientConfiguration
-import net.corda.client.rpc.GracefulReconnect
-import net.corda.client.rpc.internal.ReconnectingCordaRPCOps
-import net.corda.client.rpc.notUsed
-import net.corda.core.contracts.Amount
-import net.corda.core.flows.StateMachineRunId
-import net.corda.core.internal.concurrent.transpose
-import net.corda.core.messaging.StateMachineUpdate
-import net.corda.core.node.services.Vault
-import net.corda.core.node.services.vault.PageSpecification
-import net.corda.core.node.services.vault.QueryCriteria
-import net.corda.core.node.services.vault.builder
-import net.corda.core.utilities.NetworkHostAndPort
-import net.corda.core.utilities.OpaqueBytes
-import net.corda.core.utilities.contextLogger
-import net.corda.core.utilities.getOrThrow
-import net.corda.core.utilities.seconds
-import net.corda.finance.contracts.asset.Cash
-import net.corda.finance.flows.CashIssueAndPaymentFlow
-import net.corda.finance.schemas.CashSchemaV1
-import net.corda.node.services.Permissions
-import net.corda.node.services.rpc.RpcReconnectTests.Companion.NUMBER_OF_FLOWS_TO_RUN
-import net.corda.testing.core.DUMMY_BANK_A_NAME
-import net.corda.testing.core.DUMMY_BANK_B_NAME
-import net.corda.testing.driver.DriverParameters
-import net.corda.testing.driver.NodeHandle
-import net.corda.testing.driver.OutOfProcess
-import net.corda.testing.driver.driver
-import net.corda.testing.driver.internal.OutOfProcessImpl
-import net.corda.testing.driver.internal.incrementalPortAllocation
-import net.corda.testing.node.User
-import net.corda.testing.node.internal.FINANCE_CORDAPPS
-import org.assertj.core.api.Assertions.assertThat
-import org.junit.Test
-import java.util.*
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.TimeUnit
-import java.util.concurrent.atomic.AtomicInteger
-import kotlin.concurrent.thread
-import kotlin.math.absoluteValue
-import kotlin.math.max
-import kotlin.test.assertEquals
-import kotlin.test.assertTrue
-import kotlin.test.currentStackTrace
-
-/**
- * This is a stress test for the rpc reconnection logic, which triggers failures in a probabilistic way.
- *
- * You can adjust the variable [NUMBER_OF_FLOWS_TO_RUN] to adjust the number of flows to run and the duration of the test.
- */
-class RpcReconnectTests {
-
-    companion object {
-        // this many flows take ~5 minutes
-        const val NUMBER_OF_FLOWS_TO_RUN = 100
-
-        private val log = contextLogger()
-    }
-
-    private val portAllocator = incrementalPortAllocation()
-
-    private lateinit var proxy: RandomFailingProxy
-    private lateinit var node: NodeHandle
-    private lateinit var currentAddressPair: AddressPair
-
-    /**
-     * This test showcases and stress tests the demo [ReconnectingCordaRPCOps].
-     *
-     * Note that during node failure events can be lost and starting flows can become unreliable.
-     * The only available way to retry failed flows is to attempt a "logical retry" which is also showcased.
-     *
-     * This test runs flows in a loop and in the background kills the node or restarts it.
-     * Also the RPC connection is made through a proxy that introduces random latencies and is also periodically killed.
-     */
-    @Suppress("ComplexMethod")
-    @Test(timeout=420_000)
-	fun `test that the RPC client is able to reconnect and proceed after node failure, restart, or connection reset`() {
-        val nodeRunningTime = { Random().nextInt(12000) + 8000 }
-
-        val demoUser = User("demo", "demo", setOf(Permissions.all()))
-
-        // When this reaches 0 - the test will end.
-        val flowsCountdownLatch = CountDownLatch(NUMBER_OF_FLOWS_TO_RUN)
-        // These are the expected progress steps for the CashIssueAndPayFlow.
-        val expectedProgress = listOf(
-                "Starting",
-                "Issuing cash",
-                "Generating transaction",
-                "Signing transaction",
-                "Finalising transaction",
-                "Broadcasting transaction to participants",
-                "Paying recipient",
-                "Generating anonymous identities",
-                "Generating transaction",
-                "Signing transaction",
-                "Finalising transaction",
-                "Requesting signature by notary service",
-                "Requesting signature by Notary service",
-                "Validating response from Notary service",
-                "Broadcasting transaction to participants",
-                "Done"
-        )
-
-        driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS, startNodesInProcess = false, inMemoryDB = false)) {
-            fun startBankA(address: NetworkHostAndPort) = startNode(providedName = DUMMY_BANK_A_NAME, rpcUsers = listOf(demoUser), customOverrides = mapOf("rpcSettings.address" to address.toString()))
-            fun startProxy(addressPair: AddressPair) = RandomFailingProxy(serverPort = addressPair.proxyAddress.port, remotePort = addressPair.nodeAddress.port).start()
-
-            val addresses = (1..2).map { getRandomAddressPair() }
-            currentAddressPair = addresses[0]
-
-            proxy = startProxy(currentAddressPair)
-            val (bankA, bankB) = listOf(
-                    startBankA(currentAddressPair.nodeAddress),
-                    startNode(providedName = DUMMY_BANK_B_NAME, rpcUsers = listOf(demoUser))
-            ).transpose().getOrThrow()
-            node = bankA
-
-            val notary = defaultNotaryIdentity
-            val baseAmount = Amount.parseCurrency("0 USD")
-            val issuerRef = OpaqueBytes.of(0x01)
-
-            var numDisconnects = 0
-            var numReconnects = 0
-            val maxStackOccurrences = AtomicInteger()
-
-            val addressesForRpc = addresses.map { it.proxyAddress }
-            // DOCSTART rpcReconnectingRPC
-            val onReconnect = {
-                numReconnects++
-                // We only expect to see a single reconnectOnError in the stack trace.  Otherwise we're in danger of stack overflow recursion
-                maxStackOccurrences.set(max(maxStackOccurrences.get(), currentStackTrace().count { it.methodName == "reconnectOnError" }))
-                Unit
-            }
-            val reconnect = GracefulReconnect(onDisconnect = { numDisconnects++ }, onReconnect = onReconnect)
-            val config = CordaRPCClientConfiguration.DEFAULT.copy(
-                    connectionRetryInterval = 1.seconds,
-                    connectionRetryIntervalMultiplier = 1.0
-            )
-            val client = CordaRPCClient(addressesForRpc, configuration = config)
-            val bankAReconnectingRPCConnection = client.start(demoUser.username, demoUser.password, gracefulReconnect = reconnect)
-            val bankAReconnectingRpc = bankAReconnectingRPCConnection.proxy as ReconnectingCordaRPCOps
-            // DOCEND rpcReconnectingRPC
-
-            // Observe the vault and collect the observations.
-            val vaultEvents = Collections.synchronizedList(mutableListOf<Vault.Update<Cash.State>>())
-            // DOCSTART rpcReconnectingRPCVaultTracking
-            val vaultFeed = bankAReconnectingRpc.vaultTrackByWithPagingSpec(
-                    Cash.State::class.java,
-                    QueryCriteria.VaultQueryCriteria(),
-                    PageSpecification(1, 1))
-            val vaultSubscription = vaultFeed.updates.subscribe { update: Vault.Update<Cash.State> ->
-                log.info("vault update produced ${update.produced.map { it.state.data.amount }} consumed ${update.consumed.map { it.ref }}")
-                vaultEvents.add(update)
-            }
-            // DOCEND rpcReconnectingRPCVaultTracking
-
-            // Observe the stateMachine and collect the observations.
-            val stateMachineEvents = Collections.synchronizedList(mutableListOf<StateMachineUpdate>())
-            val stateMachineSubscription = bankAReconnectingRpc.stateMachinesFeed().updates.subscribe { update ->
-                log.info(update.toString())
-                stateMachineEvents.add(update)
-            }
-
-            // While the flows are running, randomly apply a different failure scenario.
-            val nrRestarts = AtomicInteger()
-            thread(name = "Node killer") {
-                while (true) {
-                    if (flowsCountdownLatch.count == 0L) break
-
-                    // Let the node run for a random time interval.
-                    nodeRunningTime().also { ms ->
-                        log.info("Running node for ${ms / 1000} s.")
-                        Thread.sleep(ms.toLong())
-                    }
-
-                    if (flowsCountdownLatch.count == 0L) break
-                    when (Random().nextInt().rem(7).absoluteValue) {
-                        0 -> {
-                            log.info("Forcefully killing node and proxy.")
-                            (node as OutOfProcessImpl).onStopCallback()
-                            (node as OutOfProcess).process.destroyForcibly()
-                            proxy.stop()
-                            node = startBankA(currentAddressPair.nodeAddress).get()
-                            proxy.start()
-                        }
-                        1 -> {
-                            log.info("Forcefully killing node.")
-                            (node as OutOfProcessImpl).onStopCallback()
-                            (node as OutOfProcess).process.destroyForcibly()
-                            node = startBankA(currentAddressPair.nodeAddress).get()
-                        }
-                        2 -> {
-                            log.info("Shutting down node.")
-                            node.stop()
-                            proxy.stop()
-                            node = startBankA(currentAddressPair.nodeAddress).get()
-                            proxy.start()
-                        }
-                        3, 4 -> {
-                            log.info("Killing proxy.")
-                            proxy.stop()
-                            Thread.sleep(Random().nextInt(5000).toLong())
-                            proxy.start()
-                        }
-                        5 -> {
-                            log.info("Dropping connection.")
-                            proxy.failConnection()
-                        }
-                        6 -> {
-                            log.info("Performing failover to a different node")
-                            node.stop()
-                            proxy.stop()
-                            currentAddressPair = (addresses - currentAddressPair).first()
-                            node = startBankA(currentAddressPair.nodeAddress).get()
-                            proxy = startProxy(currentAddressPair)
-                        }
-                    }
-                    nrRestarts.incrementAndGet()
-                }
-            }
-
-            // Start nrOfFlowsToRun and provide a logical retry function that checks the vault.
-            val flowProgressEvents = mutableMapOf<StateMachineRunId, MutableList<String>>()
-            for (amount in (1..NUMBER_OF_FLOWS_TO_RUN)) {
-                // DOCSTART rpcReconnectingRPCFlowStarting
-                bankAReconnectingRpc.runFlowWithLogicalRetry(
-                        runFlow = { rpc ->
-                            log.info("Starting CashIssueAndPaymentFlow for $amount")
-                            val flowHandle = rpc.startTrackedFlowDynamic(
-                                    CashIssueAndPaymentFlow::class.java,
-                                    baseAmount.plus(Amount.parseCurrency("$amount USD")),
-                                    issuerRef,
-                                    bankB.nodeInfo.legalIdentities.first(),
-                                    false,
-                                    notary
-                            )
-                            val flowId = flowHandle.id
-                            log.info("Started flow $amount with flowId: $flowId")
-                            flowProgressEvents.addEvent(flowId, null)
-
-                            flowHandle.stepsTreeFeed?.updates?.notUsed()
-                            flowHandle.stepsTreeIndexFeed?.updates?.notUsed()
-                            // No reconnecting possible.
-                            flowHandle.progress.subscribe(
-                                    { prog ->
-                                        flowProgressEvents.addEvent(flowId, prog)
-                                        log.info("Progress $flowId : $prog")
-                                    },
-                                    { error ->
-                                        log.error("Error thrown in the flow progress observer", error)
-                                    })
-                            flowHandle.id
-                        },
-                        hasFlowStarted = { rpc ->
-                            // Query for a state that is the result of this flow.
-                            val criteria = QueryCriteria.VaultCustomQueryCriteria(builder { CashSchemaV1.PersistentCashState::pennies.equal(amount.toLong() * 100) }, status = Vault.StateStatus.ALL)
-                            val results = rpc.vaultQueryByCriteria(criteria, Cash.State::class.java)
-                            log.info("$amount - Found states ${results.states}")
-                            // The flow has completed if a state is found
-                            results.states.isNotEmpty()
-                        },
-                        onFlowConfirmed = {
-                            flowsCountdownLatch.countDown()
-                            log.info("Flow started for $amount. Remaining flows: ${flowsCountdownLatch.count}")
-                        }
-                )
-                // DOCEND rpcReconnectingRPCFlowStarting
-
-                Thread.sleep(Random().nextInt(250).toLong())
-            }
-
-            log.info("Started all flows")
-
-            // Wait until all flows have been started.
-            val flowsConfirmed = flowsCountdownLatch.await(10, TimeUnit.MINUTES)
-
-            if (flowsConfirmed) {
-                log.info("Confirmed all flows have started.")
-            } else {
-                log.info("Timed out waiting for confirmation that all flows have started. Remaining flows: ${flowsCountdownLatch.count}")
-            }
-
-
-            // Wait for all events to come in and flows to finish.
-            Thread.sleep(4000)
-
-            val nrFailures = nrRestarts.get()
-            log.info("Checking results after $nrFailures restarts.")
-
-            // We should get one disconnect and one reconnect for each failure
-            assertThat(numDisconnects).isEqualTo(numReconnects)
-            assertThat(numReconnects).isLessThanOrEqualTo(nrFailures)
-            assertThat(maxStackOccurrences.get()).isLessThan(2)
-
-            // Query the vault and check that states were created for all flows.
-            fun readCashStates() = bankAReconnectingRpc
-                    .vaultQueryByWithPagingSpec(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(status = Vault.StateStatus.CONSUMED), PageSpecification(1, 10000))
-                    .states
-
-            var allCashStates = readCashStates()
-            var nrRetries = 0
-
-            // It might be necessary to wait more for all events to arrive when the node is slow.
-            while (allCashStates.size < NUMBER_OF_FLOWS_TO_RUN && nrRetries++ < 50) {
-                Thread.sleep(2000)
-                allCashStates = readCashStates()
-            }
-
-            val allCash = allCashStates.map { it.state.data.amount.quantity }.toSet()
-            val missingCash = (1..NUMBER_OF_FLOWS_TO_RUN).filterNot { allCash.contains(it.toLong() * 100) }
-            log.info("Missing cash states: $missingCash")
-
-            assertEquals(NUMBER_OF_FLOWS_TO_RUN, allCashStates.size, "Not all flows were executed successfully")
-
-            // The progress status for each flow can only miss the last events, because the node might have been killed.
-            val missingProgressEvents = flowProgressEvents.filterValues { expectedProgress.subList(0, it.size) != it }
-            assertTrue(missingProgressEvents.isEmpty(), "The flow progress tracker is missing events: $missingProgressEvents")
-
-            // DOCSTART missingVaultEvents
-            // Check that enough vault events were received.
-            // This check is fuzzy because events can go missing during node restarts.
-            // Ideally there should be nrOfFlowsToRun events receive but some might get lost for each restart.
-            assertThat(vaultEvents!!.size + nrFailures * 3).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN)
-            // DOCEND missingVaultEvents
-
-            // Check that no flow was triggered twice.
-            val duplicates = allCashStates.groupBy { it.state.data.amount }.filterValues { it.size > 1 }
-            assertTrue(duplicates.isEmpty(), "${duplicates.size} flows were retried illegally.")
-
-            log.info("State machine events seen: ${stateMachineEvents!!.size}")
-            // State machine events are very likely to get lost more often because they seem to be sent with a delay.
-            assertThat(stateMachineEvents.count { it is StateMachineUpdate.Added }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3)
-            assertThat(stateMachineEvents.count { it is StateMachineUpdate.Removed }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3)
-
-            // Stop the observers.
-            vaultSubscription.unsubscribe()
-            stateMachineSubscription.unsubscribe()
-            bankAReconnectingRPCConnection.close()
-        }
-
-        proxy.close()
-    }
-
-    @Synchronized
-    fun MutableMap<StateMachineRunId, MutableList<String>>.addEvent(id: StateMachineRunId, progress: String?): Boolean {
-        return getOrPut(id) { mutableListOf() }.let { if (progress != null) it.add(progress) else false }
-    }
-    private fun getRandomAddressPair() = AddressPair(getRandomAddress(), getRandomAddress())
-    private fun getRandomAddress() = NetworkHostAndPort("localhost", portAllocator.nextPort())
-
-    data class AddressPair(val proxyAddress: NetworkHostAndPort, val nodeAddress: NetworkHostAndPort)
-}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt

@@ -0,0 +1,288 @@
+package net.corda.node.services.statemachine
+
+import co.paralleluniverse.fibers.Suspendable
+import net.corda.core.flows.FlowLogic
+import net.corda.core.flows.FlowSession
+import net.corda.core.flows.HospitalizeFlowException
+import net.corda.core.flows.InitiatedBy
+import net.corda.core.flows.InitiatingFlow
+import net.corda.core.flows.StartableByRPC
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.identity.Party
+import net.corda.core.internal.list
+import net.corda.core.internal.readAllLines
+import net.corda.core.messaging.CordaRPCOps
+import net.corda.core.messaging.startFlow
+import net.corda.core.node.AppServiceHub
+import net.corda.core.node.services.CordaService
+import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.SingletonSerializeAsToken
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.seconds
+import net.corda.core.utilities.unwrap
+import net.corda.node.services.Permissions
+import net.corda.testing.core.DUMMY_NOTARY_NAME
+import net.corda.testing.driver.DriverDSL
+import net.corda.testing.driver.DriverParameters
+import net.corda.testing.driver.NodeHandle
+import net.corda.testing.driver.NodeParameters
+import net.corda.testing.driver.driver
+import net.corda.testing.driver.internal.OutOfProcessImpl
+import net.corda.testing.node.NotarySpec
+import net.corda.testing.node.TestCordapp
+import net.corda.testing.node.User
+import net.corda.testing.node.internal.InternalDriverDSL
+import org.jboss.byteman.agent.submit.ScriptText
+import org.jboss.byteman.agent.submit.Submit
+import org.junit.Before
+import java.time.Duration
+import java.util.concurrent.TimeUnit
+import kotlin.test.assertEquals
+
+abstract class StateMachineErrorHandlingTest {
+
+    val rpcUser = User("user1", "test", permissions = setOf(Permissions.all()))
+    var counter = 0
+
+    @Before
+    fun setup() {
+        counter = 0
+    }
+
+    internal fun startDriver(notarySpec: NotarySpec = NotarySpec(DUMMY_NOTARY_NAME), dsl: DriverDSL.() -> Unit) {
+        driver(
+            DriverParameters(
+                notarySpecs = listOf(notarySpec),
+                startNodesInProcess = false,
+                inMemoryDB = false,
+                systemProperties = mapOf("co.paralleluniverse.fibers.verifyInstrumentation" to "true")
+            )
+        ) {
+            dsl()
+        }
+    }
+
+    internal fun DriverDSL.createBytemanNode(
+        providedName: CordaX500Name,
+        additionalCordapps: Collection<TestCordapp> = emptyList()
+    ): Pair<NodeHandle, Int> {
+        val port = nextPort()
+        val nodeHandle = (this as InternalDriverDSL).startNode(
+            NodeParameters(
+                providedName = providedName,
+                rpcUsers = listOf(rpcUser),
+                additionalCordapps = additionalCordapps
+            ),
+            bytemanPort = port
+        ).getOrThrow()
+        return nodeHandle to port
+    }
+
+    internal fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection<TestCordapp> = emptyList()): NodeHandle {
+        return startNode(
+            NodeParameters(
+                providedName = providedName,
+                rpcUsers = listOf(rpcUser),
+                additionalCordapps = additionalCordapps
+            )
+        ).getOrThrow()
+    }
+
+    internal fun submitBytemanRules(rules: String, port: Int) {
+        val submit = Submit("localhost", port)
+        submit.addScripts(listOf(ScriptText("Test script", rules)))
+    }
+
+    internal fun getBytemanOutput(nodeHandle: NodeHandle): List<String> {
+        return nodeHandle.baseDirectory
+            .list()
+            .first { it.toString().contains("net.corda.node.Corda") && it.toString().contains("stdout.log") }
+            .readAllLines()
+    }
+
+    internal fun OutOfProcessImpl.stop(timeout: Duration): Boolean {
+        return process.run {
+            destroy()
+            waitFor(timeout.seconds, TimeUnit.SECONDS)
+        }.also { onStopCallback() }
+    }
+
+    @Suppress("LongParameterList")
+    internal fun CordaRPCOps.assertHospitalCounts(
+        discharged: Int = 0,
+        observation: Int = 0,
+        propagated: Int = 0,
+        dischargedRetry: Int = 0,
+        observationRetry: Int = 0,
+        propagatedRetry: Int = 0
+    ) {
+        val counts = startFlow(StateMachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.getOrThrow(20.seconds)
+        assertEquals(discharged, counts.discharged)
+        assertEquals(observation, counts.observation)
+        assertEquals(propagated, counts.propagated)
+        assertEquals(dischargedRetry, counts.dischargeRetry)
+        assertEquals(observationRetry, counts.observationRetry)
+        assertEquals(propagatedRetry, counts.propagatedRetry)
+    }
+
+    internal fun CordaRPCOps.assertHospitalCountsAllZero() = assertHospitalCounts()
+
+    internal fun CordaRPCOps.assertNumberOfCheckpoints(
+        runnable: Int = 0,
+        failed: Int = 0,
+        completed: Int = 0,
+        hospitalized: Int = 0
+    ) {
+        val counts = startFlow(StateMachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds)
+        assertEquals(runnable, counts.runnable, "There should be $runnable runnable checkpoints")
+        assertEquals(failed, counts.failed, "There should be $failed failed checkpoints")
+        assertEquals(completed, counts.completed, "There should be $completed completed checkpoints")
+        assertEquals(hospitalized, counts.hospitalized, "There should be $hospitalized hospitalized checkpoints")
+    }
+
+    internal fun CordaRPCOps.assertNumberOfCheckpointsAllZero() = assertNumberOfCheckpoints()
+
+    @StartableByRPC
+    @InitiatingFlow
+    class SendAMessageFlow(private val party: Party) : FlowLogic<String>() {
+        @Suspendable
+        override fun call(): String {
+            val session = initiateFlow(party)
+            session.send("hello there")
+            logger.info("Finished my flow")
+            return "Finished executing test flow - ${this.runId}"
+        }
+    }
+
+    @InitiatedBy(SendAMessageFlow::class)
+    class SendAMessageResponder(private val session: FlowSession) : FlowLogic<Unit>() {
+        @Suspendable
+        override fun call() {
+            session.receive<String>().unwrap { it }
+            logger.info("Finished my flow")
+        }
+    }
+
+    @StartableByRPC
+    class ThrowAnErrorFlow : FlowLogic<String>() {
+        @Suspendable
+        override fun call(): String {
+            throwException()
+            return "cant get here"
+        }
+
+        private fun throwException() {
+            logger.info("Throwing exception in flow")
+            throw IllegalStateException("throwing exception in flow")
+        }
+    }
+
+    @StartableByRPC
+    class ThrowAHospitalizeErrorFlow : FlowLogic<String>() {
+        @Suspendable
+        override fun call(): String {
+            throwException()
+            return "cant get here"
+        }
+
+        private fun throwException() {
+            logger.info("Throwing exception in flow")
+            throw HospitalizeFlowException("throwing exception in flow")
+        }
+    }
+
+    @StartableByRPC
+    class GetNumberOfCheckpointsFlow : FlowLogic<NumberOfCheckpoints>() {
+        override fun call() = NumberOfCheckpoints(
+            runnable = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.RUNNABLE),
+            failed = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.FAILED),
+            completed = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.COMPLETED),
+            hospitalized = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.HOSPITALIZED)
+        )
+
+        private fun getNumberOfCheckpointsWithStatus(status: Checkpoint.FlowStatus): Int {
+            return serviceHub.jdbcSession()
+                .prepareStatement("select count(*) from node_checkpoints where status = ? and flow_id != ?")
+                .apply {
+                    setInt(1, status.ordinal)
+                    setString(2, runId.uuid.toString())
+                }
+                .use { ps ->
+                    ps.executeQuery().use { rs ->
+                        rs.next()
+                        rs.getLong(1)
+                    }
+                }.toInt()
+        }
+    }
+
+    @CordaSerializable
+    data class NumberOfCheckpoints(
+        val runnable: Int = 0,
+        val failed: Int = 0,
+        val completed: Int = 0,
+        val hospitalized: Int = 0
+    )
+
+    // Internal use for testing only!!
+    @StartableByRPC
+    class GetHospitalCountersFlow : FlowLogic<HospitalCounts>() {
+        override fun call(): HospitalCounts =
+            HospitalCounts(
+                serviceHub.cordaService(HospitalCounter::class.java).dischargedCounter,
+                serviceHub.cordaService(HospitalCounter::class.java).observationCounter,
+                serviceHub.cordaService(HospitalCounter::class.java).propagatedCounter,
+                serviceHub.cordaService(HospitalCounter::class.java).dischargeRetryCounter,
+                serviceHub.cordaService(HospitalCounter::class.java).observationRetryCounter,
+                serviceHub.cordaService(HospitalCounter::class.java).propagatedRetryCounter
+            )
+    }
+
+    @CordaSerializable
+    data class HospitalCounts(
+        val discharged: Int,
+        val observation: Int,
+        val propagated: Int,
+        val dischargeRetry: Int,
+        val observationRetry: Int,
+        val propagatedRetry: Int
+    )
+
+    @Suppress("UNUSED_PARAMETER")
+    @CordaService
+    class HospitalCounter(services: AppServiceHub) : SingletonSerializeAsToken() {
+        var dischargedCounter: Int = 0
+        var observationCounter: Int = 0
+        var propagatedCounter: Int = 0
+        var dischargeRetryCounter: Int = 0
+        var observationRetryCounter: Int = 0
+        var propagatedRetryCounter: Int = 0
+
+        init {
+            StaffedFlowHospital.onFlowDischarged.add { _, _ ->
+                dischargedCounter++
+            }
+            StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ ->
+                observationCounter++
+            }
+            StaffedFlowHospital.onFlowErrorPropagated.add { _, _ ->
+                propagatedCounter++
+            }
+            StaffedFlowHospital.onFlowResuscitated.add { _, _, outcome ->
+                when (outcome) {
+                    StaffedFlowHospital.Outcome.DISCHARGE -> dischargeRetryCounter++
+                    StaffedFlowHospital.Outcome.OVERNIGHT_OBSERVATION -> observationRetryCounter++
+                    StaffedFlowHospital.Outcome.UNTREATABLE -> propagatedRetryCounter++
+                }
+            }
+        }
+    }
+
+    internal val actionExecutorClassName: String by lazy {
+        Class.forName("net.corda.node.services.statemachine.ActionExecutorImpl").name
+    }
+
+    internal val stateMachineManagerClassName: String by lazy {
+        Class.forName("net.corda.node.services.statemachine.SingleThreadedStateMachineManager").name
+    }
+}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt

@@ -1,6 +1,5 @@
 package net.corda.node.services.statemachine
 
-import net.corda.client.rpc.CordaRPCClient
 import net.corda.core.flows.ReceiveFinalityFlow
 import net.corda.core.internal.ResolveTransactionsFlow
 import net.corda.core.messaging.startFlow
@@ -22,7 +21,7 @@ import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 
 @Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
-class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
+class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() {
 
     /**
      * Throws an exception when recoding a transaction inside of [ReceiveFinalityFlow] on the responding
@@ -33,10 +32,10 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
      * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its
      * send to the responding node and the responding node successfully received it.
      */
-    @Test(timeout=300_000)
-	fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() {
+    @Test(timeout = 300_000)
+    fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() {
         startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) {
-            val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
             val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS)
 
             // could not get rule for FinalityDoctor + observation counter to work
@@ -67,14 +66,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-            val charlieClient =
-                CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(
+            alice.rpc.startFlow(
                 ::CashIssueAndPaymentFlow,
                 500.DOLLARS,
                 OpaqueBytes.of(0x01),
@@ -83,15 +77,11 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 defaultNotaryIdentity
             ).returnValue.getOrThrow(30.seconds)
 
-            val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(0, discharge)
-            assertEquals(1, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            assertEquals(1, charlieClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-            // 1 ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow
-            assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            charlie.rpc.assertHospitalCounts(observation = 1)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(1, charlie.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -104,10 +94,10 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
      * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its
      * send to the responding node and the responding node successfully received it.
      */
-    @Test(timeout=300_000)
-	fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() {
+    @Test(timeout = 300_000)
+    fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() {
         startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) {
-            val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
             val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS)
 
             // could not get rule for FinalityDoctor + observation counter to work
@@ -138,14 +128,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-            val charlieClient =
-                CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(
+            alice.rpc.startFlow(
                 ::CashIssueAndPaymentFlow,
                 500.DOLLARS,
                 OpaqueBytes.of(0x01),
@@ -154,15 +139,11 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 defaultNotaryIdentity
             ).returnValue.getOrThrow(30.seconds)
 
-            val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(0, discharge)
-            assertEquals(1, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            assertEquals(1, charlieClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-            // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow
-            assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            charlie.rpc.assertHospitalCounts(observation = 1)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(1, charlie.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -170,22 +151,22 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
      * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding
      * flow's node.
      *
-     * The exception is thrown 5 times.
+     * The exception is thrown 3 times.
      *
      * The responding flow is retried 3 times and then completes successfully.
      *
      * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the
      * flow is retried instead of moving straight to observation.
      */
-    @Test(timeout=300_000)
-	fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() {
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() {
         startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) {
-            val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
             val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -201,38 +182,17 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
                 
                 RULE Throw exception on executeCommitTransaction action
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("finality_flag") && readCounter("counter") < 5
+                IF flagged("finality_flag") && readCounter("counter") < 3
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
-                
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-            val charlieClient =
-                CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(
+            alice.rpc.startFlow(
                 ::CashIssueAndPaymentFlow,
                 500.DOLLARS,
                 OpaqueBytes.of(0x01),
@@ -241,20 +201,14 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 defaultNotaryIdentity
             ).returnValue.getOrThrow(30.seconds)
 
-            val output = getBytemanOutput(charlie)
+            // This sleep is a bit suspect...
+            Thread.sleep(1000)
 
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            assertEquals(0, charlieClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(0, charlie.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -262,7 +216,7 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
      * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding
      * flow's node.
      *
-     * The exception is thrown 7 times.
+     * The exception is thrown 4 times.
      *
      * The responding flow is retried 3 times and is then kept in for observation.
      *
@@ -272,15 +226,15 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
      * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the
      * flow is retried instead of moving straight to observation.
      */
-    @Test(timeout=300_000)
-	fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() {
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() {
         startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) {
-            val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS)
             val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -296,39 +250,18 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
                 
                 RULE Throw exception on executeCommitTransaction action
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("finality_flag") && readCounter("counter") < 7
+                IF flagged("finality_flag") && readCounter("counter") < 4
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
-                
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
-
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-            val charlieClient =
-                CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
+            submitBytemanRules(rules, port)
 
             assertFailsWith<TimeoutException> {
-                aliceClient.startFlow(
+                alice.rpc.startFlow(
                     ::CashIssueAndPaymentFlow,
                     500.DOLLARS,
                     OpaqueBytes.of(0x01),
@@ -338,20 +271,14 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ).returnValue.getOrThrow(30.seconds)
             }
 
-            val output = getBytemanOutput(charlie)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(1, observation)
-            assertEquals(1, aliceClient.stateMachinesSnapshot().size)
-            assertEquals(1, charlieClient.stateMachinesSnapshot().size)
-            // 1 for CashIssueAndPaymentFlow and 1 for GetNumberOfCheckpointsFlow
-            assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-            // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow
-            assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpoints(runnable = 1)
+            charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            charlie.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(1, charlie.rpc.stateMachinesSnapshot().size)
         }
     }
 }

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt

@@ -0,0 +1,581 @@
+package net.corda.node.services.statemachine
+
+import net.corda.core.CordaRuntimeException
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.seconds
+import net.corda.node.services.api.CheckpointStorage
+import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.CHARLIE_NAME
+import net.corda.testing.core.singleIdentity
+import net.corda.testing.driver.internal.OutOfProcessImpl
+import org.junit.Test
+import java.sql.Connection
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeoutException
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertTrue
+
+@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
+class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() {
+
+    private companion object {
+        val executor: ExecutorService = Executors.newSingleThreadExecutor()
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint
+     * (remains in an unstarted state).
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     *
+     * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state).
+     *
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when calling [FlowStateMachineImpl.processEvent].
+     *
+     * This is not an expected place for an exception to occur, but allows us to test what happens when a random exception is propagated
+     * up to [FlowStateMachineImpl.run] during flow initialisation.
+     *
+     * A "Transaction context is missing" exception is thrown due to where the exception is thrown (no transaction is created so this is
+     * thrown when leaving [FlowStateMachineImpl.processEventsUntilFlowIsResumed] due to the finally block).
+     */
+    @Test(timeout = 300_000)
+    fun `unexpected error during flow initialisation throws exception to client`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+            val rules = """
+                RULE Create Counter
+                CLASS ${FlowStateMachineImpl::class.java.name}
+                METHOD processEvent
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+                
+                RULE Throw exception
+                CLASS ${FlowStateMachineImpl::class.java.name}
+                METHOD processEvent
+                AT ENTRY
+                IF readCounter("counter") < 1
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            assertFailsWith<CordaRuntimeException> {
+                alice.rpc.startFlow(
+                    StateMachineErrorHandlingTest::SendAMessageFlow,
+                    charlie.nodeInfo.singleIdentity()
+                ).returnValue.getOrThrow(30.seconds)
+            }
+
+            alice.rpc.assertNumberOfCheckpoints(failed = 1)
+            alice.rpc.assertHospitalCounts(propagated = 1)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint
+     * (remains in an unstarted state).
+     *
+     * A [SQLException] is then thrown when trying to rollback the flow's database transaction.
+     *
+     * The [SQLException] should be suppressed and the flow should continue to retry and complete successfully.
+     */
+    @Test(timeout = 300_000)
+    fun `error during initialisation when trying to rollback the flow's database transaction the flow is able to retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") == 0
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception when rolling back transaction in transition executor
+                INTERFACE ${Connection::class.java.name}
+                METHOD rollback
+                AT ENTRY
+                IF readCounter("counter") == 1
+                DO incrementCounter("counter"); traceln("Throwing exception in transition executor"); throw new java.sql.SQLException("could not reach db", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(30.seconds)
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 1)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint
+     * (remains in an unstarted state).
+     *
+     * A [SQLException] is then thrown when trying to close the flow's database transaction.
+     *
+     * The [SQLException] should be suppressed and the flow should continue to retry and complete successfully.
+     */
+    @Test(timeout = 300_000)
+    fun `error during initialisation when trying to close the flow's database transaction the flow is able to retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") == 0
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception when rolling back transaction in transition executor
+                INTERFACE ${Connection::class.java.name}
+                METHOD close
+                AT ENTRY
+                IF readCounter("counter") == 1
+                DO incrementCounter("counter"); traceln("Throwing exception in transition executor"); throw new java.sql.SQLException("could not reach db", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(30.seconds)
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 1)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint
+     * (remains in an unstarted state).
+     *
+     * The exception is thrown 4 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation.
+     *
+     * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state).
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            executor.execute {
+                alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity())
+            }
+
+            // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead
+            Thread.sleep(30.seconds.toMillis())
+
+            alice.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+            val terminated = (alice as OutOfProcessImpl).stop(60.seconds)
+            assertTrue(terminated, "The node must be shutdown before it can be restarted")
+            val (alice2, _) = createBytemanNode(ALICE_NAME)
+            Thread.sleep(20.seconds.toMillis())
+            alice2.rpc.assertNumberOfCheckpointsAllZero()
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint
+     * (remains in an unstarted state).
+     *
+     * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another
+     * exception is then thrown during the retry itself.
+     *
+     * The flow then retries the retry causing the flow to complete successfully.
+     */
+    @Test(timeout = 300_000)
+    fun `error during retrying a flow that failed when committing its original checkpoint will retry the flow again and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Throw exception on executeCommitTransaction action after first suspend + commit
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF !flagged("commit_exception_flag")
+                DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+                
+                RULE Throw exception on retry
+                CLASS $stateMachineManagerClassName
+                METHOD onExternalStartFlow
+                AT ENTRY
+                IF flagged("commit_exception_flag") && !flagged("retry_exception_flag")
+                DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(
+                discharged = 1,
+                dischargedRetry = 1
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event on a responding node before the flow has initialised and
+     * saved its first checkpoint (remains in an unstarted state).
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     *
+     * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state).
+     */
+    @Test(timeout = 300_000)
+    fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() {
+        startDriver {
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME)
+            val alice = createNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertHospitalCounts(discharged = 3)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event on a responding node before the flow has initialised and
+     * saved its first checkpoint (remains in an unstarted state).
+     *
+     * The exception is thrown 4 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation.
+     *
+     * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state).
+     */
+    @Test(timeout = 300_000)
+    fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() {
+        startDriver {
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME)
+            val alice = createNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            executor.execute {
+                alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity())
+            }
+
+            // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead
+            Thread.sleep(30.seconds.toMillis())
+
+            alice.rpc.assertNumberOfCheckpoints(runnable = 1)
+            charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            charlie.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(1, charlie.rpc.stateMachinesSnapshot().size)
+            val terminated = (charlie as OutOfProcessImpl).stop(60.seconds)
+            assertTrue(terminated, "The node must be shutdown before it can be restarted")
+            val (charlie2, _) = createBytemanNode(CHARLIE_NAME)
+            Thread.sleep(10.seconds.toMillis())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie2.rpc.assertNumberOfCheckpointsAllZero()
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted
+     * state) on a responding node.
+     *
+     * The exception is thrown 3 times.
+     *
+     * An exception is also thrown from [CheckpointStorage.getCheckpoint].
+     *
+     * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when
+     * retrying the flow after it failed to commit it's original checkpoint.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     */
+    @Test(timeout = 300_000)
+    fun `responding flow - session init can be retried when there is a transient connection error to the database`() {
+        startDriver {
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME)
+            val alice = createNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter) 
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception on getCheckpoint
+                INTERFACE ${CheckpointStorage::class.java.name}
+                METHOD getCheckpoint
+                AT ENTRY
+                IF true
+                DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 0
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(0, charlie.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted
+     * state) on a responding node.
+     *
+     * The exception is thrown 4 times.
+     *
+     * An exception is also thrown from [CheckpointStorage.getCheckpoint].
+     *
+     * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when
+     * retrying the flow after it failed to commit it's original checkpoint.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * fails and is kept for in for observation.
+     */
+    @Test(timeout = 300_000)
+    fun `responding flow - session init can be retried when there is a transient connection error to the database goes to observation if error persists`() {
+        startDriver {
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME)
+            val alice = createNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter) 
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception on getCheckpoint
+                INTERFACE ${CheckpointStorage::class.java.name}
+                METHOD getCheckpoint
+                AT ENTRY
+                IF true
+                DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            assertFailsWith<TimeoutException> {
+                alice.rpc.startFlow(
+                    StateMachineErrorHandlingTest::SendAMessageFlow,
+                    charlie.nodeInfo.singleIdentity()
+                ).returnValue.getOrThrow(
+                    30.seconds
+                )
+            }
+
+            charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            charlie.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(1, charlie.rpc.stateMachinesSnapshot().size)
+        }
+    }
+}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt

@@ -0,0 +1,661 @@
+package net.corda.node.services.statemachine
+
+import net.corda.core.CordaRuntimeException
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.seconds
+import net.corda.node.services.api.CheckpointStorage
+import net.corda.node.services.messaging.DeduplicationHandler
+import net.corda.node.services.statemachine.transitions.TopLevelTransition
+import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.CHARLIE_NAME
+import net.corda.testing.core.singleIdentity
+import org.junit.Test
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeoutException
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+
+@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
+class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() {
+
+    private companion object {
+        val executor: ExecutorService = Executors.newSingleThreadExecutor()
+    }
+
+    /**
+     * Throws an exception when performing an [Action.SendInitial] action.
+     *
+     * The exception is thrown 4 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and is then kept in
+     * the hospital for observation.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with SendInitial action is retried 3 times and kept for observation if error persists`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeSendMultiple action
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            assertFailsWith<TimeoutException> {
+                alice.rpc.startFlow(
+                    StateMachineErrorHandlingTest::SendAMessageFlow,
+                    charlie.nodeInfo.singleIdentity()
+                ).returnValue.getOrThrow(
+                    30.seconds
+                )
+            }
+
+            alice.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.SendInitial] event.
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with SendInitial action that does not persist will retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeSendMultiple action
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when executing [DeduplicationHandler.afterDatabaseTransaction] from inside an [Action.AcknowledgeMessages] action.
+     *
+     * The exception is thrown every time [DeduplicationHandler.afterDatabaseTransaction] is executed inside of
+     * [ActionExecutorImpl.executeAcknowledgeMessages]
+     *
+     * The exceptions should be swallowed. Therefore there should be no trips to the hospital and no retries.
+     * The flow should complete successfully as the error is swallowed.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with AcknowledgeMessages action is swallowed and flow completes successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Set flag when inside executeAcknowledgeMessages
+                CLASS $actionExecutorClassName
+                METHOD executeAcknowledgeMessages
+                AT INVOKE ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction()
+                IF !flagged("exception_flag")
+                DO flag("exception_flag"); traceln("Setting flag to true")
+                ENDRULE
+                
+                RULE Throw exception when executing ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction when inside executeAcknowledgeMessages
+                INTERFACE ${DeduplicationHandler::class.java.name}
+                METHOD afterDatabaseTransaction
+                AT ENTRY
+                IF flagged("exception_flag")
+                DO traceln("Throwing exception"); clear("exception_flag"); traceln("SETTING FLAG TO FALSE"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCountsAllZero()
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event when trying to propagate an error (processing an
+     * [Event.StartErrorPropagation] event)
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the flow to retry the [Event.StartErrorPropagation] event until it succeeds. This this scenario it is retried 3 times,
+     * on the final retry the flow successfully propagates the error and completes exceptionally.
+     */
+    @Test(timeout = 300_000)
+    fun `error during error propagation the flow is able to retry and recover`() {
+        startDriver {
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS ${ThrowAnErrorFlow::class.java.name}
+                METHOD throwException
+                AT ENTRY
+                IF !flagged("my_flag")
+                DO traceln("SETTING FLAG TO TRUE"); flag("my_flag")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("my_flag") && readCounter("counter") < 3
+                DO traceln("Throwing exception"); incrementCounter("counter"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            assertFailsWith<CordaRuntimeException> {
+                alice.rpc.startFlow(StateMachineErrorHandlingTest::ThrowAnErrorFlow).returnValue.getOrThrow(60.seconds)
+            }
+
+            alice.rpc.assertNumberOfCheckpoints(failed = 1)
+            alice.rpc.assertHospitalCounts(
+                propagated = 1,
+                propagatedRetry = 3
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when replaying a flow that has already successfully created its initial checkpoint.
+     *
+     * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another
+     * exception is then thrown during the retry itself.
+     *
+     * The flow is discharged and replayed from the hospital. An exception is then thrown during the retry that causes the flow to be
+     * retried again.
+     */
+    @Test(timeout = 300_000)
+    fun `error during flow retry when executing retryFlowFromSafePoint the flow is able to retry and recover`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Set flag when executing first suspend
+                CLASS ${TopLevelTransition::class.java.name}
+                METHOD suspendTransition
+                AT ENTRY
+                IF !flagged("suspend_flag")
+                DO flag("suspend_flag"); traceln("Setting suspend flag to true")
+                ENDRULE
+                
+                RULE Throw exception on executeCommitTransaction action after first suspend + commit
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag")
+                DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+                
+                RULE Set flag when executing first commit
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("suspend_flag") && !flagged("commit_flag")
+                DO flag("commit_flag"); traceln("Setting commit flag to true")
+                ENDRULE
+                
+                RULE Throw exception on retry
+                CLASS $stateMachineManagerClassName
+                METHOD addAndStartFlow
+                AT ENTRY
+                IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("retry_exception_flag")
+                DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(40.seconds)
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(
+                discharged = 1,
+                dischargedRetry = 1
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event after the flow has suspended (has moved to a started state).
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     *
+     * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action that occurs after the first suspend will retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            // seems to be restarting the flow from the beginning every time
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+                
+                RULE Set flag when executing first suspend
+                CLASS ${TopLevelTransition::class.java.name}
+                METHOD suspendTransition
+                AT ENTRY
+                IF !flagged("suspend_flag")
+                DO flag("suspend_flag"); traceln("Setting suspend flag to true")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action after first suspend + commit
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Set flag when executing first commit
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("suspend_flag") && !flagged("commit_flag")
+                DO flag("commit_flag"); traceln("Setting commit flag to true")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing.
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     *
+     * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            // seems to be restarting the flow from the beginning every time
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+                
+                RULE Set flag when adding action to remove checkpoint
+                CLASS ${TopLevelTransition::class.java.name}
+                METHOD flowFinishTransition
+                AT ENTRY
+                IF !flagged("remove_checkpoint_flag")
+                DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction when removing checkpoint
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3
+                DO incrementCounter("counter"); clear("remove_checkpoint_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws a [ConstraintViolationException] when performing an [Action.CommitTransaction] event when the flow is finishing.
+     *
+     * The exception is thrown 4 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation.
+     *
+     * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition with CommitTransaction action and ConstraintViolationException that occurs when completing a flow will retry and be kept for observation if error persists`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+                
+                RULE Set flag when adding action to remove checkpoint
+                CLASS ${TopLevelTransition::class.java.name}
+                METHOD flowFinishTransition
+                AT ENTRY
+                IF !flagged("remove_checkpoint_flag")
+                DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction when removing checkpoint
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("remove_checkpoint_flag") && readCounter("counter") < 4
+                DO incrementCounter("counter"); 
+                    clear("remove_checkpoint_flag"); 
+                    traceln("Throwing exception"); 
+                    throw new org.hibernate.exception.ConstraintViolationException("This flow has a terminal condition", new java.sql.SQLException(), "made up constraint")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            assertFailsWith<TimeoutException> {
+                alice.rpc.startFlow(
+                    StateMachineErrorHandlingTest::SendAMessageFlow,
+                    charlie.nodeInfo.singleIdentity()
+                ).returnValue.getOrThrow(
+                    30.seconds
+                )
+            }
+
+            alice.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted
+     * state).
+     *
+     * The exception is thrown 3 times.
+     *
+     * An exception is also thrown from [CheckpointStorage.getCheckpoint].
+     *
+     * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when
+     * retrying the flow after it failed to commit it's original checkpoint.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     */
+    @Test(timeout = 300_000)
+    fun `flow can be retried when there is a transient connection error to the database`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter) 
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 3
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception on getCheckpoint
+                INTERFACE ${CheckpointStorage::class.java.name}
+                METHOD getCheckpoint
+                AT ENTRY
+                IF true
+                DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 0
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted
+     * state).
+     *
+     * The exception is thrown 4 times.
+     *
+     * An exception is also thrown from [CheckpointStorage.getCheckpoint].
+     *
+     * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when
+     * retrying the flow after it failed to commit it's original checkpoint.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * fails and is kept for in for observation.
+     */
+    @Test(timeout = 300_000)
+    fun `flow can be retried when there is a transient connection error to the database goes to observation if error persists`() {
+        startDriver {
+            val charlie = createNode(CHARLIE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter) 
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction action
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+                
+                RULE Throw exception on getCheckpoint
+                INTERFACE ${CheckpointStorage::class.java.name}
+                METHOD getCheckpoint
+                AT ENTRY
+                IF true
+                DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            executor.execute {
+                alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity())
+            }
+
+            // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead
+            Thread.sleep(30.seconds.toMillis())
+
+            alice.rpc.assertNumberOfCheckpoints(hospitalized = 1)
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(1, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing on a responding node.
+     *
+     * The exception is thrown 3 times.
+     *
+     * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
+     * succeeds and the flow finishes.
+     */
+    @Test(timeout = 300_000)
+    fun `responding flow - error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() {
+        startDriver {
+            val (charlie, port) = createBytemanNode(CHARLIE_NAME)
+            val alice = createNode(ALICE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+                
+                RULE Set flag when adding action to remove checkpoint
+                CLASS ${TopLevelTransition::class.java.name}
+                METHOD flowFinishTransition
+                AT ENTRY
+                IF !flagged("remove_checkpoint_flag")
+                DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true")
+                ENDRULE
+
+                RULE Throw exception on executeCommitTransaction when removing checkpoint
+                CLASS $actionExecutorClassName
+                METHOD executeCommitTransaction
+                AT ENTRY
+                IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3
+                DO incrementCounter("counter"); 
+                    clear("remove_checkpoint_flag"); 
+                    traceln("Throwing exception"); 
+                    throw new java.sql.SQLException("die dammit die", "1")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            alice.rpc.startFlow(
+                StateMachineErrorHandlingTest::SendAMessageFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
+                30.seconds
+            )
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertNumberOfCheckpointsAllZero()
+            charlie.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+            assertEquals(0, charlie.rpc.stateMachinesSnapshot().size)
+        }
+    }
+}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt

@@ -0,0 +1,181 @@
+package net.corda.node.services.statemachine
+
+import co.paralleluniverse.fibers.Suspendable
+import net.corda.core.flows.FlowLogic
+import net.corda.core.flows.KilledFlowException
+import net.corda.core.flows.StartableByRPC
+import net.corda.core.messaging.startFlow
+import net.corda.core.messaging.startTrackedFlow
+import net.corda.core.utilities.ProgressTracker
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.seconds
+import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.CHARLIE_NAME
+import net.corda.testing.core.singleIdentity
+import org.junit.Test
+import java.time.Duration
+import java.time.temporal.ChronoUnit
+import java.util.concurrent.TimeoutException
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertTrue
+
+@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
+class StateMachineKillFlowErrorHandlingTest : StateMachineErrorHandlingTest() {
+
+    /**
+     * Triggers `killFlow` while the flow is suspended causing a [InterruptedException] to be thrown and passed through the hospital.
+     *
+     * The flow terminates and is not retried.
+     *
+     * No pass through the hospital is recorded. As the flow is marked as `isRemoved`.
+     */
+    @Test(timeout = 300_000)
+    fun `error during transition due to killing a flow will terminate the flow`() {
+        startDriver {
+            val alice = createNode(ALICE_NAME)
+
+            val flow = alice.rpc.startTrackedFlow(StateMachineKillFlowErrorHandlingTest::SleepFlow)
+
+            var flowKilled = false
+            flow.progress.subscribe {
+                if (it == SleepFlow.STARTED.label) {
+                    Thread.sleep(5000)
+                    flowKilled = alice.rpc.killFlow(flow.id)
+                }
+            }
+
+            assertFailsWith<KilledFlowException> { flow.returnValue.getOrThrow(20.seconds) }
+
+            assertTrue(flowKilled)
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCountsAllZero()
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Triggers `killFlow` during user application code.
+     *
+     * The user application code is mimicked by a [Thread.sleep] which is importantly not placed inside the [Suspendable]
+     * call function. Placing it inside a [Suspendable] function causes quasar to behave unexpectedly.
+     *
+     * Although the call to kill the flow is made during user application code. It will not be removed / stop processing
+     * until the next suspension point is reached within the flow.
+     *
+     * The flow terminates and is not retried.
+     *
+     * No pass through the hospital is recorded. As the flow is marked as `isRemoved`.
+     */
+    @Test(timeout = 300_000)
+    fun `flow killed during user code execution stops and removes the flow correctly`() {
+        startDriver {
+            val alice = createNode(ALICE_NAME)
+
+            val flow = alice.rpc.startTrackedFlow(StateMachineKillFlowErrorHandlingTest::ThreadSleepFlow)
+
+            var flowKilled = false
+            flow.progress.subscribe {
+                if (it == ThreadSleepFlow.STARTED.label) {
+                    Thread.sleep(5000)
+                    flowKilled = alice.rpc.killFlow(flow.id)
+                }
+            }
+
+            assertFailsWith<KilledFlowException> { flow.returnValue.getOrThrow(30.seconds) }
+
+            assertTrue(flowKilled)
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCountsAllZero()
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    /**
+     * Triggers `killFlow` after the flow has already been sent to observation. The flow is not running at this point and
+     * all that remains is its checkpoint in the database.
+     *
+     * The flow terminates and is not retried.
+     *
+     * Killing the flow does not lead to any passes through the hospital. All the recorded passes through the hospital are
+     * from the original flow that was put in for observation.
+     */
+    @Test(timeout = 300_000)
+    fun `flow killed when it is in the flow hospital for observation is removed correctly`() {
+        startDriver {
+            val (alice, port) = createBytemanNode(ALICE_NAME)
+            val charlie = createNode(CHARLIE_NAME)
+
+            val rules = """
+                RULE Create Counter
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF createCounter("counter", $counter)
+                DO traceln("Counter created")
+                ENDRULE
+
+                RULE Throw exception on executeSendMultiple action
+                CLASS $actionExecutorClassName
+                METHOD executeSendMultiple
+                AT ENTRY
+                IF readCounter("counter") < 4
+                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
+                ENDRULE
+            """.trimIndent()
+
+            submitBytemanRules(rules, port)
+
+            val flow = alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity())
+
+            assertFailsWith<TimeoutException> { flow.returnValue.getOrThrow(20.seconds) }
+
+            alice.rpc.killFlow(flow.id)
+
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(
+                discharged = 3,
+                observation = 1
+            )
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
+        }
+    }
+
+    @StartableByRPC
+    class SleepFlow : FlowLogic<Unit>() {
+
+        object STARTED : ProgressTracker.Step("I am ready to die")
+
+        override val progressTracker = ProgressTracker(STARTED)
+
+        @Suspendable
+        override fun call() {
+            sleep(Duration.of(1, ChronoUnit.SECONDS))
+            progressTracker.currentStep = STARTED
+            sleep(Duration.of(2, ChronoUnit.MINUTES))
+        }
+    }
+
+    @StartableByRPC
+    class ThreadSleepFlow : FlowLogic<Unit>() {
+
+        object STARTED : ProgressTracker.Step("I am ready to die")
+
+        override val progressTracker = ProgressTracker(STARTED)
+
+        @Suspendable
+        override fun call() {
+            sleep(Duration.of(1, ChronoUnit.SECONDS))
+            progressTracker.currentStep = STARTED
+            logger.info("Starting ${ThreadSleepFlow::class.qualifiedName} application sleep")
+            sleep()
+            logger.info("Finished ${ThreadSleepFlow::class.qualifiedName} application sleep")
+            sleep(Duration.of(2, ChronoUnit.MINUTES))
+        }
+
+        // Sleep is moved outside of `@Suspendable` function to prevent issues with Quasar
+        private fun sleep() {
+            Thread.sleep(20000)
+        }
+    }
+}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt

@@ -1,7 +1,6 @@
 package net.corda.node.services.statemachine
 
 import co.paralleluniverse.fibers.Suspendable
-import net.corda.client.rpc.CordaRPCClient
 import net.corda.core.flows.FlowLogic
 import net.corda.core.flows.FlowSession
 import net.corda.core.flows.InitiatedBy
@@ -20,13 +19,14 @@ import org.junit.Test
 import kotlin.test.assertEquals
 
 @Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
-class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
+class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() {
 
     /**
      * This test checks that flow calling an initiating subflow will recover correctly.
      *
      * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first send to a counterparty.
-     * The exception is thrown 5 times.
+     *
+     * The exception is thrown 3 times.
      *
      * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
      * succeeds and the flow finishes.
@@ -37,15 +37,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify
      * that 3 retries are attempted before recovering.
      */
-    @Test(timeout=300_000)
-	fun `initiating subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() {
+    @Test(timeout = 300_000)
+    fun `initiating subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() {
         startDriver {
             val charlie = createNode(CHARLIE_NAME)
-            val alice = createBytemanNode(ALICE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -69,66 +69,34 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
 
                 RULE Throw exception on executeCommitTransaction action after first suspend + commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("subflow_flag") && flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 5
+                IF flagged("subflow_flag") && flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 3
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
                 
                 RULE Set flag when executing first commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF flagged("subflow_flag") && flagged("suspend_flag") && !flagged("commit_flag")
                 DO flag("commit_flag"); traceln("Setting commit flag to true")
                 ENDRULE
-                
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow(
+            alice.rpc.startFlow(
+                StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
                 30.seconds
             )
 
-            val output = getBytemanOutput(alice)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -136,7 +104,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * This test checks that flow calling an initiating subflow will recover correctly.
      *
      * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first receive from a counterparty.
-     * The exception is thrown 5 times.
+     *
+     * The exception is thrown 3 times.
      *
      * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
      * succeeds and the flow finishes.
@@ -147,15 +116,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify
      * that 3 retries are attempted before recovering.
      */
-    @Test(timeout=300_000)
-	fun `initiating subflow - error during transition with CommitTransaction action that occurs after the first receive will retry and complete successfully`() {
+    @Test(timeout = 300_000)
+    fun `initiating subflow - error during transition with CommitTransaction action that occurs after the first receive will retry and complete successfully`() {
         startDriver {
             val charlie = createNode(CHARLIE_NAME)
-            val alice = createBytemanNode(ALICE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -179,58 +148,26 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
 
                 RULE Throw exception on executeCommitTransaction action after first suspend + commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("subflow_flag") && flagged("suspend_flag") && readCounter("counter") < 5
+                IF flagged("subflow_flag") && flagged("suspend_flag") && readCounter("counter") < 3
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
-                
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow(
+            alice.rpc.startFlow(
+                StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
                 30.seconds
             )
 
-            val output = getBytemanOutput(alice)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -238,7 +175,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * This test checks that flow calling an inline subflow will recover correctly.
      *
      * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first send to a counterparty.
-     * The exception is thrown 5 times.
+     *
+     * The exception is thrown 3 times.
      *
      * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
      * succeeds and the flow finishes.
@@ -249,15 +187,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify
      * that 3 retries are attempted before recovering.
      */
-    @Test(timeout=300_000)
-	fun `inline subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() {
+    @Test(timeout = 300_000)
+    fun `inline subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() {
         startDriver {
             val charlie = createNode(CHARLIE_NAME)
-            val alice = createBytemanNode(ALICE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -273,58 +211,26 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
                 
                 RULE Throw exception on executeCommitTransaction action after first suspend + commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("subflow_flag") && readCounter("counter") < 5
+                IF flagged("subflow_flag") && readCounter("counter") < 3
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
-                
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow(
+            alice.rpc.startFlow(
+                StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
                 30.seconds
             )
 
-            val output = getBytemanOutput(alice)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
         }
     }
 
@@ -332,7 +238,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * This test checks that flow calling an inline subflow will recover correctly.
      *
      * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first receive from a counterparty.
-     * The exception is thrown 5 times.
+     *
+     * The exception is thrown 3 times.
      *
      * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition
      * succeeds and the flow finishes.
@@ -343,15 +250,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
      * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify
      * that 3 retries are attempted before recovering.
      */
-    @Test(timeout=300_000)
-	fun `inline subflow - error during transition with CommitTransaction action that occurs during the first receive will retry and complete successfully`() {
+    @Test(timeout = 300_000)
+    fun `inline subflow - error during transition with CommitTransaction action that occurs during the first receive will retry and complete successfully`() {
         startDriver {
             val charlie = createNode(CHARLIE_NAME)
-            val alice = createBytemanNode(ALICE_NAME)
+            val (alice, port) = createBytemanNode(ALICE_NAME)
 
             val rules = """
                 RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF createCounter("counter", $counter)
@@ -367,66 +274,34 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() {
                 ENDRULE
                 
                 RULE Throw exception on executeCommitTransaction action after first suspend + commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
-                IF flagged("subflow_flag") && flagged("commit_flag") && readCounter("counter") < 5
+                IF flagged("subflow_flag") && flagged("commit_flag") && readCounter("counter") < 3
                 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
                 ENDRULE
                 
                 RULE Set flag when executing first commit
-                CLASS ${ActionExecutorImpl::class.java.name}
+                CLASS $actionExecutorClassName
                 METHOD executeCommitTransaction
                 AT ENTRY
                 IF flagged("subflow_flag") && !flagged("commit_flag")
                 DO flag("commit_flag"); traceln("Setting commit flag to true")
                 ENDRULE
-                
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
             """.trimIndent()
 
-            submitBytemanRules(rules)
+            submitBytemanRules(rules, port)
 
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow(
+            alice.rpc.startFlow(
+                StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow,
+                charlie.nodeInfo.singleIdentity()
+            ).returnValue.getOrThrow(
                 30.seconds
             )
 
-            val output = getBytemanOutput(alice)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
+            alice.rpc.assertNumberOfCheckpointsAllZero()
+            alice.rpc.assertHospitalCounts(discharged = 3)
+            assertEquals(0, alice.rpc.stateMachinesSnapshot().size)
         }
     }
 

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt

@@ -1,166 +0,0 @@
-package net.corda.node.services.statemachine
-
-import co.paralleluniverse.fibers.Suspendable
-import net.corda.core.flows.FlowLogic
-import net.corda.core.flows.FlowSession
-import net.corda.core.flows.InitiatedBy
-import net.corda.core.flows.InitiatingFlow
-import net.corda.core.flows.StartableByRPC
-import net.corda.core.identity.CordaX500Name
-import net.corda.core.identity.Party
-import net.corda.core.internal.list
-import net.corda.core.internal.readAllLines
-import net.corda.core.node.AppServiceHub
-import net.corda.core.node.services.CordaService
-import net.corda.core.serialization.CordaSerializable
-import net.corda.core.serialization.SingletonSerializeAsToken
-import net.corda.core.utilities.getOrThrow
-import net.corda.core.utilities.unwrap
-import net.corda.node.services.Permissions
-import net.corda.testing.core.DUMMY_NOTARY_NAME
-import net.corda.testing.driver.DriverDSL
-import net.corda.testing.driver.DriverParameters
-import net.corda.testing.driver.NodeHandle
-import net.corda.testing.driver.NodeParameters
-import net.corda.testing.driver.driver
-import net.corda.testing.node.NotarySpec
-import net.corda.testing.node.TestCordapp
-import net.corda.testing.node.User
-import net.corda.testing.node.internal.InternalDriverDSL
-import org.jboss.byteman.agent.submit.ScriptText
-import org.jboss.byteman.agent.submit.Submit
-import org.junit.Before
-
-abstract class StatemachineErrorHandlingTest {
-
-    val rpcUser = User("user1", "test", permissions = setOf(Permissions.all()))
-    var counter = 0
-
-    @Before
-    fun setup() {
-        counter = 0
-    }
-
-    internal fun startDriver(notarySpec: NotarySpec = NotarySpec(DUMMY_NOTARY_NAME), dsl: DriverDSL.() -> Unit) {
-        driver(
-            DriverParameters(
-                notarySpecs = listOf(notarySpec),
-                startNodesInProcess = false,
-                inMemoryDB = false,
-                systemProperties = mapOf("co.paralleluniverse.fibers.verifyInstrumentation" to "true")
-            )
-        ) {
-            dsl()
-        }
-    }
-
-    internal fun DriverDSL.createBytemanNode(
-        providedName: CordaX500Name,
-        additionalCordapps: Collection<TestCordapp> = emptyList()
-    ): NodeHandle {
-        return (this as InternalDriverDSL).startNode(
-            NodeParameters(
-                providedName = providedName,
-                rpcUsers = listOf(rpcUser),
-                additionalCordapps = additionalCordapps
-            ),
-            bytemanPort = 12000
-        ).getOrThrow()
-    }
-
-    internal fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection<TestCordapp> = emptyList()): NodeHandle {
-        return startNode(
-            NodeParameters(
-                providedName = providedName,
-                rpcUsers = listOf(rpcUser),
-                additionalCordapps = additionalCordapps
-            )
-        ).getOrThrow()
-    }
-
-    internal fun submitBytemanRules(rules: String) {
-        val submit = Submit("localhost", 12000)
-        submit.addScripts(listOf(ScriptText("Test script", rules)))
-    }
-
-    internal fun getBytemanOutput(nodeHandle: NodeHandle): List<String> {
-        return nodeHandle.baseDirectory
-                .list()
-                .filter { it.toString().contains("net.corda.node.Corda") && it.toString().contains("stdout.log") }
-                .flatMap {  it.readAllLines() }
-     }
-
-    @StartableByRPC
-    @InitiatingFlow
-    class SendAMessageFlow(private val party: Party) : FlowLogic<String>() {
-        @Suspendable
-        override fun call(): String {
-            val session = initiateFlow(party)
-            session.send("hello there")
-            return "Finished executing test flow - ${this.runId}"
-        }
-    }
-
-    @InitiatedBy(SendAMessageFlow::class)
-    class SendAMessageResponder(private val session: FlowSession) : FlowLogic<Unit>() {
-        @Suspendable
-        override fun call() {
-            session.receive<String>().unwrap { it }
-        }
-    }
-
-    @StartableByRPC
-    class GetNumberOfUncompletedCheckpointsFlow : FlowLogic<Long>() {
-        override fun call(): Long {
-            val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})"
-            return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps ->
-                ps.executeQuery().use { rs ->
-                    rs.next()
-                    rs.getLong(1)
-                }
-            }
-        }
-    }
-
-    @StartableByRPC
-    class GetNumberOfHospitalizedCheckpointsFlow : FlowLogic<Long>() {
-        override fun call(): Long {
-            val sqlStatement = "select count(*) from node_checkpoints where status in (${Checkpoint.FlowStatus.HOSPITALIZED.ordinal})"
-            return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps ->
-                ps.executeQuery().use { rs ->
-                    rs.next()
-                    rs.getLong(1)
-                }
-            }
-        }
-    }
-
-    // Internal use for testing only!!
-    @StartableByRPC
-    class GetHospitalCountersFlow : FlowLogic<HospitalCounts>() {
-        override fun call(): HospitalCounts =
-            HospitalCounts(
-                serviceHub.cordaService(HospitalCounter::class.java).dischargeCounter,
-                serviceHub.cordaService(HospitalCounter::class.java).observationCounter
-            )
-    }
-
-    @CordaSerializable
-    data class HospitalCounts(val discharge: Int, val observation: Int)
-
-    @Suppress("UNUSED_PARAMETER")
-    @CordaService
-    class HospitalCounter(services: AppServiceHub) : SingletonSerializeAsToken() {
-        var observationCounter: Int = 0
-        var dischargeCounter: Int = 0
-
-        init {
-            StaffedFlowHospital.onFlowDischarged.add { _, _ ->
-                ++dischargeCounter
-            }
-            StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ ->
-                ++observationCounter
-            }
-        }
-    }
-}

node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt

@@ -1,321 +0,0 @@
-package net.corda.node.services.statemachine
-
-import co.paralleluniverse.fibers.Suspendable
-import net.corda.client.rpc.CordaRPCClient
-import net.corda.core.flows.FlowLogic
-import net.corda.core.flows.KilledFlowException
-import net.corda.core.flows.StartableByRPC
-import net.corda.core.messaging.startFlow
-import net.corda.core.messaging.startTrackedFlow
-import net.corda.core.utilities.ProgressTracker
-import net.corda.core.utilities.getOrThrow
-import net.corda.core.utilities.seconds
-import net.corda.testing.core.ALICE_NAME
-import net.corda.testing.core.CHARLIE_NAME
-import net.corda.testing.core.singleIdentity
-import org.junit.Test
-import java.time.Duration
-import java.time.temporal.ChronoUnit
-import java.util.concurrent.TimeoutException
-import kotlin.test.assertEquals
-import kotlin.test.assertFailsWith
-import kotlin.test.assertTrue
-
-@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped
-class StatemachineKillFlowErrorHandlingTest : StatemachineErrorHandlingTest() {
-
-    /**
-     * Triggers `killFlow` while the flow is suspended causing a [InterruptedException] to be thrown and passed through the hospital.
-     *
-     * The flow terminates and is not retried.
-     *
-     * No pass through the hospital is recorded. As the flow is marked as `isRemoved`.
-     */
-    @Test(timeout=300_000)
-	fun `error during transition due to killing a flow will terminate the flow`() {
-        startDriver {
-            val alice = createBytemanNode(ALICE_NAME)
-
-            val rules = """
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
-                
-                RULE Increment terminal counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ TERMINAL
-                IF true
-                DO traceln("Byteman test - terminal")
-                ENDRULE
-            """.trimIndent()
-
-            submitBytemanRules(rules)
-
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            val flow = aliceClient.startTrackedFlow(StatemachineKillFlowErrorHandlingTest::SleepFlow)
-
-            var flowKilled = false
-            flow.progress.subscribe {
-                if (it == SleepFlow.STARTED.label) {
-                    Thread.sleep(5000)
-                    flowKilled = aliceClient.killFlow(flow.id)
-                }
-            }
-
-            assertFailsWith<KilledFlowException> { flow.returnValue.getOrThrow(20.seconds) }
-
-            val output = getBytemanOutput(alice)
-
-            assertTrue(flowKilled)
-            // Check the stdout for the lines generated by byteman
-            assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(0, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-        }
-    }
-
-    /**
-     * Triggers `killFlow` during user application code.
-     *
-     * The user application code is mimicked by a [Thread.sleep] which is importantly not placed inside the [Suspendable]
-     * call function. Placing it inside a [Suspendable] function causes quasar to behave unexpectedly.
-     *
-     * Although the call to kill the flow is made during user application code. It will not be removed / stop processing
-     * until the next suspension point is reached within the flow.
-     *
-     * The flow terminates and is not retried.
-     *
-     * No pass through the hospital is recorded. As the flow is marked as `isRemoved`.
-     */
-    @Test(timeout=300_000)
-	fun `flow killed during user code execution stops and removes the flow correctly`() {
-        startDriver {
-            val alice = createBytemanNode(ALICE_NAME)
-
-            val rules = """
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
-                
-                RULE Increment terminal counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ TERMINAL
-                IF true
-                DO traceln("Byteman test - terminal")
-                ENDRULE
-            """.trimIndent()
-
-            submitBytemanRules(rules)
-
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            val flow = aliceClient.startTrackedFlow(StatemachineKillFlowErrorHandlingTest::ThreadSleepFlow)
-
-            var flowKilled = false
-            flow.progress.subscribe {
-                if (it == ThreadSleepFlow.STARTED.label) {
-                    Thread.sleep(5000)
-                    flowKilled = aliceClient.killFlow(flow.id)
-                }
-            }
-
-            assertFailsWith<KilledFlowException> { flow.returnValue.getOrThrow(30.seconds) }
-
-            val output = getBytemanOutput(alice)
-
-            assertTrue(flowKilled)
-            // Check the stdout for the lines generated by byteman
-            assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size
-            println(numberOfTerminalDiagnoses)
-            assertEquals(0, numberOfTerminalDiagnoses)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(0, discharge)
-            assertEquals(0, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-        }
-    }
-
-    /**
-     * Triggers `killFlow` after the flow has already been sent to observation. The flow is not running at this point and
-     * all that remains is its checkpoint in the database.
-     *
-     * The flow terminates and is not retried.
-     *
-     * Killing the flow does not lead to any passes through the hospital. All the recorded passes through the hospital are
-     * from the original flow that was put in for observation.
-     */
-    @Test(timeout=300_000)
-	fun `flow killed when it is in the flow hospital for observation is removed correctly`() {
-        startDriver {
-            val alice = createBytemanNode(ALICE_NAME)
-            val charlie = createNode(CHARLIE_NAME)
-
-            val rules = """
-                RULE Create Counter
-                CLASS ${ActionExecutorImpl::class.java.name}
-                METHOD executeSendMultiple
-                AT ENTRY
-                IF createCounter("counter", $counter)
-                DO traceln("Counter created")
-                ENDRULE
-
-                RULE Throw exception on executeSendMultiple action
-                CLASS ${ActionExecutorImpl::class.java.name}
-                METHOD executeSendMultiple
-                AT ENTRY
-                IF readCounter("counter") < 4
-                DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die")
-                ENDRULE
-                
-                RULE Entering internal error staff member
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT ENTRY
-                IF true
-                DO traceln("Reached internal transition error staff member")
-                ENDRULE
-
-                RULE Increment discharge counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ DISCHARGE
-                IF true
-                DO traceln("Byteman test - discharging")
-                ENDRULE
-                
-                RULE Increment observation counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ OVERNIGHT_OBSERVATION
-                IF true
-                DO traceln("Byteman test - overnight observation")
-                ENDRULE
-                
-                RULE Increment terminal counter
-                CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name}
-                METHOD consult
-                AT READ TERMINAL
-                IF true
-                DO traceln("Byteman test - terminal")
-                ENDRULE
-            """.trimIndent()
-
-            submitBytemanRules(rules)
-
-            val aliceClient =
-                CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy
-
-            val flow = aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity())
-
-            assertFailsWith<TimeoutException> { flow.returnValue.getOrThrow(20.seconds) }
-
-            aliceClient.killFlow(flow.id)
-
-            val output = getBytemanOutput(alice)
-
-            // Check the stdout for the lines generated by byteman
-            assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size)
-            assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size)
-            val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size
-            assertEquals(0, numberOfTerminalDiagnoses)
-            val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get()
-            assertEquals(3, discharge)
-            assertEquals(1, observation)
-            assertEquals(0, aliceClient.stateMachinesSnapshot().size)
-            // 1 for GetNumberOfCheckpointsFlow
-            assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get())
-        }
-    }
-
-    @StartableByRPC
-    class SleepFlow : FlowLogic<Unit>() {
-
-        object STARTED : ProgressTracker.Step("I am ready to die")
-
-        override val progressTracker = ProgressTracker(STARTED)
-
-        @Suspendable
-        override fun call() {
-            sleep(Duration.of(1, ChronoUnit.SECONDS))
-            progressTracker.currentStep = STARTED
-            sleep(Duration.of(2, ChronoUnit.MINUTES))
-        }
-    }
-
-    @StartableByRPC
-    class ThreadSleepFlow : FlowLogic<Unit>() {
-
-        object STARTED : ProgressTracker.Step("I am ready to die")
-
-        override val progressTracker = ProgressTracker(STARTED)
-
-        @Suspendable
-        override fun call() {
-            sleep(Duration.of(1, ChronoUnit.SECONDS))
-            progressTracker.currentStep = STARTED
-            logger.info("Starting ${ThreadSleepFlow::class.qualifiedName} application sleep")
-            sleep()
-            logger.info("Finished ${ThreadSleepFlow::class.qualifiedName} application sleep")
-            sleep(Duration.of(2, ChronoUnit.MINUTES))
-        }
-
-        // Sleep is moved outside of `@Suspendable` function to prevent issues with Quasar
-        private fun sleep() {
-            Thread.sleep(20000)
-        }
-    }
-}

node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/DataObject.kt

@@ -0,0 +1,14 @@
+package net.corda.contracts.serialization.generics
+
+import net.corda.core.serialization.CordaSerializable
+
+@CordaSerializable
+data class DataObject(val value: Long) : Comparable<DataObject> {
+    override fun toString(): String {
+        return "$value data points"
+    }
+
+    override fun compareTo(other: DataObject): Int {
+        return value.compareTo(other.value)
+    }
+}

node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/GenericTypeContract.kt

@@ -0,0 +1,47 @@
+package net.corda.contracts.serialization.generics
+
+import net.corda.core.contracts.CommandData
+import net.corda.core.contracts.Contract
+import net.corda.core.contracts.ContractState
+import net.corda.core.contracts.requireThat
+import net.corda.core.identity.AbstractParty
+import net.corda.core.transactions.LedgerTransaction
+import java.util.Optional
+
+@Suppress("unused")
+class GenericTypeContract : Contract {
+    override fun verify(tx: LedgerTransaction) {
+        val states = tx.outputsOfType<State>()
+        requireThat {
+            "Requires at least one data state" using states.isNotEmpty()
+        }
+        val purchases = tx.commandsOfType<Purchase>()
+        requireThat {
+            "Requires at least one purchase" using purchases.isNotEmpty()
+        }
+        for (purchase in purchases) {
+            requireThat {
+                "Purchase has a price" using purchase.value.price.isPresent
+            }
+        }
+    }
+
+    @Suppress("CanBeParameter", "MemberVisibilityCanBePrivate")
+    class State(val owner: AbstractParty, val data: DataObject?) : ContractState {
+        override val participants: List<AbstractParty> = listOf(owner)
+
+        @Override
+        override fun toString(): String {
+            return data.toString()
+        }
+    }
+
+    /**
+     * The [price] field is the important feature of the [Purchase]
+     * class because its type is [Optional] with a CorDapp-specific
+     * generic type parameter. It does not matter that the [price]
+     * is not used; it only matters that the [Purchase] command
+     * must be serialized as part of building a new transaction.
+     */
+    class Purchase(val price: Optional<DataObject>) : CommandData
+}

node/src/integration-test/kotlin/net/corda/flows/serialization/generics/GenericTypeFlow.kt

@@ -0,0 +1,27 @@
+package net.corda.flows.serialization.generics
+
+import co.paralleluniverse.fibers.Suspendable
+import net.corda.contracts.serialization.generics.DataObject
+import net.corda.contracts.serialization.generics.GenericTypeContract.Purchase
+import net.corda.contracts.serialization.generics.GenericTypeContract.State
+import net.corda.core.contracts.Command
+import net.corda.core.crypto.SecureHash
+import net.corda.core.flows.FlowLogic
+import net.corda.core.flows.StartableByRPC
+import net.corda.core.transactions.TransactionBuilder
+import java.util.Optional
+
+@StartableByRPC
+class GenericTypeFlow(private val purchase: DataObject?) : FlowLogic<SecureHash>() {
+    @Suspendable
+    override fun call(): SecureHash {
+        val notary = serviceHub.networkMapCache.notaryIdentities[0]
+        val stx = serviceHub.signInitialTransaction(
+            TransactionBuilder(notary)
+                .addOutputState(State(ourIdentity, purchase))
+                .addCommand(Command(Purchase(Optional.ofNullable(purchase)), ourIdentity.owningKey))
+        )
+        stx.verify(serviceHub, checkSufficientSignatures = false)
+        return stx.id
+    }
+}

node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt

@@ -15,6 +15,7 @@ import net.corda.node.services.Permissions
 import net.corda.node.services.config.PasswordEncryption
 import net.corda.testing.core.ALICE_NAME
 import net.corda.testing.node.internal.NodeBasedTest
+import net.corda.testing.node.internal.cordappForClasses
 import org.apache.activemq.artemis.api.core.ActiveMQSecurityException
 import org.apache.shiro.authc.credential.DefaultPasswordService
 import org.junit.After
@@ -32,7 +33,7 @@ import kotlin.test.assertFailsWith
  * check authentication/authorization of RPC connections.
  */
 @RunWith(Parameterized::class)
-class AuthDBTests : NodeBasedTest() {
+class AuthDBTests : NodeBasedTest(cordappPackages = CORDAPPS) {
     private lateinit var node: NodeWithInfo
     private lateinit var client: CordaRPCClient
     private lateinit var db: UsersDB
@@ -43,6 +44,9 @@ class AuthDBTests : NodeBasedTest() {
         @JvmStatic
         @Parameterized.Parameters(name = "password encryption format = {0}")
         fun encFormats() = arrayOf(PasswordEncryption.NONE, PasswordEncryption.SHIRO_1_CRYPT)
+
+        @Suppress("SpreadOperator")
+        private val CORDAPPS = setOf(cordappForClasses(*AuthDBTests::class.nestedClasses.map { it.java }.toTypedArray()))
     }
 
     @Suppress("MemberVisibilityCanBePrivate")

node/src/integration-test/kotlin/net/corda/node/ContractWithGenericTypeTest.kt

@@ -0,0 +1,78 @@
+package net.corda.node
+
+import net.corda.client.rpc.CordaRPCClient
+import net.corda.contracts.serialization.generics.DataObject
+import net.corda.core.contracts.TransactionVerificationException.ContractRejection
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.getOrThrow
+import net.corda.core.utilities.loggerFor
+import net.corda.flows.serialization.generics.GenericTypeFlow
+import net.corda.node.services.Permissions
+import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.DUMMY_NOTARY_NAME
+import net.corda.testing.driver.DriverParameters
+import net.corda.testing.driver.driver
+import net.corda.testing.driver.internal.incrementalPortAllocation
+import net.corda.testing.node.NotarySpec
+import net.corda.testing.node.User
+import net.corda.testing.node.internal.cordappWithPackages
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.Test
+import org.junit.jupiter.api.assertThrows
+
+@Suppress("FunctionName")
+class ContractWithGenericTypeTest {
+    companion object {
+        const val DATA_VALUE = 5000L
+
+        @JvmField
+        val logger = loggerFor<ContractWithGenericTypeTest>()
+
+        @JvmField
+        val user = User("u", "p", setOf(Permissions.all()))
+
+        fun parameters(): DriverParameters {
+            return DriverParameters(
+                portAllocation = incrementalPortAllocation(),
+                startNodesInProcess = false,
+                notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, validating = true)),
+                cordappsForAllNodes = listOf(
+                    cordappWithPackages("net.corda.flows.serialization.generics").signed(),
+                    cordappWithPackages("net.corda.contracts.serialization.generics").signed()
+                )
+            )
+        }
+    }
+
+    @Test(timeout = 300_000)
+	fun `flow with value of generic type`() {
+        driver(parameters()) {
+            val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow()
+            val txID = CordaRPCClient(hostAndPort = alice.rpcAddress)
+                .start(user.username, user.password)
+                .use { client ->
+                    client.proxy.startFlow(::GenericTypeFlow, DataObject(DATA_VALUE))
+                    .returnValue
+                    .getOrThrow()
+                }
+            logger.info("TX-ID=$txID")
+        }
+    }
+
+    @Test(timeout = 300_000)
+    fun `flow without value of generic type`() {
+        driver(parameters()) {
+            val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow()
+            val ex = assertThrows<ContractRejection> {
+                CordaRPCClient(hostAndPort = alice.rpcAddress)
+                    .start(user.username, user.password)
+                    .use { client ->
+                        client.proxy.startFlow(::GenericTypeFlow, null)
+                            .returnValue
+                            .getOrThrow()
+                    }
+            }
+            assertThat(ex).hasMessageContaining("Contract verification failed: Failed requirement: Purchase has a price,")
+        }
+    }
+}

node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt

@@ -1,114 +0,0 @@
-package net.corda.node
-
-import net.corda.core.utilities.getOrThrow
-import net.corda.node.logging.logFile
-import net.corda.testing.driver.DriverParameters
-import net.corda.testing.driver.NodeParameters
-import net.corda.testing.driver.driver
-import net.corda.testing.driver.internal.incrementalPortAllocation
-import org.assertj.core.api.Assertions.assertThatThrownBy
-import org.junit.Test
-import org.junit.Assert.assertTrue
-
-
-class NodeConfigParsingTests {
-
-    @Test(timeout=300_000)
-	fun `config is overriden by underscore variable`() {
-        val portAllocator = incrementalPortAllocation()
-        val sshPort = portAllocator.nextPort()
-
-        driver(DriverParameters(
-                environmentVariables = mapOf("corda_sshd_port" to sshPort.toString()),
-                startNodesInProcess = false,
-                portAllocation = portAllocator,
-                cordappsForAllNodes = emptyList())) {
-            val hasSsh = startNode().get()
-                    .logFile()
-                    .readLines()
-                    .filter { it.contains("SSH server listening on port") }
-                    .any { it.contains(sshPort.toString()) }
-            assertTrue(hasSsh)
-        }
-    }
-
-    @Test(timeout=300_000)
-	fun `config is overriden by case insensitive underscore variable`() {
-        val portAllocator = incrementalPortAllocation()
-        val sshPort = portAllocator.nextPort()
-
-        driver(DriverParameters(
-                environmentVariables = mapOf("CORDA_sshd_port" to sshPort.toString()),
-                startNodesInProcess = false,
-                portAllocation = portAllocator,
-                cordappsForAllNodes = emptyList())) {
-            val hasSsh = startNode().get()
-                    .logFile()
-                    .readLines()
-                    .filter { it.contains("SSH server listening on port") }
-                    .any { it.contains(sshPort.toString()) }
-            assertTrue(hasSsh)
-        }
-    }
-
-    @Test(timeout=300_000)
-	fun `config is overriden by case insensitive dot variable`() {
-        val portAllocator = incrementalPortAllocation()
-        val sshPort = portAllocator.nextPort()
-
-        driver(DriverParameters(
-                environmentVariables = mapOf("CORDA.sshd.port" to sshPort.toString(),
-                                             "corda.devMode" to true.toString()),
-                startNodesInProcess = false,
-                portAllocation = portAllocator,
-                cordappsForAllNodes = emptyList())) {
-            val hasSsh = startNode(NodeParameters()).get()
-                    .logFile()
-                    .readLines()
-                    .filter { it.contains("SSH server listening on port") }
-                    .any { it.contains(sshPort.toString()) }
-            assertTrue(hasSsh)
-        }
-    }
-
-    @Test(timeout=300_000)
-	fun `shadowing is forbidden`() {
-        val portAllocator = incrementalPortAllocation()
-        val sshPort = portAllocator.nextPort()
-
-        driver(DriverParameters(
-                environmentVariables = mapOf(
-                        "CORDA_sshd_port" to sshPort.toString(),
-                        "corda.sshd.port" to sshPort.toString()),
-                startNodesInProcess = false,
-                portAllocation = portAllocator,
-                notarySpecs = emptyList())) {
-
-            assertThatThrownBy {
-                startNode().getOrThrow()
-            }
-        }
-    }
-
-    @Test(timeout=300_000)
-	fun `bad keys are ignored and warned for`() {
-        val portAllocator = incrementalPortAllocation()
-        driver(DriverParameters(
-                environmentVariables = mapOf(
-                        "corda_bad_key" to "2077"),
-                startNodesInProcess = false,
-                portAllocation = portAllocator,
-                notarySpecs = emptyList(),
-                cordappsForAllNodes = emptyList())) {
-
-                val hasWarning = startNode()
-                        .getOrThrow()
-                        .logFile()
-                        .readLines()
-                        .any {
-                            it.contains("(property or environment variable) cannot be mapped to an existing Corda")
-                        }
-                assertTrue(hasWarning)
-        }
-    }
-}

node/src/integration-test/kotlin/net/corda/node/NodeRPCTests.kt

@@ -11,7 +11,7 @@ import kotlin.test.assertEquals
 import kotlin.test.assertTrue
 
 class NodeRPCTests {
-    private val CORDA_VERSION_REGEX = "\\d+(\\.\\d+)?(-\\w+)?".toRegex()
+    private val CORDA_VERSION_REGEX = "\\d+(\\.\\d+)?(\\.\\d+)?(-\\w+)?".toRegex()
     private val CORDA_VENDOR = "Corda Open Source"
     private val CORDAPPS = listOf(FINANCE_CONTRACTS_CORDAPP, FINANCE_WORKFLOWS_CORDAPP)
     private val CORDAPP_TYPES = setOf("Contract CorDapp", "Workflow CorDapp")

node/src/integration-test/kotlin/net/corda/node/customcheckpointserializer/CustomCheckpointSerializerTest.kt

@@ -0,0 +1,99 @@
+package net.corda.node.customcheckpointserializer
+
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
+import net.corda.core.crypto.generateKeyPair
+import net.corda.core.serialization.EncodingWhitelist
+import net.corda.core.serialization.internal.CheckpointSerializationContext
+import net.corda.core.serialization.internal.checkpointDeserialize
+import net.corda.core.serialization.internal.checkpointSerialize
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.serialization.internal.AllWhitelist
+import net.corda.serialization.internal.CheckpointSerializationContextImpl
+import net.corda.serialization.internal.CordaSerializationEncoding
+import net.corda.testing.core.internal.CheckpointSerializationEnvironmentRule
+import org.junit.Assert
+import org.junit.Rule
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.Parameterized
+
+@RunWith(Parameterized::class)
+class CustomCheckpointSerializerTest(private val compression: CordaSerializationEncoding?) {
+    companion object {
+        @Parameterized.Parameters(name = "{0}")
+        @JvmStatic
+        fun compression() = arrayOf<CordaSerializationEncoding?>(null) + CordaSerializationEncoding.values()
+    }
+
+    @get:Rule
+    val serializationRule = CheckpointSerializationEnvironmentRule(inheritable = true)
+    private val context: CheckpointSerializationContext = CheckpointSerializationContextImpl(
+            deserializationClassLoader = javaClass.classLoader,
+            whitelist = AllWhitelist,
+            properties = emptyMap(),
+            objectReferencesEnabled = true,
+            encoding = compression,
+            encodingWhitelist = rigorousMock<EncodingWhitelist>().also {
+                if (compression != null) doReturn(true).whenever(it).acceptEncoding(compression)
+            },
+            checkpointCustomSerializers = listOf(
+                    TestCorDapp.TestAbstractClassSerializer(),
+                    TestCorDapp.TestClassSerializer(),
+                    TestCorDapp.TestInterfaceSerializer(),
+                    TestCorDapp.TestFinalClassSerializer(),
+                    TestCorDapp.BrokenPublicKeySerializer()
+            )
+    )
+
+    @Test(timeout=300_000)
+    fun `test custom checkpoint serialization`() {
+        testBrokenMapSerialization(DifficultToSerialize.BrokenMapClass())
+    }
+
+    @Test(timeout=300_000)
+    fun `test custom checkpoint serialization using interface`() {
+        testBrokenMapSerialization(DifficultToSerialize.BrokenMapInterfaceImpl())
+    }
+
+    @Test(timeout=300_000)
+    fun `test custom checkpoint serialization using abstract class`() {
+        testBrokenMapSerialization(DifficultToSerialize.BrokenMapAbstractImpl())
+    }
+
+    @Test(timeout=300_000)
+    fun `test custom checkpoint serialization using final class`() {
+        testBrokenMapSerialization(DifficultToSerialize.BrokenMapFinal())
+    }
+
+    @Test(timeout=300_000)
+    fun `test PublicKey serializer has not been overridden`() {
+
+        val publicKey = generateKeyPair().public
+
+        // Serialize/deserialize
+        val checkpoint = publicKey.checkpointSerialize(context)
+        val deserializedCheckpoint = checkpoint.checkpointDeserialize(context)
+
+        // Check the elements are as expected
+        Assert.assertArrayEquals(publicKey.encoded, deserializedCheckpoint.encoded)
+    }
+
+
+    private fun testBrokenMapSerialization(brokenMap : MutableMap<String, String>): MutableMap<String, String> {
+        // Add elements to the map
+        brokenMap.putAll(mapOf("key" to "value"))
+
+        // Serialize/deserialize
+        val checkpoint = brokenMap.checkpointSerialize(context)
+        val deserializedCheckpoint = checkpoint.checkpointDeserialize(context)
+
+        // Check the elements are as expected
+        Assert.assertEquals(1, deserializedCheckpoint.size)
+        Assert.assertEquals("value", deserializedCheckpoint.get("key"))
+
+        // Return map for extra checks
+        return deserializedCheckpoint
+    }
+}
+