diff --git a/.ci/dev/compatibility/DockerfileJDK11Compile b/.ci/dev/compatibility/DockerfileJDK11 similarity index 100% rename from .ci/dev/compatibility/DockerfileJDK11Compile rename to .ci/dev/compatibility/DockerfileJDK11 diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index 96396ca2c0..d24a3f7ac4 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -3,11 +3,31 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Sense environment + */ +boolean isReleaseTag = (env.TAG_NAME =~ /^release.*JDK11$/) + +/* +** calculate the stage for NexusIQ evaluation +** * build for snapshots +** * stage-release: for release candidates and for health checks +** * operate: for final release +*/ +def nexusIqStage = "build" +if (isReleaseTag) { + switch (env.TAG_NAME) { + case ~/.*-RC\d+(-.*)?/: nexusIqStage = "stage-release"; break; + case ~/.*-HC\d+(-.*)?/: nexusIqStage = "stage-release"; break; + default: nexusIqStage = "operate" + } +} pipeline { - agent { label 'k8s' } + agent { + label 'k8s' + } options { timestamps() - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) timeout(time: 3, unit: 'HOURS') } @@ -16,10 +36,33 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { - stage('Corda Pull Request - Generate Build Image') { + stage('Sonatype Check') { + steps { + sh "./gradlew --no-daemon clean jar" + script { + sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "jenkins-${groupId}-${artifactId}-jdk11-${version}" + } + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: manualApplication(nexusAppId), + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: nexusIqStage + ) + } + } + + stage('Generate Build Image') { steps { withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { sh "./gradlew " + @@ -28,8 +71,11 @@ pipeline { "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + "-Ddocker.buildbase.tag=11latest " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.dockerfile=DockerfileJDK11Azul" + - " clean pushBuildImage --stacktrace" + " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace" } sh "kubectl auth can-i get pods" } @@ -65,12 +111,49 @@ pipeline { } } } + + stage('Publish to Artifactory') { + agent { + dockerfile { + reuseNode true + additionalBuildArgs "--build-arg USER=stresstester" + filename '.ci/dev/compatibility/DockerfileJDK11' + } + } + when { + expression { isReleaseTag } + } + steps { + rtServer( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'r3-corda-releases' + ) + rtGradleRun( + usesPlugin: true, + useWrapper: true, + switches: '-s --info', + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + rtPublishBuildInfo( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } } post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index 670717da68..f6e9c43195 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -8,12 +8,13 @@ pipeline { dockerfile { label 'k8s' additionalBuildArgs "--build-arg USER=stresstester" - filename '.ci/dev/compatibility/DockerfileJDK11Compile' + filename '.ci/dev/compatibility/DockerfileJDK11' } } options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } stages { @@ -35,4 +36,4 @@ pipeline { deleteDir() /* clean up our workspace */ } } -} \ No newline at end of file +} diff --git a/.ci/dev/integration/Jenkinsfile b/.ci/dev/integration/Jenkinsfile deleted file mode 100644 index eba467e5a7..0000000000 --- a/.ci/dev/integration/Jenkinsfile +++ /dev/null @@ -1,62 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Corda - Run Tests') { - stage('Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelIntegrationTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/integrationTest', - description: 'Integration Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/.ci/dev/mswin/Jenkinsfile b/.ci/dev/mswin/Jenkinsfile new file mode 100644 index 0000000000..b0e3766e0c --- /dev/null +++ b/.ci/dev/mswin/Jenkinsfile @@ -0,0 +1,101 @@ +#!groovy +/** + * Jenkins pipeline to build Corda on MS Windows server. + * Because it takes a long time to run tests sequentially, unit tests and + * integration tests are started in parallel on separate agents. + * + * Additionally, pull requests by default run only unit tests. + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +/** + * Sense environment + */ +boolean isReleaseBranch = (env.BRANCH_NAME =~ /^release\/os\/.*/) + +pipeline { + agent none + options { + ansiColor('xterm') + timestamps() + timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) + + /* + * a bit awkward to read + * is parameter is true -> push events are *not* ignored + * if parameter is false -> push events *are* ignored + */ + overrideIndexTriggers (!isReleaseBranch) + } + + parameters { + booleanParam defaultValue: (isReleaseBranch), description: 'Run integration tests?', name: 'DO_INTEGRATION_TESTS' + } + + /* + * Do no receive Github's push events for release branches -> suitable for nightly builds + * but projects for pull requests will receive them as normal, and PR builds are started ASAP + */ + triggers { + pollSCM ignorePostCommitHooks: isReleaseBranch, scmpoll_spec: '@midnight' + } + + stages { + stage('Tests') { + parallel { + stage('Unit Tests') { + agent { label 'mswin' } + steps { + bat "./gradlew --no-daemon " + + "--stacktrace " + + "-Pcompilation.warningsAsErrors=false " + + "-Ptests.failFast=true " + + "clean test" + } + post { + always { + archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + bat '.ci/kill_corda_procs.cmd' + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + + } + stage('Integration Tests') { + when { + expression { params.DO_INTEGRATION_TESTS } + beforeAgent true + } + agent { label 'mswin' } + steps { + bat "./gradlew --no-daemon " + + "clean integrationTest" + } + post { + always { + archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + bat '.ci/kill_corda_procs.cmd' + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + } + } + } + } +} diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index a122ea0fd3..c8d5b0e73a 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -8,8 +8,8 @@ pipeline { options { timestamps() overrideIndexTriggers(false) - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } triggers { pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight' @@ -20,64 +20,74 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { - stage('Corda Pull Request - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean jar deployNodes install pushBuildImage --stacktrace" - } - sh "kubectl auth can-i get pods" - } + stage('Deploy Nodes') { + steps { + sh "./gradlew --no-daemon jar deployNodes" } + } - stage('Testing phase') { - parallel { - stage('Regression Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" - } + stage('Generate Build Image') { + steps { + withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { + sh "./gradlew " + + "-Dkubenetize=true " + + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + + " clean pushBuildImage --stacktrace" + } + sh "kubectl auth can-i get pods" + } + } + + stage('Testing phase') { + parallel { + stage('Regression Test') { + steps { + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Dgit.branch=\"\${GIT_BRANCH}\" " + + "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + + " parallelRegressionTest --stacktrace" } - stage('Slow Integration Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " allParallelSlowIntegrationTest --stacktrace" - } + } + stage('Slow Integration Test') { + steps { + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Dgit.branch=\"\${GIT_BRANCH}\" " + + "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + + " allParallelSlowIntegrationTest --stacktrace" } } } } - + } post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true, keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ } } } - diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index 0fdd5b0055..a64813c92f 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { @@ -40,6 +41,12 @@ pipeline { sh ".ci/check-api-changes.sh" } } + + stage('Deploy Nodes') { + steps { + sh "./gradlew --no-daemon jar deployNodes" + } + } } post { diff --git a/.ci/dev/publish-api-docs/Jenkinsfile b/.ci/dev/publish-api-docs/Jenkinsfile new file mode 100644 index 0000000000..d99d17ef44 --- /dev/null +++ b/.ci/dev/publish-api-docs/Jenkinsfile @@ -0,0 +1,35 @@ +@Library('corda-shared-build-pipeline-steps') + +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +pipeline { + agent { label 'standard' } + options { + ansiColor('xterm') + timestamps() + timeout(time: 3, unit: 'HOURS') + } + + environment { + ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" + } + + stages { + stage('Publish Archived API Docs to Artifactory') { + when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' } + steps { + sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs" + } + } + } + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} diff --git a/.ci/dev/publish-branch/Jenkinsfile.nightly b/.ci/dev/publish-branch/Jenkinsfile.nightly index 460117e500..485124ab66 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.nightly +++ b/.ci/dev/publish-branch/Jenkinsfile.nightly @@ -1,18 +1,34 @@ #!groovy +/** + * Jenkins pipeline to build Corda OS nightly snapshots + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/* +** calculate the stage for NexusIQ evaluation +** * build for snapshots +*/ +def nexusIqStage = "build" + pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() ansiColor('xterm') overrideIndexTriggers(false) - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } triggers { @@ -24,9 +40,29 @@ pipeline { // in the name ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Nightly to Artifactory" .replaceAll("/", " :: ") + DOCKER_URL = "https://index.docker.io/v1/" } stages { + stage('Sonatype Check') { + steps { + sh "./gradlew --no-daemon clean jar" + script { + sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" + } + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: manualApplication(nexusAppId), + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: nexusIqStage + ) + } + } + stage('Publish to Artifactory') { steps { rtServer ( @@ -58,6 +94,17 @@ pipeline { ) } } + + stage('Publish Nightly to Docker Hub') { + steps { + withCredentials([ + usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials', + usernameVariable: 'DOCKER_USERNAME', + passwordVariable: 'DOCKER_PASSWORD')]) { + sh "./gradlew pushOfficialImages" + } + } + } } diff --git a/.ci/dev/publish-branch/Jenkinsfile.preview b/.ci/dev/publish-branch/Jenkinsfile.preview index 1b39ae3237..e66deeabab 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.preview +++ b/.ci/dev/publish-branch/Jenkinsfile.preview @@ -11,8 +11,8 @@ pipeline { timestamps() ansiColor('xterm') overrideIndexTriggers(false) - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index f5bc830757..25c30fff31 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,29 +1,98 @@ +#!groovy +/** + * Jenkins pipeline to build Corda OS release branches and tags + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +/** + * Sense environment + */ +boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?version-properties" + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: //'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "jenkins-${groupId}-${artifactId}-${version}" + } + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: manualApplication(nexusAppId), + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: nexusIqStage + ) + } + } + + stage('Deploy Nodes') { + steps { + sh "./gradlew --no-daemon jar deployNodes" + } + } + + stage('Generate Build Image') { steps { withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { sh "./gradlew " + "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean jar deployNodes install pushBuildImage --stacktrace" + " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace" } sh "kubectl auth can-i get pods" } @@ -59,13 +128,57 @@ pipeline { } } } + + stage('Publish to Artifactory') { + when { + expression { isReleaseTag } + } + steps { + rtServer( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'corda-releases' + ) + rtGradleRun( + usesPlugin: true, + useWrapper: true, + switches: '-s --info', + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + rtPublishBuildInfo( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } + + stage('Publish Release to Docker Hub') { + when { + expression { !isInternalRelease && isReleaseTag } + } + steps { + withCredentials([ + usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials', + usernameVariable: 'DOCKER_USERNAME', + passwordVariable: 'DOCKER_PASSWORD')]) { + sh "./gradlew pushOfficialImages" + } + } + } } post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true script { try { @@ -97,38 +210,40 @@ pipeline { } } } - + script { - // We want to send a summary email, but want to limit to once per day. - // Comparing the dates of the previous and current builds achieves this, - // i.e. we will only send an email for the first build on a given day. - def prevBuildDate = new Date( - currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() - def currentBuildDate = new Date( - currentBuild.timeInMillis).clearTime() + if (!isReleaseTag) { + // We want to send a summary email, but want to limit to once per day. + // Comparing the dates of the previous and current builds achieves this, + // i.e. we will only send an email for the first build on a given day. + def prevBuildDate = new Date( + currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() + def currentBuildDate = new Date( + currentBuild.timeInMillis).clearTime() - if (prevBuildDate != currentBuildDate) { - def statusSymbol = '\u2753' - switch(currentBuild.result) { - case 'SUCCESS': - statusSymbol = '\u2705' - break; - case 'UNSTABLE': - case 'FAILURE': - statusSymbol = '\u274c' - break; - default: - break; + if (prevBuildDate != currentBuildDate) { + def statusSymbol = '\u2753' + switch(currentBuild.result) { + case 'SUCCESS': + statusSymbol = '\u2705' + break; + case 'UNSTABLE': + case 'FAILURE': + statusSymbol = '\u274c' + break; + default: + break; + } + + echo('First build for this date, sending summary email') + emailext to: '$DEFAULT_RECIPIENTS', + subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', + mimeType: 'text/html', + body: '${SCRIPT, template="groovy-html.template"}' + } else { + echo('Already sent summary email today, suppressing') } - - echo('First build for this date, sending summary email') - emailext to: '$DEFAULT_RECIPIENTS', - subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', - mimeType: 'text/html', - body: '${SCRIPT, template="groovy-html.template"}' - } else { - echo('Already sent summary email today, suppressing') } } } diff --git a/.ci/dev/unit/Jenkinsfile b/.ci/dev/unit/Jenkinsfile deleted file mode 100644 index b2d2d54393..0000000000 --- a/.ci/dev/unit/Jenkinsfile +++ /dev/null @@ -1,60 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda Pull Request - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Unit Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelUnitTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/unitTest', - description: 'Unit Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..2e77ac821f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,62 @@ +# All documentation should be reviewed by the technical writers +*.md @corda/technical-writers + +# By default anything under core or node-api is the Kernel team +core @corda/kernel +node-api @corda/kernel +node/src/main/kotlin/net/corda/node/internal @corda/kernel +node/src/main/kotlin/net/corda/node/services @corda/kernel + +# Determinstic components +core-deterministic @chrisr3 +jdk8u-deterministic @chrisr3 +node/djvm @chrisr3 +serialization-deterministic @chrisr3 +serialization-djvm @chrisr3 +serialization-tests @chrisr3 + +# Demobench defaults to Chris, but Viktor for the main code +tools/demobench @chrisr3 +tools/demobench/src/main/kotlin/net/corda/demobench @vkolomeyko + +# General Corda code + +client/rpc @vkolomeyko + +core/src/main/kotlin/net/corda/core/flows @dimosr +core/src/main/kotlin/net/corda/core/internal/notary @thschroeter +core/src/main/kotlin/net/corda/core/messaging @vkolomeyko + +node/src/integration-test/kotlin/net/corda/node/persistence @blsemo +node/src/integration-test/kotlin/net/corda/node/services/persistence @blsemo +node/src/main/kotlin/net/corda/node/internal/artemis @rekalov +node/src/main/kotlin/net/corda/node/services/identity @rekalov +node/src/main/kotlin/net/corda/node/services/keys @rekalov +node/src/main/kotlin/net/corda/node/services/messaging @dimosr +node/src/main/kotlin/net/corda/node/services/network @rekalov +node/src/main/kotlin/net/corda/node/services/persistence @blsemo +node/src/main/kotlin/net/corda/node/services/rpc @vkolomeyko +node/src/main/kotlin/net/corda/node/services/statemachine @lankydan +node/src/main/kotlin/net/corda/node/utilities/registration @rekalov +node/src/main/kotlin/net/corda/notary @thschroeter + +node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging @vkolomeyko +node-api/src/main/kotlin/net/corda/nodeapi/internal/crypto @rekalov +node-api/src/main/kotlin/net/corda/nodeapi/internal/cryptoservice @rekalov +node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle @vkolomeyko +node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence @blsemo +node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper @vkolomeyko +node-api/src/test/kotlin/net/corda/nodeapi/internal/bridging @rekalov + +common/logging/src/main/kotlin/net/corda/common/logging/errorReporting @JamesHR3 +common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting @JamesHR3 + +# Single file ownerships go at the end, as they are most specific and take precedence over other ownerships + +core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt @adelel1 +core/src/main/kotlin/net/corda/core/internal/AttachmentTrustCalculator.kt @adelel1 +core/src/main/kotlin/net/corda/core/internal/AttachmentWithContext.kt @adelel1 +core/src/main/kotlin/net/corda/core/internal/CertRole.kt @rekalov +core/src/main/kotlin/net/corda/core/node/services/AttachmentStorage.kt @adelel1 +core/src/main/kotlin/net/corda/core/node/services/IdentityService.kt @rekalov +core/src/main/kotlin/net/corda/core/node/services/NetworkMapCache.kt @rekalov diff --git a/Jenkinsfile b/Jenkinsfile index 604235e193..c27f461148 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -9,6 +9,7 @@ pipeline { options { timestamps() timeout(time: 3, unit: 'HOURS') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } environment { @@ -16,6 +17,9 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -26,8 +30,11 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean jar deployNodes pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelIntegrationTest --stacktrace" + " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace" } sh "kubectl auth can-i get pods" } @@ -72,7 +79,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/build.gradle b/build.gradle index 8ae347da27..6c421444de 100644 --- a/build.gradle +++ b/build.gradle @@ -105,7 +105,7 @@ buildscript { ext.eddsa_version = '0.3.0' ext.dependency_checker_version = '5.2.0' ext.commons_collections_version = '4.3' - ext.beanutils_version = '1.9.3' + ext.beanutils_version = '1.9.4' ext.crash_version = '1.7.4' ext.jsr305_version = constants.getProperty("jsr305Version") ext.shiro_version = '1.4.1' @@ -155,22 +155,39 @@ buildscript { ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion" repositories { mavenLocal() - mavenCentral() - jcenter() - maven { - url 'https://kotlin.bintray.com/kotlinx' - } - maven { - url "$artifactory_contextUrl/corda-dependencies-dev" - } - maven { - url "$artifactory_contextUrl/corda-releases" + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { + url 'https://kotlin.bintray.com/kotlinx' + } + maven { + url "${artifactory_contextUrl}/corda-dependencies-dev" + } + maven { + url "${artifactory_contextUrl}/corda-releases" + } } } dependencies { classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" classpath "org.jetbrains.kotlin:kotlin-allopen:$kotlin_version" - classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.4' classpath "net.corda.plugins:publish-utils:$gradle_plugins_version" classpath "net.corda.plugins:quasar-utils:$gradle_plugins_version" classpath "net.corda.plugins:cordformation:$gradle_plugins_version" @@ -204,7 +221,6 @@ plugins { apply plugin: 'project-report' apply plugin: 'com.github.ben-manes.versions' apply plugin: 'net.corda.plugins.publish-utils' -apply plugin: 'maven-publish' apply plugin: 'com.jfrog.artifactory' apply plugin: "com.bmuschko.docker-remote-api" apply plugin: "com.r3.dependx.dependxies" @@ -275,7 +291,7 @@ allprojects { toolVersion = "0.8.3" } - tasks.withType(JavaCompile) { + tasks.withType(JavaCompile).configureEach { options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" << "-Xlint:-options" << "-parameters" options.compilerArgs << '-XDenableSunApiLintControl' if (warnings_as_errors) { @@ -287,7 +303,7 @@ allprojects { options.encoding = 'UTF-8' } - tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) { + tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach { kotlinOptions { languageVersion = "1.2" apiVersion = "1.2" @@ -302,7 +318,7 @@ allprojects { task.dependsOn tasks.withType(AbstractCompile) } - tasks.withType(Jar) { task -> + tasks.withType(Jar).configureEach { task -> // Includes War and Ear manifest { attributes('Corda-Release-Version': corda_release_version) @@ -314,7 +330,7 @@ allprojects { } } - tasks.withType(Test) { + tasks.withType(Test).configureEach { forkEvery = 10 ignoreFailures = project.hasProperty('tests.ignoreFailures') ? project.property('tests.ignoreFailures').toBoolean() : false failFast = project.hasProperty('tests.failFast') ? project.property('tests.failFast').toBoolean() : false @@ -339,7 +355,7 @@ allprojects { systemProperty 'java.security.egd', 'file:/dev/./urandom' } - tasks.withType(Test) { + tasks.withType(Test).configureEach { if (name.contains("integrationTest")) { maxParallelForks = (System.env.CORDA_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_INT_TESTING_FORKS".toInteger() } @@ -357,11 +373,29 @@ allprojects { repositories { mavenLocal() - mavenCentral() - jcenter() - maven { url "$artifactory_contextUrl/corda-dependencies" } - maven { url 'https://repo.gradle.org/gradle/libs-releases' } - maven { url "$artifactory_contextUrl/corda-dev" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + maven { url 'https://repo.gradle.org/gradle/libs-releases' } + maven { url "${artifactory_contextUrl}/corda-dev" } + } } configurations { @@ -520,7 +554,7 @@ tasks.register('detektBaseline', JavaExec) { args(params) } -tasks.withType(Test) { +tasks.withType(Test).configureEach { reports.html.destination = file("${reporting.baseDir}/${name}") } @@ -626,7 +660,7 @@ dependxiesModule { skipTasks = "test,integrationTest,smokeTest,slowIntegrationTest" } -task generateApi(type: net.corda.plugins.GenerateApi) { +tasks.register('generateApi', net.corda.plugins.apiscanner.GenerateApi) { baseName = "api-corda" } @@ -662,7 +696,7 @@ if (file('corda-docs-only-build').exists() || (System.getenv('CORDA_DOCS_ONLY_BU } wrapper { - gradleVersion = "5.4.1" + gradleVersion = '5.6.4' distributionType = Wrapper.DistributionType.ALL } diff --git a/client/rpc/src/integration-test/java/net/corda/client/rpc/CordaRPCJavaClientTest.java b/client/rpc/src/integration-test/java/net/corda/client/rpc/CordaRPCJavaClientTest.java index 89fc874e1b..3f2f2ec680 100644 --- a/client/rpc/src/integration-test/java/net/corda/client/rpc/CordaRPCJavaClientTest.java +++ b/client/rpc/src/integration-test/java/net/corda/client/rpc/CordaRPCJavaClientTest.java @@ -13,6 +13,7 @@ import net.corda.node.internal.NodeWithInfo; import net.corda.testing.internal.InternalTestUtilsKt; import net.corda.testing.node.User; import net.corda.testing.node.internal.NodeBasedTest; +import net.corda.testing.node.internal.TestCordappInternal; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -30,10 +31,18 @@ import static net.corda.node.services.Permissions.invokeRpc; import static net.corda.node.services.Permissions.startFlow; import static net.corda.testing.core.TestConstants.ALICE_NAME; import static net.corda.testing.core.TestConstants.DUMMY_NOTARY_NAME; +import static net.corda.testing.node.internal.InternalTestUtilsKt.FINANCE_CORDAPPS; +import static net.corda.testing.node.internal.InternalTestUtilsKt.cordappWithPackages; public class CordaRPCJavaClientTest extends NodeBasedTest { public CordaRPCJavaClientTest() { - super(Arrays.asList("net.corda.finance.contracts", CashSchemaV1.class.getPackage().getName()), Collections.singletonList(DUMMY_NOTARY_NAME)); + super(cordapps(), Collections.singletonList(DUMMY_NOTARY_NAME)); + } + + private static Set cordapps() { + Set cordapps = new HashSet<>(FINANCE_CORDAPPS); + cordapps.add(cordappWithPackages(CashSchemaV1.class.getPackage().getName())); + return cordapps; } private List perms = Arrays.asList( diff --git a/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/CordaRPCClientTest.kt b/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/CordaRPCClientTest.kt index 59f05cb8e2..1ebd6f29b5 100644 --- a/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/CordaRPCClientTest.kt +++ b/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/CordaRPCClientTest.kt @@ -40,6 +40,7 @@ import net.corda.testing.core.expect import net.corda.testing.core.expectEvents import net.corda.testing.core.sequence import net.corda.testing.node.User +import net.corda.testing.node.internal.FINANCE_CORDAPPS import net.corda.testing.node.internal.NodeBasedTest import net.corda.testing.node.internal.ProcessUtilities import net.corda.testing.node.internal.poll @@ -62,7 +63,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertTrue -class CordaRPCClientTest : NodeBasedTest(listOf("net.corda.finance"), notaries = listOf(DUMMY_NOTARY_NAME)) { +class CordaRPCClientTest : NodeBasedTest(FINANCE_CORDAPPS, notaries = listOf(DUMMY_NOTARY_NAME)) { companion object { val rpcUser = User("user1", "test", permissions = setOf(all())) val log = contextLogger() diff --git a/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt b/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt index 8084484735..22b11cee50 100644 --- a/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt +++ b/client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt @@ -2,9 +2,7 @@ package net.corda.client.rpc import net.corda.core.context.Actor import net.corda.core.context.Trace -import net.corda.core.internal.packageName import net.corda.core.messaging.CordaRPCOps -import net.corda.finance.schemas.CashSchemaV1 import net.corda.node.internal.NodeWithInfo import net.corda.node.services.Permissions import net.corda.testing.core.ALICE_NAME @@ -14,7 +12,7 @@ import org.assertj.core.api.Assertions.assertThat import org.junit.Before import org.junit.Test -class FlowsExecutionModeTests : NodeBasedTest(emptyList()) { +class FlowsExecutionModeTests : NodeBasedTest() { private val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) private lateinit var node: NodeWithInfo diff --git a/client/rpc/src/integration-test/kotlin/net/corda/client/rpcreconnect/CordaRPCClientReconnectionTest.kt b/client/rpc/src/integration-test/kotlin/net/corda/client/rpcreconnect/CordaRPCClientReconnectionTest.kt index 70ae13731f..fbf55e194b 100644 --- a/client/rpc/src/integration-test/kotlin/net/corda/client/rpcreconnect/CordaRPCClientReconnectionTest.kt +++ b/client/rpc/src/integration-test/kotlin/net/corda/client/rpcreconnect/CordaRPCClientReconnectionTest.kt @@ -15,6 +15,7 @@ import net.corda.finance.DOLLARS import net.corda.finance.contracts.asset.Cash import net.corda.finance.flows.CashIssueFlow import net.corda.node.services.Permissions +import net.corda.nodeapi.exceptions.RejectedCommandException import net.corda.testing.core.CHARLIE_NAME import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.NodeHandle @@ -49,6 +50,38 @@ class CordaRPCClientReconnectionTest { val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) } + + + + + @Test(timeout=300_000) + fun `rpc node start when FlowsDrainingModeEnabled throws RejectedCommandException and won't attempt to reconnect`() { + driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS)) { + val address = NetworkHostAndPort("localhost", portAllocator.nextPort()) + + fun startNode(): NodeHandle { + return startNode( + providedName = CHARLIE_NAME, + rpcUsers = listOf(CordaRPCClientTest.rpcUser), + customOverrides = mapOf("rpcSettings.address" to address.toString()) + ).getOrThrow() + } + + val node = startNode() + val client = CordaRPCClient(node.rpcAddress, + config.copy(maxReconnectAttempts = 1)) + + (client.start(rpcUser.username, rpcUser.password, gracefulReconnect = gracefulReconnect)).use { + val rpcOps = it.proxy as ReconnectingCordaRPCOps + rpcOps.setFlowsDrainingModeEnabled(true) + + assertThatThrownBy { rpcOps.startTrackedFlow(::CashIssueFlow, 10.DOLLARS, OpaqueBytes.of(0), defaultNotaryIdentity).returnValue.get() } + .isInstanceOf(RejectedCommandException::class.java).hasMessage("Node is draining before shutdown. Cannot start new flows through RPC.") + } + } + } + + @Test(timeout=300_000) fun `rpc client calls and returned observables continue working when the server crashes and restarts`() { driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS)) { diff --git a/client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt b/client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt index 6308ae1ebf..bf858c9290 100644 --- a/client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt +++ b/client/rpc/src/main/kotlin/net/corda/client/rpc/CordaRPCClient.kt @@ -158,7 +158,8 @@ open class CordaRPCClientConfiguration @JvmOverloads constructor( open val connectionRetryIntervalMultiplier: Double = 1.5, /** - * Maximum reconnect attempts on failover or disconnection. The default is -1 which means unlimited. + * Maximum reconnect attempts on failover or disconnection. + * Any negative value would mean that there will be an infinite number of reconnect attempts. */ open val maxReconnectAttempts: Int = unlimitedReconnectAttempts, diff --git a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt index 9dc7d5cc70..88c059ff9b 100644 --- a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt +++ b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt @@ -76,10 +76,10 @@ import kotlin.reflect.jvm.javaMethod * forwarded to the [UnicastSubject]. Note that the observations themselves may contain further [Observable]s, which are * handled in the same way. * - * To do the above we take advantage of Kryo's datastructure traversal. When the client is deserialising a message from - * the server that may contain Observables it is supplied with an [ObservableContext] that exposes the map used to demux - * the observations. When an [Observable] is encountered during traversal a new [UnicastSubject] is added to the map and - * we carry on. Each observation later contains the corresponding Observable ID, and we just forward that to the + * To do the above we take advantage of serialisation data structure traversal. When the client is deserialising a message from + * the server that may contain [Observable]s, it is supplied with an [ObservableContext] that exposes the map used to demux + * the observations. When a new [Observable] is encountered during traversal a new [UnicastSubject] is added to the map and + * we carry on. Each observation later contains the corresponding [Observable] ID, and we just forward that to the * associated [UnicastSubject]. * * The client may signal that it no longer consumes a particular [Observable]. This may be done explicitly by @@ -88,12 +88,12 @@ import kotlin.reflect.jvm.javaMethod * The cleanup happens in batches using a dedicated reaper, scheduled on [reaperExecutor]. * * The client will attempt to failover in case the server become unreachable. Depending on the [ServerLocator] instance - * passed in the constructor, failover is either handle at Artemis level or client level. If only one transport + * passed in the constructor, failover is either handled at Artemis level or client level. If only one transport * was used to create the [ServerLocator], failover is handled by Artemis (retrying based on [CordaRPCClientConfiguration]. * If a list of transport configurations was used, failover is handled locally. Artemis is able to do it, however the * brokers on server side need to be configured in HA mode and the [ServerLocator] needs to be created with HA as well. */ -class RPCClientProxyHandler( +internal class RPCClientProxyHandler( private val rpcConfiguration: CordaRPCClientConfiguration, private val rpcUsername: String, private val rpcPassword: String, @@ -247,7 +247,7 @@ class RPCClientProxyHandler( try { sessionFactory = serverLocator.createSessionFactory() } catch (e: ActiveMQNotConnectedException) { - throw (RPCException("Cannot connect to server(s). Tried with all available servers.", e)) + throw RPCException("Cannot connect to server(s). Tried with all available servers.", e) } // Depending on how the client is constructed, connection failure is treated differently if (serverLocator.staticTransportConfigurations.size == 1) { @@ -380,9 +380,11 @@ class RPCClientProxyHandler( is RPCApi.ServerToClient.Observation -> { val observable: UnicastSubject>? = observableContext.observableMap.getIfPresent(serverToClient.id) if (observable == null) { - log.debug("Observation ${serverToClient.content} arrived to unknown Observable with ID ${serverToClient.id}. " + - "This may be due to an observation arriving before the server was " + - "notified of observable shutdown") + log.debug { + "Observation ${serverToClient.content} arrived to unknown Observable with ID ${serverToClient.id}. " + + "This may be due to an observation arriving before the server was " + + "notified of observable shutdown" + } } else { // We schedule the onNext() on an executor sticky-pooled based on the Observable ID. observationExecutorPool.run(serverToClient.id) { executor -> @@ -461,7 +463,7 @@ class RPCClientProxyHandler( } } observableContext.observableMap.invalidateAll() - rpcReplyMap.forEach { _, replyFuture -> + rpcReplyMap.forEach { (_, replyFuture) -> replyFuture.setException(ConnectionFailureException()) } @@ -528,23 +530,26 @@ class RPCClientProxyHandler( } private fun attemptReconnect() { - var reconnectAttempts = rpcConfiguration.maxReconnectAttempts.times(serverLocator.staticTransportConfigurations.size) + // This can be a negative number as `rpcConfiguration.maxReconnectAttempts = -1` means infinite number of re-connects + val maxReconnectCount = rpcConfiguration.maxReconnectAttempts.times(serverLocator.staticTransportConfigurations.size) + log.debug { "maxReconnectCount = $maxReconnectCount" } + var reconnectAttempt = 1 var retryInterval = rpcConfiguration.connectionRetryInterval val maxRetryInterval = rpcConfiguration.connectionMaxRetryInterval - var transportIterator = serverLocator.staticTransportConfigurations.iterator() - while (transportIterator.hasNext() && reconnectAttempts != 0) { - val transport = transportIterator.next() - if (!transportIterator.hasNext()) - transportIterator = serverLocator.staticTransportConfigurations.iterator() + fun shouldRetry(reconnectAttempt: Int) = + if (maxReconnectCount < 0) true else reconnectAttempt <= maxReconnectCount - log.debug("Trying to connect using ${transport.params}") + while (shouldRetry(reconnectAttempt)) { + val transport = serverLocator.staticTransportConfigurations.let { it[(reconnectAttempt - 1) % it.size] } + + log.debug { "Trying to connect using ${transport.params}" } try { if (!serverLocator.isClosed) { sessionFactory = serverLocator.createSessionFactory(transport) } else { log.warn("Stopping reconnect attempts.") - log.debug("Server locator is closed or garbage collected. Proxy may have been closed during reconnect.") + log.debug { "Server locator is closed or garbage collected. Proxy may have been closed during reconnect." } break } } catch (e: ActiveMQException) { @@ -552,12 +557,12 @@ class RPCClientProxyHandler( Thread.sleep(retryInterval.toMillis()) } catch (e: InterruptedException) {} // Could not connect, try with next server transport. - reconnectAttempts-- + reconnectAttempt++ retryInterval = minOf(maxRetryInterval, retryInterval.times(rpcConfiguration.connectionRetryIntervalMultiplier.toLong())) continue } - log.debug("Connected successfully after $reconnectAttempts attempts using ${transport.params}.") + log.debug { "Connected successfully after $reconnectAttempt attempts using ${transport.params}." } log.info("RPC server available.") sessionFactory!!.addFailoverListener(this::haFailoverHandler) initSessions() @@ -566,8 +571,12 @@ class RPCClientProxyHandler( break } - if (reconnectAttempts == 0 || sessionFactory == null) - log.error("Could not reconnect to the RPC server.") + val maxReconnectReached = !shouldRetry(reconnectAttempt) + if (maxReconnectReached || sessionFactory == null) { + val errMessage = "Could not reconnect to the RPC server after trying $reconnectAttempt times." + + if (sessionFactory != null) "" else " It was never possible to to establish connection with any of the endpoints." + log.error(errMessage) + } } private fun initSessions() { @@ -620,10 +629,11 @@ class RPCClientProxyHandler( sendingEnabled.set(false) log.warn("Terminating observables.") val m = observableContext.observableMap.asMap() + val connectionFailureException = ConnectionFailureException() m.keys.forEach { k -> observationExecutorPool.run(k) { try { - m[k]?.onError(ConnectionFailureException()) + m[k]?.onError(connectionFailureException) } catch (e: Exception) { log.error("Unexpected exception when RPC connection failure handling", e) } @@ -631,8 +641,8 @@ class RPCClientProxyHandler( } observableContext.observableMap.invalidateAll() - rpcReplyMap.forEach { _, replyFuture -> - replyFuture.setException(ConnectionFailureException()) + rpcReplyMap.forEach { (_, replyFuture) -> + replyFuture.setException(connectionFailureException) } rpcReplyMap.clear() @@ -666,5 +676,5 @@ class RPCClientProxyHandler( } } -private typealias RpcReplyMap = ConcurrentHashMap> +private typealias RpcReplyMap = ConcurrentHashMap> diff --git a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt index af3c7ab1b5..ff833dd03b 100644 --- a/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt +++ b/client/rpc/src/main/kotlin/net/corda/client/rpc/internal/ReconnectingCordaRPCOps.kt @@ -325,8 +325,8 @@ class ReconnectingCordaRPCOps private constructor( } when (e.targetException) { is RejectedCommandException -> { - log.warn("Node is being shutdown. Operation ${method.name} rejected. Retrying when node is up...", e) - reconnectingRPCConnection.reconnectOnError(e) + log.warn("Node is being shutdown. Operation ${method.name} rejected. Shutting down...", e) + throw e.targetException } is ConnectionFailureException -> { log.warn("Failed to perform operation ${method.name}. Connection dropped. Retrying....", e) diff --git a/common/logging/src/main/kotlin/net/corda/common/logging/ExceptionsErrorCodeFunctions.kt b/common/logging/src/main/kotlin/net/corda/common/logging/ExceptionsErrorCodeFunctions.kt index a34e436bb8..3064fce1cc 100644 --- a/common/logging/src/main/kotlin/net/corda/common/logging/ExceptionsErrorCodeFunctions.kt +++ b/common/logging/src/main/kotlin/net/corda/common/logging/ExceptionsErrorCodeFunctions.kt @@ -32,8 +32,9 @@ fun Message.withErrorCodeFor(error: Throwable?, level: Level): Message { return when { error != null && level.isInRange(Level.FATAL, Level.WARN) -> { + val logMessage = this.formattedMessage val message = error.walkExceptionCausedByList().asSequence().mapNotNull(Throwable::message).joinToString(" - ") - CompositeMessage("$message [errorCode=${error.errorCode()}, moreInformationAt=${error.errorCodeLocationUrl()}]", format, parameters, throwable) + CompositeMessage("$logMessage - $message [errorCode=${error.errorCode()}, moreInformationAt=${error.errorCodeLocationUrl()}]", format, parameters, throwable) } else -> this } diff --git a/common/logging/src/main/kotlin/net/corda/common/logging/errorReporting/ErrorReporterImpl.kt b/common/logging/src/main/kotlin/net/corda/common/logging/errorReporting/ErrorReporterImpl.kt index 0e508959e8..553303870e 100644 --- a/common/logging/src/main/kotlin/net/corda/common/logging/errorReporting/ErrorReporterImpl.kt +++ b/common/logging/src/main/kotlin/net/corda/common/logging/errorReporting/ErrorReporterImpl.kt @@ -1,6 +1,7 @@ package net.corda.common.logging.errorReporting import org.slf4j.Logger +import java.lang.Exception import java.text.MessageFormat import java.util.* @@ -31,6 +32,10 @@ internal class ErrorReporterImpl(private val resourceLocation: String, override fun report(error: ErrorCode<*>, logger: Logger) { val errorResource = ErrorResource.fromErrorCode(error, resourceLocation, locale) val message = "${errorResource.getErrorMessage(error.parameters.toTypedArray())} ${getErrorInfo(error)}" - logger.error(message) + if (error is Exception) { + logger.error(message, error) + } else { + logger.error(message) + } } } \ No newline at end of file diff --git a/common/logging/src/main/resources/error-codes/database-failed-startup.properties b/common/logging/src/main/resources/error-codes/database-failed-startup.properties index 996de3ba76..2c21cd3e9a 100644 --- a/common/logging/src/main/resources/error-codes/database-failed-startup.properties +++ b/common/logging/src/main/resources/error-codes/database-failed-startup.properties @@ -1,4 +1,4 @@ -errorTemplate = Failed to create the datasource. See the logs for further information and the cause. +errorTemplate = Failed to create the datasource: {0}. See the logs for further information and the cause. shortDescription = The datasource could not be created for unknown reasons. actionsToFix = The logs in the logs directory should contain more information on what went wrong. aliases = \ No newline at end of file diff --git a/common/logging/src/main/resources/error-codes/database-failed-startup_en_US.properties b/common/logging/src/main/resources/error-codes/database-failed-startup_en_US.properties index 1abe8840bb..194292abf5 100644 --- a/common/logging/src/main/resources/error-codes/database-failed-startup_en_US.properties +++ b/common/logging/src/main/resources/error-codes/database-failed-startup_en_US.properties @@ -1,3 +1,3 @@ -errorTemplate = Failed to create the datasource. See the logs for further information and the cause. +errorTemplate = Failed to create the datasource: {0}. See the logs for further information and the cause. shortDescription = The datasource could not be created for unknown reasons. actionsToFix = The logs in the logs directory should contain more information on what went wrong. \ No newline at end of file diff --git a/common/logging/src/test/kotlin/net/corda/commmon/logging/ExceptionsErrorCodeFunctionsTest.kt b/common/logging/src/test/kotlin/net/corda/commmon/logging/ExceptionsErrorCodeFunctionsTest.kt new file mode 100644 index 0000000000..45b44abedb --- /dev/null +++ b/common/logging/src/test/kotlin/net/corda/commmon/logging/ExceptionsErrorCodeFunctionsTest.kt @@ -0,0 +1,34 @@ +package net.corda.commmon.logging + +import com.natpryce.hamkrest.assertion.assertThat +import com.natpryce.hamkrest.contains +import net.corda.common.logging.withErrorCodeFor +import org.apache.logging.log4j.Level +import org.apache.logging.log4j.message.SimpleMessage +import org.junit.Test +import kotlin.test.assertEquals + +class ExceptionsErrorCodeFunctionsTest { + + @Test(timeout=3_000) + fun `error code for message prints out message and full stack trace`() { + val originalMessage = SimpleMessage("This is a test message") + var previous: Exception? = null + val throwables = (0..10).map { + val current = TestThrowable(it, previous) + previous = current + current + } + val exception = throwables.last() + val message = originalMessage.withErrorCodeFor(exception, Level.ERROR) + assertThat(message.formattedMessage, contains("This is a test message".toRegex())) + for (i in (0..10)) { + assertThat(message.formattedMessage, contains("This is exception $i".toRegex())) + } + assertEquals(message.format, originalMessage.format) + assertEquals(message.parameters, originalMessage.parameters) + assertEquals(message.throwable, originalMessage.throwable) + } + + private class TestThrowable(index: Int, cause: Exception?) : Exception("This is exception $index", cause) +} \ No newline at end of file diff --git a/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/DatabaseErrorsTest.kt b/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/DatabaseErrorsTest.kt index d8697e9415..753325a0cd 100644 --- a/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/DatabaseErrorsTest.kt +++ b/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/DatabaseErrorsTest.kt @@ -6,7 +6,7 @@ import java.net.InetAddress class DatabaseErrorsTest : ErrorCodeTest(NodeDatabaseErrors::class.java) { override val dataForCodes = mapOf( NodeDatabaseErrors.COULD_NOT_CONNECT to listOf(), - NodeDatabaseErrors.FAILED_STARTUP to listOf(), + NodeDatabaseErrors.FAILED_STARTUP to listOf("This is a test message"), NodeDatabaseErrors.MISSING_DRIVER to listOf(), NodeDatabaseErrors.PASSWORD_REQUIRED_FOR_H2 to listOf(InetAddress.getLocalHost()) ) diff --git a/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/ErrorReporterImplTest.kt b/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/ErrorReporterImplTest.kt index 40efb4e164..95f9d38141 100644 --- a/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/ErrorReporterImplTest.kt +++ b/common/logging/src/test/kotlin/net/corda/commmon/logging/errorReporting/ErrorReporterImplTest.kt @@ -7,6 +7,7 @@ import net.corda.common.logging.errorReporting.ErrorContextProvider import net.corda.common.logging.errorReporting.ErrorReporterImpl import org.junit.After import org.junit.Test +import org.mockito.ArgumentMatchers.any import org.mockito.ArgumentMatchers.anyString import org.mockito.Mockito import org.slf4j.Logger @@ -24,6 +25,7 @@ class ErrorReporterImplTest { private val loggerMock = Mockito.mock(Logger::class.java).also { Mockito.`when`(it.error(anyString())).then { logs.addAll(it.arguments) } + Mockito.`when`(it.error(anyString(), any(Exception::class.java))).then { params -> logs.addAll(params.arguments) } } private val contextProvider: ErrorContextProvider = object : ErrorContextProvider { @@ -39,7 +41,8 @@ class ErrorReporterImplTest { private enum class TestErrors : ErrorCodes { CASE1, CASE2, - CASE_3; + CASE_3, + CASE4; override val namespace = TestNamespaces.TEST.toString() } @@ -59,6 +62,11 @@ class ErrorReporterImplTest { override val parameters = listOf() } + private class TestError4(cause: Exception?) : Exception("This is test error 4", cause), ErrorCode { + override val code = TestErrors.CASE4 + override val parameters = listOf() + } + private fun createReporterImpl(localeTag: String?) : ErrorReporterImpl { val locale = if (localeTag != null) Locale.forLanguageTag(localeTag) else Locale.getDefault() return ErrorReporterImpl("errorReporting", locale, contextProvider) @@ -118,4 +126,12 @@ class ErrorReporterImplTest { testReporter.report(error, loggerMock) assertEquals(listOf("This is the third test message [Code: test-case-3 URL: $TEST_URL/en-US]"), logs) } + + @Test(timeout = 3_000) + fun `exception based error code logs the stack trace`() { + val error = TestError4(Exception("A test exception")) + val testReporter = createReporterImpl("en-US") + testReporter.report(error, loggerMock) + assertEquals(listOf("This is the fourth test message [Code: test-case4 URL: $TEST_URL/en-US]", error), logs) + } } \ No newline at end of file diff --git a/common/logging/src/test/resources/errorReporting/test-case4.properties b/common/logging/src/test/resources/errorReporting/test-case4.properties new file mode 100644 index 0000000000..e4911daacf --- /dev/null +++ b/common/logging/src/test/resources/errorReporting/test-case4.properties @@ -0,0 +1,4 @@ +errorTemplate = This is the fourth test message +shortDescription = Test description +actionsToFix = Actions +aliases = \ No newline at end of file diff --git a/common/logging/src/test/resources/errorReporting/test-case4_en_US.properties b/common/logging/src/test/resources/errorReporting/test-case4_en_US.properties new file mode 100644 index 0000000000..e4911daacf --- /dev/null +++ b/common/logging/src/test/resources/errorReporting/test-case4_en_US.properties @@ -0,0 +1,4 @@ +errorTemplate = This is the fourth test message +shortDescription = Test description +actionsToFix = Actions +aliases = \ No newline at end of file diff --git a/constants.properties b/constants.properties index b4a092855f..c9877793fb 100644 --- a/constants.properties +++ b/constants.properties @@ -4,7 +4,7 @@ cordaVersion=4.6 versionSuffix=SNAPSHOT -gradlePluginsVersion=5.0.10 +gradlePluginsVersion=5.0.11 kotlinVersion=1.2.71 java8MinUpdateVersion=171 # ***************************************************************# @@ -25,7 +25,7 @@ classgraphVersion=4.8.78 disruptorVersion=3.4.2 typesafeConfigVersion=1.3.4 jsr305Version=3.0.2 -artifactoryPluginVersion=4.7.3 +artifactoryPluginVersion=4.16.1 snakeYamlVersion=1.19 caffeineVersion=2.7.0 metricsVersion=4.1.0 diff --git a/core-deterministic/README.md b/core-deterministic/README.md new file mode 100644 index 0000000000..766d178882 --- /dev/null +++ b/core-deterministic/README.md @@ -0,0 +1,2 @@ +## corda-core-deterministic. +This artifact is a deterministic subset of the binary contents of `corda-core`. diff --git a/core-deterministic/build.gradle b/core-deterministic/build.gradle index 636ce86800..5be42d8084 100644 --- a/core-deterministic/build.gradle +++ b/core-deterministic/build.gradle @@ -54,8 +54,8 @@ tasks.named('jar', Jar) { enabled = false } -def coreJarTask = tasks.getByPath(':core:jar') -def originalJar = coreJarTask.outputs.files.singleFile +def coreJarTask = project(':core').tasks.named('jar', Jar) +def originalJar = coreJarTask.map { it.outputs.files.singleFile } def patchCore = tasks.register('patchCore', Zip) { dependsOn coreJarTask @@ -132,7 +132,7 @@ def jarFilter = tasks.register('jarFilter', JarFilterTask) { } } -task determinise(type: ProGuardTask) { +def determinise = tasks.register('determinise', ProGuardTask) { injars jarFilter outjars file("$buildDir/proguard/$jarBaseName-${project.version}.jar") @@ -166,17 +166,20 @@ task determinise(type: ProGuardTask) { keepclassmembers 'class net.corda.core.** { public synthetic ; }' } -task metafix(type: MetaFixerTask) { +def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) + +def metafix = tasks.register('metafix', MetaFixerTask) { outputDir file("$buildDir/libs") jars determinise suffix "" // Strip timestamps from the JAR to make it reproducible. preserveTimestamps = false + finalizedBy checkDeterminism } // DOCSTART 01 -def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { +checkDeterminism.configure { dependsOn jdkTask injars metafix @@ -197,20 +200,31 @@ def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { // DOCEND 01 defaultTasks "determinise" -determinise.finalizedBy metafix -metafix.finalizedBy checkDeterminism -assemble.dependsOn checkDeterminism +determinise.configure { + finalizedBy metafix +} +tasks.named('assemble') { + dependsOn checkDeterminism +} -def deterministicJar = metafix.outputs.files.singleFile +def deterministicJar = metafix.map { it.outputs.files.singleFile } artifacts { - deterministicArtifacts file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix - publish file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix + deterministicArtifacts deterministicJar + publish deterministicJar +} + +tasks.named('sourceJar', Jar) { + from 'README.md' + include 'README.md' +} + +tasks.named('javadocJar', Jar) { + from 'README.md' + include 'README.md' } publish { dependenciesFrom configurations.deterministicArtifacts - publishSources = false - publishJavadoc = false name jarBaseName } diff --git a/core-tests/build.gradle b/core-tests/build.gradle index 8d1919f474..c5e184e448 100644 --- a/core-tests/build.gradle +++ b/core-tests/build.gradle @@ -1,7 +1,6 @@ apply plugin: 'kotlin' apply plugin: 'kotlin-jpa' apply plugin: 'net.corda.plugins.quasar-utils' -apply plugin: 'net.corda.plugins.publish-utils' description 'Corda core tests' @@ -99,7 +98,7 @@ configurations { testArtifacts.extendsFrom testRuntimeClasspath } -tasks.withType(Test) { +tasks.withType(Test).configureEach { // fork a new test process for every test class forkEvery = 10 } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt index 4f1406711a..d5f36ec022 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt @@ -53,7 +53,7 @@ class ReceiveFinalityFlowTest { val paymentReceiverId = paymentReceiverFuture.getOrThrow() assertThat(bob.services.vaultService.queryBy>().states).isEmpty() - bob.assertFlowSentForObservationDueToConstraintError(paymentReceiverId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(paymentReceiverId) // Restart Bob with the contracts CorDapp so that it can recover from the error bob = mockNet.restartNode(bob, @@ -71,7 +71,7 @@ class ReceiveFinalityFlowTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -79,6 +79,6 @@ class ReceiveFinalityFlowTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt index 5cfaf252cb..4ca58d6b46 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt @@ -55,7 +55,7 @@ class AttachmentsClassLoaderSerializationTests { arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! }, testNetworkParameters(), SecureHash.zeroHash, - { attachmentTrustCalculator.calculate(it) }) { classLoader -> + { attachmentTrustCalculator.calculate(it) }, attachmentsClassLoaderCache = null) { classLoader -> val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader) val contract = contractClass.getDeclaredConstructor().newInstance() as Contract assertEquals("helloworld", contract.declaredField("magicString").value) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt index fcc081efb6..986b6052ef 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt @@ -23,6 +23,7 @@ import net.corda.core.internal.inputStream import net.corda.core.node.NetworkParameters import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.internal.AttachmentsClassLoader +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.testing.common.internal.testNetworkParameters import net.corda.node.services.attachments.NodeAttachmentTrustCalculator import net.corda.testing.contracts.DummyContract @@ -521,6 +522,7 @@ class AttachmentsClassLoaderTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) val transaction = createLedgerTransaction( inputs, outputs, @@ -532,7 +534,8 @@ class AttachmentsClassLoaderTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt index ec14b84029..500cfa9816 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt @@ -10,6 +10,7 @@ import net.corda.core.internal.AbstractAttachment import net.corda.core.internal.TESTDSL_UPLOADER import net.corda.core.internal.createLedgerTransaction import net.corda.core.node.NotaryInfo +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.WireTransaction import net.corda.testing.common.internal.testNetworkParameters @@ -18,6 +19,7 @@ import net.corda.testing.core.* import net.corda.testing.internal.createWireTransaction import net.corda.testing.internal.fakeAttachment import net.corda.coretesting.internal.rigorousMock +import net.corda.testing.internal.TestingNamedCacheFactory import org.junit.Rule import org.junit.Test import java.math.BigInteger @@ -131,6 +133,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) val transaction = createLedgerTransaction( inputs, outputs, @@ -142,7 +145,8 @@ class TransactionTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() @@ -183,6 +187,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) fun buildTransaction() = createLedgerTransaction( inputs, @@ -195,7 +200,8 @@ class TransactionTests { privacySalt, testNetworkParameters(notaries = listOf(NotaryInfo(DUMMY_NOTARY, true))), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) assertFailsWith { buildTransaction().verify() } diff --git a/core/src/main/kotlin/net/corda/core/contracts/Structures.kt b/core/src/main/kotlin/net/corda/core/contracts/Structures.kt index 16d2c8cd85..bfae2f8900 100644 --- a/core/src/main/kotlin/net/corda/core/contracts/Structures.kt +++ b/core/src/main/kotlin/net/corda/core/contracts/Structures.kt @@ -89,6 +89,7 @@ interface OwnableState : ContractState { // DOCEND 3 /** Something which is scheduled to happen at a point in time. */ +@KeepForDJVM interface Scheduled { val scheduledAt: Instant } @@ -101,6 +102,7 @@ interface Scheduled { * lifecycle processing needs to take place. e.g. a fixing or a late payment etc. */ @CordaSerializable +@KeepForDJVM data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instant) : Scheduled /** @@ -115,7 +117,7 @@ data class ScheduledStateRef(val ref: StateRef, override val scheduledAt: Instan * for a particular [ContractState] have been processed/fired etc. If the activity is not "on ledger" then the * scheduled activity shouldn't be either. */ -@DeleteForDJVM +@KeepForDJVM data class ScheduledActivity(val logicRef: FlowLogicRef, override val scheduledAt: Instant) : Scheduled // DOCSTART 2 @@ -134,7 +136,7 @@ interface LinearState : ContractState { val linearId: UniqueIdentifier } // DOCEND 2 -@DeleteForDJVM +@KeepForDJVM interface SchedulableState : ContractState { /** * Indicate whether there is some activity to be performed at some future point in time with respect to this diff --git a/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt b/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt index 7781c38b95..1b08620e2a 100644 --- a/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt +++ b/core/src/main/kotlin/net/corda/core/flows/FlowLogicRef.kt @@ -1,7 +1,9 @@ package net.corda.core.flows import net.corda.core.CordaInternal +import net.corda.core.DeleteForDJVM import net.corda.core.DoNotImplement +import net.corda.core.KeepForDJVM import net.corda.core.serialization.CordaSerializable /** @@ -11,11 +13,13 @@ import net.corda.core.serialization.CordaSerializable * the flow to run at the scheduled time. */ @DoNotImplement +@KeepForDJVM interface FlowLogicRefFactory { /** * Construct a FlowLogicRef. This is intended for cases where the calling code has the relevant class already * and can provide it directly. */ + @DeleteForDJVM fun create(flowClass: Class>, vararg args: Any?): FlowLogicRef /** @@ -30,12 +34,14 @@ interface FlowLogicRefFactory { * [SchedulableFlow] annotation. */ @CordaInternal + @DeleteForDJVM fun createForRPC(flowClass: Class>, vararg args: Any?): FlowLogicRef /** * Converts a [FlowLogicRef] object that was obtained from the calls above into a [FlowLogic], after doing some * validation to ensure it points to a legitimate flow class. */ + @DeleteForDJVM fun toFlowLogic(ref: FlowLogicRef): FlowLogic<*> } @@ -59,4 +65,5 @@ class IllegalFlowLogicException(val type: String, msg: String) : // TODO: align this with the existing [FlowRef] in the bank-side API (probably replace some of the API classes) @CordaSerializable @DoNotImplement +@KeepForDJVM interface FlowLogicRef \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt index 5897984c1d..2e59429fb5 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt @@ -5,6 +5,7 @@ import net.corda.core.DeleteForDJVM import net.corda.core.internal.notary.NotaryService import net.corda.core.node.ServiceHub import net.corda.core.node.StatesToRecord +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import java.util.concurrent.ExecutorService // TODO: This should really be called ServiceHubInternal but that name is already taken by net.corda.node.services.api.ServiceHubInternal. @@ -21,6 +22,8 @@ interface ServiceHubCoreInternal : ServiceHub { val notaryService: NotaryService? fun createTransactionsResolver(flow: ResolveTransactionsFlow): TransactionsResolver + + val attachmentsClassLoaderCache: AttachmentsClassLoaderCache } interface TransactionsResolver { diff --git a/core/src/main/kotlin/net/corda/core/internal/concurrent/CordaFutureImpl.kt b/core/src/main/kotlin/net/corda/core/internal/concurrent/CordaFutureImpl.kt index dcaceb2295..0f62fd752f 100644 --- a/core/src/main/kotlin/net/corda/core/internal/concurrent/CordaFutureImpl.kt +++ b/core/src/main/kotlin/net/corda/core/internal/concurrent/CordaFutureImpl.kt @@ -27,6 +27,12 @@ fun CordaFuture.thenMatch(success: (V) -> W, failure: (Throwabl /** When this future is done and the outcome is failure, log the throwable. */ fun CordaFuture<*>.andForget(log: Logger) = thenMatch({}, { log.error("Background task failed:", it) }) +/** + * Returns a future that will also apply the passed closure when it completes. + * + * @param accept A function to execute when completing the original future. + * @return A future returning the same result as the original future that this function was executed on. + */ fun CordaFuture.doOnComplete(accept: (RESULT) -> Unit): CordaFuture { return CordaFutureImpl().also { result -> thenMatch({ diff --git a/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt b/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt index 35e4d0a65d..d511ba7860 100644 --- a/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt +++ b/core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt @@ -47,7 +47,7 @@ data class CordappImpl( } companion object { - fun jarName(url: URL): String = url.toPath().fileName.toString().removeSuffix(".jar") + fun jarName(url: URL): String = (url.toPath().fileName ?: "").toString().removeSuffix(".jar") /** CorDapp manifest entries */ const val CORDAPP_CONTRACT_NAME = "Cordapp-Contract-Name" @@ -81,7 +81,7 @@ data class CordappImpl( serializationCustomSerializers = emptyList(), customSchemas = emptySet(), jarPath = Paths.get("").toUri().toURL(), - info = CordappImpl.UNKNOWN_INFO, + info = UNKNOWN_INFO, allFlows = emptyList(), jarHash = SecureHash.allOnesHash, minimumPlatformVersion = 1, diff --git a/core/src/main/kotlin/net/corda/core/messaging/CordaRPCOps.kt b/core/src/main/kotlin/net/corda/core/messaging/CordaRPCOps.kt index a63f6013e7..6098b0c707 100644 --- a/core/src/main/kotlin/net/corda/core/messaging/CordaRPCOps.kt +++ b/core/src/main/kotlin/net/corda/core/messaging/CordaRPCOps.kt @@ -302,7 +302,12 @@ interface CordaRPCOps : RPCOps { /** Checks whether an attachment with the given hash is stored on the node. */ fun attachmentExists(id: SecureHash): Boolean - /** Download an attachment JAR by ID. */ + /** + * Download an attachment JAR by ID. + * @param id the id of the attachment to open + * @return the stream of the JAR + * @throws RPCException if the attachment doesn't exist + * */ fun openAttachment(id: SecureHash): InputStream /** Uploads a jar to the node, returns it's hash. */ diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt index eed759a08e..e93be2de5d 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt @@ -1,5 +1,8 @@ package net.corda.core.serialization.internal +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine +import net.corda.core.DeleteForDJVM import net.corda.core.contracts.Attachment import net.corda.core.contracts.ContractAttachment import net.corda.core.contracts.TransactionVerificationException @@ -21,6 +24,7 @@ import java.lang.ref.WeakReference import java.net.* import java.security.Permission import java.util.* +import java.util.function.Function /** * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only @@ -289,31 +293,27 @@ class AttachmentsClassLoader(attachments: List, */ @VisibleForTesting object AttachmentsClassLoaderBuilder { - private const val CACHE_SIZE = 1000 + const val CACHE_SIZE = 16 - // We use a set here because the ordering of attachments doesn't affect code execution, due to the no - // overlap rule, and attachments don't have any particular ordering enforced by the builders. So we - // can just do unordered comparisons here. But the same attachments run with different network parameters - // may behave differently, so that has to be a part of the cache key. - private data class Key(val hashes: Set, val params: NetworkParameters) - - // This runs in the DJVM so it can't use caffeine. - private val cache: MutableMap = createSimpleCache(CACHE_SIZE).toSynchronised() + private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE) /** * Runs the given block with serialization execution context set up with a (possibly cached) attachments classloader. * * @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup. */ + @Suppress("LongParameterList") fun withAttachmentsClassloaderContext(attachments: List, params: NetworkParameters, txId: SecureHash, isAttachmentTrusted: (Attachment) -> Boolean, parent: ClassLoader = ClassLoader.getSystemClassLoader(), + attachmentsClassLoaderCache: AttachmentsClassLoaderCache?, block: (ClassLoader) -> T): T { val attachmentIds = attachments.map(Attachment::id).toSet() - val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) { + val cache = attachmentsClassLoaderCache ?: fallBackCache + val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function { // Create classloader and load serializers, whitelisted classes val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, isAttachmentTrusted, parent) val serializers = try { @@ -336,7 +336,7 @@ object AttachmentsClassLoaderBuilder { .withWhitelist(whitelistedClasses) .withCustomSerializers(serializers) .withoutCarpenter() - } + }) // Deserialize all relevant classes in the transaction classloader. return SerializationFactory.defaultFactory.withCurrentContext(serializationContext) { @@ -420,6 +420,36 @@ private class AttachmentsHolderImpl : AttachmentsHolder { } } +interface AttachmentsClassLoaderCache { + fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext +} + +@DeleteForDJVM +class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache { + + private val cache: Cache = cacheFactory.buildNamed(Caffeine.newBuilder(), "AttachmentsClassLoader_cache") + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function") + } +} + +class AttachmentsClassLoaderSimpleCacheImpl(cacheSize: Int) : AttachmentsClassLoaderCache { + + private val cache: MutableMap + = createSimpleCache(cacheSize).toSynchronised() + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.computeIfAbsent(key, mappingFunction) + } +} + +// We use a set here because the ordering of attachments doesn't affect code execution, due to the no +// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we +// can just do unordered comparisons here. But the same attachments run with different network parameters +// may behave differently, so that has to be a part of the cache key. +data class AttachmentsClassLoaderKey(val hashes: Set, val params: NetworkParameters) + private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) { override fun getContentLengthLong(): Long = attachment.size.toLong() override fun getInputStream(): InputStream = attachment.open() diff --git a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt index c590047267..277dccc1d2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt @@ -153,7 +153,8 @@ data class ContractUpgradeWireTransaction( listOf(legacyAttachment, upgradedAttachment), params, id, - { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }) { transactionClassLoader -> + { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }, + attachmentsClassLoaderCache = (services as ServiceHubCoreInternal).attachmentsClassLoaderCache) { transactionClassLoader -> val resolvedInput = binaryInput.deserialize() val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader) val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment) diff --git a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt index 3dffc8182c..6c73c299c2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt @@ -26,6 +26,7 @@ import net.corda.core.internal.deserialiseComponentGroup import net.corda.core.internal.isUploaderTrusted import net.corda.core.internal.uncheckedCast import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder import net.corda.core.utilities.contextLogger import java.util.Collections.unmodifiableList @@ -87,7 +88,8 @@ private constructor( private val serializedInputs: List?, private val serializedReferences: List?, private val isAttachmentTrusted: (Attachment) -> Boolean, - private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier + private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier, + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ) : FullTransaction() { init { @@ -124,7 +126,8 @@ private constructor( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { return LedgerTransaction( inputs = inputs, @@ -141,7 +144,8 @@ private constructor( serializedInputs = protect(serializedInputs), serializedReferences = protect(serializedReferences), isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -176,7 +180,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { true }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) } } @@ -218,7 +223,8 @@ private constructor( txAttachments, getParamsWithGoo(), id, - isAttachmentTrusted = isAttachmentTrusted) { transactionClassLoader -> + isAttachmentTrusted = isAttachmentTrusted, + attachmentsClassLoaderCache = attachmentsClassLoaderCache) { transactionClassLoader -> // Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader]. // Only the copy will be used for verification, and the outer shell will be discarded. // This artifice is required to preserve backwards compatibility. @@ -254,7 +260,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = alternateVerifier + verifierFactory = alternateVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) // Read network parameters with backwards compatibility goo. @@ -320,7 +327,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } else { // This branch is only present for backwards compatibility. @@ -704,7 +712,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransaction should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -733,7 +742,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransactions should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -761,7 +771,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -791,7 +802,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } } diff --git a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt index ac7be9afeb..73b286276d 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt @@ -15,6 +15,7 @@ import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.serialize import net.corda.core.utilities.OpaqueBytes import java.security.PublicKey @@ -109,7 +110,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr services.networkParametersService.lookup(hashToResolve) }, // `as?` is used due to [MockServices] not implementing [ServiceHubCoreInternal] - isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true } + isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true }, + attachmentsClassLoaderCache = (services as? ServiceHubCoreInternal)?.attachmentsClassLoaderCache ) ) } @@ -145,7 +147,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, { null }, - { it.isUploaderTrusted() } + { it.isUploaderTrusted() }, + null ) } @@ -161,16 +164,19 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, resolveParameters, - { true } // Any attachment loaded through the DJVM should be trusted + { true }, // Any attachment loaded through the DJVM should be trusted + null ) } + @Suppress("LongParameterList", "ThrowsCount") private fun toLedgerTransactionInternal( resolveIdentity: (PublicKey) -> Party?, resolveAttachment: (SecureHash) -> Attachment?, resolveStateRefAsSerialized: (StateRef) -> SerializedBytes>?, resolveParameters: (SecureHash?) -> NetworkParameters?, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { // Look up public keys to authenticated identities. val authenticatedCommands = commands.lazyMapped { cmd, _ -> @@ -206,7 +212,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr componentGroups, serializedResolvedInputs, serializedResolvedReferences, - isAttachmentTrusted + isAttachmentTrusted, + attachmentsClassLoaderCache ) checkTransactionSize(ltx, resolvedNetworkParameters.maxTransactionSize, serializedResolvedInputs, serializedResolvedReferences) diff --git a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt index 148aaff7bf..c325c805e3 100644 --- a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt +++ b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt @@ -4,6 +4,7 @@ import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash import net.corda.core.identity.Party import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.transactions.ComponentGroup import net.corda.core.transactions.LedgerTransaction import net.corda.core.transactions.WireTransaction @@ -17,6 +18,7 @@ fun WireTransaction.accessGroupHashes() = this.groupHashes fun WireTransaction.accessGroupMerkleRoots() = this.groupsMerkleRoots fun WireTransaction.accessAvailableComponentHashes() = this.availableComponentHashes +@Suppress("LongParameterList") fun createLedgerTransaction( inputs: List>, outputs: List>, @@ -31,8 +33,9 @@ fun createLedgerTransaction( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean -): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted) + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache +): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted, attachmentsClassLoaderCache) fun createContractCreationError(txId: SecureHash, contractClass: String, cause: Throwable) = TransactionVerificationException.ContractCreationError(txId, contractClass, cause) fun createContractRejection(txId: SecureHash, contract: Contract, cause: Throwable) = TransactionVerificationException.ContractRejection(txId, contract, cause) diff --git a/detekt-baseline.xml b/detekt-baseline.xml index 2c667858ab..974e679f57 100644 --- a/detekt-baseline.xml +++ b/detekt-baseline.xml @@ -1435,7 +1435,7 @@ ThrowsCount:JarScanningCordappLoader.kt$JarScanningCordappLoader$private fun parseVersion(versionStr: String?, attributeName: String): Int ThrowsCount:LedgerDSLInterpreter.kt$Verifies$ fun failsWith(expectedMessage: String?): EnforceVerifyOrFail ThrowsCount:MockServices.kt$ fun <T : SerializeAsToken> createMockCordaService(serviceHub: MockServices, serviceConstructor: (AppServiceHub) -> T): T - ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates(registeringPublicKey: PublicKey, certificates: List<X509Certificate>) + ThrowsCount:NetworkRegistrationHelper.kt$NetworkRegistrationHelper$private fun validateCertificates( registeringPublicKey: PublicKey, registeringLegalName: CordaX500Name, expectedCertRole: CertRole, certificates: List<X509Certificate> ) ThrowsCount:NodeInfoFilesCopier.kt$NodeInfoFilesCopier$private fun atomicCopy(source: Path, destination: Path) ThrowsCount:NodeVaultService.kt$NodeVaultService$@Throws(VaultQueryException::class) private fun <T : ContractState> _queryBy(criteria: QueryCriteria, paging_: PageSpecification, sorting: Sort, contractStateType: Class<out T>, skipPagingChecks: Boolean): Vault.Page<T> ThrowsCount:NodeVaultService.kt$NodeVaultService$private fun makeUpdates(batch: Iterable<CoreTransaction>, statesToRecord: StatesToRecord, previouslySeen: Boolean): List<Vault.Update<ContractState>> @@ -1598,6 +1598,7 @@ TooGenericExceptionCaught:ScheduledFlowIntegrationTests.kt$ScheduledFlowIntegrationTests$ex: Exception TooGenericExceptionCaught:SerializationOutputTests.kt$SerializationOutputTests$t: Throwable TooGenericExceptionCaught:ShutdownManager.kt$ShutdownManager$t: Throwable + TooGenericExceptionCaught:SimpleAMQPClient.kt$SimpleAMQPClient$e: Exception TooGenericExceptionCaught:SimpleMQClient.kt$SimpleMQClient$e: Exception TooGenericExceptionCaught:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager$e: Exception TooGenericExceptionCaught:SingleThreadedStateMachineManager.kt$SingleThreadedStateMachineManager$ex: Exception @@ -1617,6 +1618,7 @@ TooGenericExceptionCaught:TransformTypes.kt$TransformTypes.Companion$e: IndexOutOfBoundsException TooGenericExceptionCaught:TransitionExecutorImpl.kt$TransitionExecutorImpl$exception: Exception TooGenericExceptionCaught:Try.kt$Try.Companion$t: Throwable + TooGenericExceptionCaught:UserValidationPlugin.kt$UserValidationPlugin$e: Throwable TooGenericExceptionCaught:Utils.kt$e: Exception TooGenericExceptionCaught:V1NodeConfigurationSpec.kt$V1NodeConfigurationSpec$e: Exception TooGenericExceptionCaught:ValidatingNotaryFlow.kt$ValidatingNotaryFlow$e: Exception diff --git a/deterministic.gradle b/deterministic.gradle index 2a7913d426..751af8bfb2 100644 --- a/deterministic.gradle +++ b/deterministic.gradle @@ -11,12 +11,12 @@ evaluationDependsOn(':jdk8u-deterministic') def jdk8uDeterministic = project(':jdk8u-deterministic') ext { - jdkTask = jdk8uDeterministic.assemble + jdkTask = jdk8uDeterministic.tasks.named('assemble') deterministic_jdk_home = jdk8uDeterministic.jdk_home deterministic_rt_jar = jdk8uDeterministic.rt_jar } -tasks.withType(AbstractCompile) { +tasks.withType(AbstractCompile).configureEach { dependsOn jdkTask // This is a bit ugly, but Gradle isn't recognising the KotlinCompile task @@ -29,7 +29,7 @@ tasks.withType(AbstractCompile) { } } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { options.compilerArgs << '-bootclasspath' << deterministic_rt_jar sourceCompatibility = VERSION_1_8 targetCompatibility = VERSION_1_8 diff --git a/docker/src/docker/Dockerfile b/docker/src/docker/Dockerfile index 2100ec59f9..d3d287a750 100644 --- a/docker/src/docker/Dockerfile +++ b/docker/src/docker/Dockerfile @@ -7,6 +7,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /opt/corda/cordapps && \ mkdir -p /opt/corda/persistence && \ + mkdir -p /opt/corda/artemis && \ mkdir -p /opt/corda/certificates && \ mkdir -p /opt/corda/drivers && \ mkdir -p /opt/corda/logs && \ @@ -20,6 +21,7 @@ RUN apt-get update && \ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ PERSISTENCE_FOLDER="/opt/corda/persistence" \ + ARTEMIS_FOLDER="/opt/corda/artemis" \ CERTIFICATES_FOLDER="/opt/corda/certificates" \ DRIVERS_FOLDER="/opt/corda/drivers" \ CONFIG_FOLDER="/etc/corda" \ @@ -34,6 +36,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ VOLUME ["/opt/corda/cordapps"] ##PERSISTENCE FOLDER VOLUME ["/opt/corda/persistence"] +##ARTEMIS FOLDER +VOLUME ["/opt/corda/artemis"] ##CERTS FOLDER VOLUME ["/opt/corda/certificates"] ##OPTIONAL JDBC DRIVERS FOLDER diff --git a/docker/src/docker/Dockerfile11 b/docker/src/docker/Dockerfile11 index dfb5eaa0e3..20b48ddcdc 100644 --- a/docker/src/docker/Dockerfile11 +++ b/docker/src/docker/Dockerfile11 @@ -19,6 +19,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* && \ mkdir -p /opt/corda/cordapps && \ mkdir -p /opt/corda/persistence && \ + mkdir -p /opt/corda/artemis && \ mkdir -p /opt/corda/certificates && \ mkdir -p /opt/corda/drivers && \ mkdir -p /opt/corda/logs && \ @@ -36,6 +37,7 @@ RUN apt-get update && \ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ PERSISTENCE_FOLDER="/opt/corda/persistence" \ + ARTEMIS_FOLDER="/opt/corda/artemis" \ CERTIFICATES_FOLDER="/opt/corda/certificates" \ DRIVERS_FOLDER="/opt/corda/drivers" \ CONFIG_FOLDER="/etc/corda" \ @@ -50,6 +52,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ VOLUME ["/opt/corda/cordapps"] ##PERSISTENCE FOLDER VOLUME ["/opt/corda/persistence"] +##ARTEMIS FOLDER +VOLUME ["/opt/corda/artemis"] ##CERTS FOLDER VOLUME ["/opt/corda/certificates"] ##OPTIONAL JDBC DRIVERS FOLDER diff --git a/docker/src/docker/DockerfileAL b/docker/src/docker/DockerfileAL index b5ce21ab00..f3c8496604 100644 --- a/docker/src/docker/DockerfileAL +++ b/docker/src/docker/DockerfileAL @@ -10,6 +10,7 @@ RUN amazon-linux-extras enable corretto8 && \ rm -rf /var/cache/yum && \ mkdir -p /opt/corda/cordapps && \ mkdir -p /opt/corda/persistence && \ + mkdir -p /opt/corda/artemis && \ mkdir -p /opt/corda/certificates && \ mkdir -p /opt/corda/drivers && \ mkdir -p /opt/corda/logs && \ @@ -23,6 +24,7 @@ RUN amazon-linux-extras enable corretto8 && \ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ PERSISTENCE_FOLDER="/opt/corda/persistence" \ + ARTEMIS_FOLDER="/opt/corda/artemis" \ CERTIFICATES_FOLDER="/opt/corda/certificates" \ DRIVERS_FOLDER="/opt/corda/drivers" \ CONFIG_FOLDER="/etc/corda" \ @@ -37,6 +39,8 @@ ENV CORDAPPS_FOLDER="/opt/corda/cordapps" \ VOLUME ["/opt/corda/cordapps"] ##PERSISTENCE FOLDER VOLUME ["/opt/corda/persistence"] +##ARTEMIS FOLDER +VOLUME ["/opt/corda/artemis"] ##CERTS FOLDER VOLUME ["/opt/corda/certificates"] ##OPTIONAL JDBC DRIVERS FOLDER diff --git a/docker/src/main/kotlin/net.corda.core/ConfigExporter.kt b/docker/src/main/kotlin/net.corda.core/ConfigExporter.kt index 2f7356cd7f..d9b74434ba 100644 --- a/docker/src/main/kotlin/net.corda.core/ConfigExporter.kt +++ b/docker/src/main/kotlin/net.corda.core/ConfigExporter.kt @@ -51,7 +51,7 @@ class ConfigExporter { } fun Config.parseAsNodeConfigWithFallback(): Validated { - val referenceConfig = ConfigFactory.parseResources("reference.conf") + val referenceConfig = ConfigFactory.parseResources("corda-reference.conf") val nodeConfig = this .withValue("baseDirectory", ConfigValueFactory.fromAnyRef("/opt/corda")) .withFallback(referenceConfig) diff --git a/docs/build.gradle b/docs/build.gradle new file mode 100644 index 0000000000..09bdac83bc --- /dev/null +++ b/docs/build.gradle @@ -0,0 +1,122 @@ +import org.apache.tools.ant.taskdefs.condition.Os + +apply plugin: 'org.jetbrains.dokka' +apply plugin: 'net.corda.plugins.publish-utils' +apply plugin: 'maven-publish' +apply plugin: 'com.jfrog.artifactory' + +def internalPackagePrefixes(sourceDirs) { + def prefixes = [] + // Kotlin allows packages to deviate from the directory structure, but let's assume they don't: + sourceDirs.collect { sourceDir -> + sourceDir.traverse(type: groovy.io.FileType.DIRECTORIES) { + if (it.name == 'internal') { + prefixes.add sourceDir.toPath().relativize(it.toPath()).toString().replace(File.separator, '.') + } + } + } + prefixes +} + +ext { + // TODO: Add '../client/jfx/src/main/kotlin' and '../client/mock/src/main/kotlin' if we decide to make them into public API + dokkaSourceDirs = files('../core/src/main/kotlin', '../client/rpc/src/main/kotlin', '../finance/workflows/src/main/kotlin', '../finance/contracts/src/main/kotlin', '../client/jackson/src/main/kotlin', + '../testing/test-utils/src/main/kotlin', '../testing/node-driver/src/main/kotlin') + internalPackagePrefixes = internalPackagePrefixes(dokkaSourceDirs) + archivedApiDocsBaseFilename = 'api-docs' +} + +dokka { + outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/kotlin") +} + +task dokkaJavadoc(type: org.jetbrains.dokka.gradle.DokkaTask) { + outputFormat = "javadoc" + outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/javadoc") +} + +[dokka, dokkaJavadoc].collect { + it.configure { + moduleName = 'corda' + processConfigurations = ['compile'] + sourceDirs = dokkaSourceDirs + includes = ['packages.md'] + jdkVersion = 8 + externalDocumentationLink { + url = new URL("http://fasterxml.github.io/jackson-core/javadoc/2.9/") + } + externalDocumentationLink { + url = new URL("https://docs.oracle.com/javafx/2/api/") + } + externalDocumentationLink { + url = new URL("http://www.bouncycastle.org/docs/docs1.5on/") + } + internalPackagePrefixes.collect { packagePrefix -> + packageOptions { + prefix = packagePrefix + suppress = true + } + } + } +} + +task apidocs(dependsOn: ['dokka', 'dokkaJavadoc']) { + group "Documentation" + description "Build API documentation" +} + +task makeHTMLDocs(type: Exec){ + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-html.sh" + } else { + commandLine "bash", "-c", "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-html.sh" + } +} + +task makePDFDocs(type: Exec){ + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-pdf.sh" + } else { + commandLine "bash", "-c", "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-pdf.sh" + } +} + +task makeDocs(dependsOn: ['makeHTMLDocs', 'makePDFDocs']) +apidocs.shouldRunAfter makeDocs + +task archiveApiDocs(type: Tar) { + dependsOn apidocs + from buildDir + include 'html/**' + extension 'tgz' + compression Compression.GZIP +} + +publishing { + publications { + if (System.getProperty('publishApiDocs') != null) { + archivedApiDocs(MavenPublication) { + artifact archiveApiDocs { + artifactId archivedApiDocsBaseFilename + } + } + } + } +} + +artifactoryPublish { + publications('archivedApiDocs') + version = version.replaceAll('-SNAPSHOT', '') + publishPom = false +} + +artifactory { + publish { + contextUrl = artifactory_contextUrl + repository { + repoKey = 'corda-dependencies-dev' + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } +} diff --git a/experimental/avalanche/build.gradle b/experimental/avalanche/build.gradle index e6f9c1ccae..540bc2947b 100644 --- a/experimental/avalanche/build.gradle +++ b/experimental/avalanche/build.gradle @@ -22,4 +22,7 @@ jar.enabled = false shadowJar { baseName = "avalanche" } -assemble.dependsOn shadowJar + +artifacts { + archives shadowJar +} diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2e954d6bc7..29c1f86072 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,5 @@ -#Wed Aug 21 10:48:19 BST 2019 -distributionUrl=https\://gradleproxy:gradleproxy@software.r3.com/artifactory/gradle-proxy/gradle-5.4.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStorePath=wrapper/dists +distributionUrl=https\://gradleproxy:gradleproxy@software.r3.com/artifactory/gradle-proxy/gradle-5.6.4-all.zip zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index b0d6d0ab5d..83f2acfdc3 100755 --- a/gradlew +++ b/gradlew @@ -7,7 +7,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -125,8 +125,8 @@ if $darwin; then GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" fi -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` JAVACMD=`cygpath --unix "$JAVACMD"` diff --git a/gradlew.bat b/gradlew.bat index 15e1ee37a7..24467a141f 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -5,7 +5,7 @@ @rem you may not use this file except in compliance with the License. @rem You may obtain a copy of the License at @rem -@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem https://www.apache.org/licenses/LICENSE-2.0 @rem @rem Unless required by applicable law or agreed to in writing, software @rem distributed under the License is distributed on an "AS IS" BASIS, diff --git a/java8.gradle b/java8.gradle index e0fbf629cc..50a462aa41 100644 --- a/java8.gradle +++ b/java8.gradle @@ -6,7 +6,7 @@ import static org.gradle.api.JavaVersion.VERSION_1_8 */ apply plugin: 'kotlin' -tasks.withType(AbstractCompile) { +tasks.withType(AbstractCompile).configureEach { // This is a bit ugly, but Gradle isn't recognising the KotlinCompile task // as it does the built-in JavaCompile task. if (it.class.name.startsWith('org.jetbrains.kotlin.gradle.tasks.KotlinCompile')) { @@ -16,7 +16,7 @@ tasks.withType(AbstractCompile) { } } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { sourceCompatibility = VERSION_1_8 targetCompatibility = VERSION_1_8 } diff --git a/jdk8u-deterministic/build.gradle b/jdk8u-deterministic/build.gradle index 92338df169..f9a91c9cc8 100644 --- a/jdk8u-deterministic/build.gradle +++ b/jdk8u-deterministic/build.gradle @@ -37,7 +37,9 @@ def copyJdk = tasks.register('copyJdk', Copy) { } } -assemble.dependsOn copyJdk +tasks.named('assemble') { + dependsOn copyJdk +} tasks.named('jar', Jar) { enabled = false } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle/NodeLifecycleEventsDistributor.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle/NodeLifecycleEventsDistributor.kt index 7c2454c857..ee4d6f7be3 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle/NodeLifecycleEventsDistributor.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/lifecycle/NodeLifecycleEventsDistributor.kt @@ -7,6 +7,8 @@ import net.corda.core.internal.concurrent.openFuture import net.corda.core.node.services.CordaServiceCriticalFailureException import net.corda.core.utilities.Try import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.persistence.contextDatabase +import net.corda.nodeapi.internal.persistence.contextDatabaseOrNull import java.io.Closeable import java.util.Collections.singleton import java.util.LinkedList @@ -93,7 +95,14 @@ class NodeLifecycleEventsDistributor : Closeable { log.warn("Not distributing $event as executor been already shutdown. Double close() case?") result.set(null) } else { + + val passTheDbToTheThread = contextDatabaseOrNull + executor.execute { + + if (passTheDbToTheThread != null) + contextDatabase = passTheDbToTheThread + val orderedSnapshot = if (event.reversedPriority) snapshot.reversed() else snapshot orderedSnapshot.forEach { log.debug("Distributing event $event to: $it") diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt index be7fb1a4d0..16c5857ae9 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt @@ -6,6 +6,7 @@ import org.hibernate.Session import org.hibernate.Transaction import rx.subjects.PublishSubject import java.sql.Connection +import java.sql.SQLException import java.util.UUID import javax.persistence.EntityManager @@ -87,6 +88,7 @@ class DatabaseTransaction( committed = true } + @Throws(SQLException::class) fun rollback() { if (sessionDelegate.isInitialized() && session.isOpen) { session.clear() @@ -97,16 +99,20 @@ class DatabaseTransaction( clearException() } + @Throws(SQLException::class) fun close() { - if (sessionDelegate.isInitialized() && session.isOpen) { - session.close() + try { + if (sessionDelegate.isInitialized() && session.isOpen) { + session.close() + } + if (database.closeConnection) { + connection.close() + } + } finally { + clearException() + contextTransactionOrNull = outerTransaction } - if (database.closeConnection) { - connection.close() - } - clearException() - contextTransactionOrNull = outerTransaction if (outerTransaction == null) { synchronized(this) { closed = true diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt index c26aa74ada..5ce3db919c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt @@ -10,6 +10,7 @@ import io.netty.handler.proxy.ProxyConnectionEvent import io.netty.handler.ssl.SniCompletionEvent import io.netty.handler.ssl.SslHandler import io.netty.handler.ssl.SslHandshakeCompletionEvent +import io.netty.handler.ssl.SslHandshakeTimeoutException import io.netty.util.ReferenceCountUtil import net.corda.core.identity.CordaX500Name import net.corda.core.utilities.contextLogger @@ -295,8 +296,8 @@ internal class AMQPChannelHandler(private val serverMode: Boolean, // This happens when the peer node is closed during SSL establishment. when { cause is ClosedChannelException -> logWarnWithMDC("SSL Handshake closed early.") + cause is SslHandshakeTimeoutException -> logWarnWithMDC("SSL Handshake timed out") // Sadly the exception thrown by Netty wrapper requires that we check the message. - cause is SSLException && cause.message == "handshake timed out" -> logWarnWithMDC("SSL Handshake timed out") cause is SSLException && (cause.message?.contains("close_notify") == true) -> logWarnWithMDC("Received close_notify during handshake") // io.netty.handler.ssl.SslHandler.setHandshakeFailureTransportFailure() diff --git a/node/build.gradle b/node/build.gradle index 58f7a5498e..99fcb9760a 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -20,6 +20,11 @@ ext { jolokia_version = constants.getProperty('jolokiaAgentVersion') } +evaluationDependsOn(':core-deterministic') +evaluationDependsOn(':serialization-deterministic') +evaluationDependsOn(':serialization-djvm:deserializers') +evaluationDependsOn(':node:djvm') + //noinspection GroovyAssignabilityCheck configurations { integrationTestCompile.extendsFrom testCompile @@ -191,7 +196,8 @@ dependencies { // Integration test helpers integrationTestCompile "junit:junit:$junit_version" integrationTestCompile "org.assertj:assertj-core:${assertj_version}" - + integrationTestCompile "org.apache.qpid:qpid-jms-client:${protonj_version}" + // BFT-Smart dependencies compile 'com.github.bft-smart:library:master-v1.1-beta-g6215ec8-87' compile 'commons-codec:commons-codec:1.13' @@ -242,12 +248,12 @@ dependencies { testCompile project(':testing:cordapps:dbfailure:dbfworkflows') } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { // Resolves a Gradle warning about not scanning for pre-processors. options.compilerArgs << '-proc:none' } -tasks.withType(Test) { +tasks.withType(Test).configureEach { if (JavaVersion.current() == JavaVersion.VERSION_11) { jvmArgs '-Djdk.attach.allowAttachSelf=true' } @@ -255,13 +261,13 @@ tasks.withType(Test) { systemProperty 'deterministic-sources.path', configurations.deterministic.asPath } -task integrationTest(type: Test) { +tasks.register('integrationTest', Test) { testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath maxParallelForks = (System.env.CORDA_NODE_INT_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_NODE_INT_TESTING_FORKS".toInteger() } -task slowIntegrationTest(type: Test) { +tasks.register('slowIntegrationTest', Test) { testClassesDirs = sourceSets.slowIntegrationTest.output.classesDirs classpath = sourceSets.slowIntegrationTest.runtimeClasspath maxParallelForks = 1 @@ -319,7 +325,7 @@ publish { name jar.baseName } -test { +tasks.named('test', Test) { maxHeapSize = "3g" maxParallelForks = (System.env.CORDA_NODE_TESTING_FORKS == null) ? 1 : "$System.env.CORDA_NODE_TESTING_FORKS".toInteger() } diff --git a/node/capsule/build.gradle b/node/capsule/build.gradle index 2760a54aed..c1c4e2f4c9 100644 --- a/node/capsule/build.gradle +++ b/node/capsule/build.gradle @@ -39,9 +39,9 @@ capsule { def nodeProject = project(':node') task buildCordaJAR(type: FatCapsule, dependsOn: [ - nodeProject.tasks.jar, - project(':core-deterministic').tasks.assemble, - project(':serialization-deterministic').tasks.assemble + nodeProject.tasks.named('jar'), + project(':core-deterministic').tasks.named('assemble'), + project(':serialization-deterministic').tasks.named('assemble') ]) { applicationClass 'net.corda.node.Corda' archiveBaseName = 'corda' @@ -51,7 +51,7 @@ task buildCordaJAR(type: FatCapsule, dependsOn: [ applicationSource = files( nodeProject.configurations.runtimeClasspath, nodeProject.tasks.jar, - nodeProject.buildDir.toString() + '/resources/main/reference.conf', + nodeProject.buildDir.toString() + '/resources/main/corda-reference.conf', "$rootDir/config/dev/log4j2.xml", 'NOTICE' // Copy CDDL notice ) @@ -119,9 +119,8 @@ task buildCordaJAR(type: FatCapsule, dependsOn: [ } } -assemble.dependsOn buildCordaJAR - artifacts { + archives buildCordaJAR runtimeArtifacts buildCordaJAR publish buildCordaJAR { classifier '' diff --git a/node/capsule/src/main/java/CordaCaplet.java b/node/capsule/src/main/java/CordaCaplet.java index 76be8aad17..7633ba0a77 100644 --- a/node/capsule/src/main/java/CordaCaplet.java +++ b/node/capsule/src/main/java/CordaCaplet.java @@ -37,7 +37,7 @@ public class CordaCaplet extends Capsule { File configFile = (config == null) ? new File(baseDir, "node.conf") : new File(config); try { ConfigParseOptions parseOptions = ConfigParseOptions.defaults().setAllowMissing(false); - Config defaultConfig = ConfigFactory.parseResources("reference.conf", parseOptions); + Config defaultConfig = ConfigFactory.parseResources("corda-reference.conf", parseOptions); Config baseDirectoryConfig = ConfigFactory.parseMap(Collections.singletonMap("baseDirectory", baseDir)); Config nodeConfig = ConfigFactory.parseFile(configFile, parseOptions); return baseDirectoryConfig.withFallback(nodeConfig).withFallback(defaultConfig).resolve(); diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt index f133881fa0..0cb4c7fd77 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/flows/FlowCheckpointVersionNodeStartupCheckTest.kt @@ -1,6 +1,7 @@ package net.corda.node.flows import co.paralleluniverse.fibers.Suspendable +import net.corda.core.CordaException import net.corda.core.flows.* import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party @@ -16,7 +17,6 @@ import net.corda.testing.driver.DriverDSL import net.corda.testing.driver.DriverParameters import net.corda.testing.driver.NodeParameters import net.corda.testing.driver.driver -import net.corda.testing.node.internal.ListenProcessDeathException import net.corda.testing.node.internal.assertUncompletedCheckpoints import net.corda.testing.node.internal.enclosedCordapp import org.assertj.core.api.Assertions.assertThat @@ -78,7 +78,7 @@ class FlowCheckpointVersionNodeStartupCheckTest { private fun DriverDSL.assertBobFailsToStartWithLogMessage(logMessage: String) { assertUncompletedCheckpoints(BOB_NAME, 1) - assertFailsWith(ListenProcessDeathException::class) { + assertFailsWith(CordaException::class) { startNode(NodeParameters( providedName = BOB_NAME, customOverrides = mapOf("devMode" to false) diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/distributed/DistributedServiceTests.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/distributed/DistributedServiceTests.kt index 55dcbcfc9f..ad04a89d8c 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/distributed/DistributedServiceTests.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/distributed/DistributedServiceTests.kt @@ -29,6 +29,8 @@ import org.junit.Ignore import org.junit.Test import rx.Observable import java.util.* +import kotlin.test.assertEquals +import kotlin.test.assertTrue class DistributedServiceTests { private lateinit var alice: NodeHandle @@ -157,9 +159,9 @@ class DistributedServiceTests { // The distribution of requests should be very close to sg like 16/17/17 as by default artemis does round robin println("Notarisation distribution: $notarisationsPerNotary") - require(notarisationsPerNotary.size == 3) + assertEquals(3, notarisationsPerNotary.size) // We allow some leeway for artemis as it doesn't always produce perfect distribution - require(notarisationsPerNotary.values.all { it > 10 }) + assertTrue { notarisationsPerNotary.values.all { it > 10 } } } private fun issueCash(amount: Amount) { diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt deleted file mode 100644 index 242cacdaad..0000000000 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/rpc/RpcReconnectTests.kt +++ /dev/null @@ -1,355 +0,0 @@ -package net.corda.node.services.rpc - -import net.corda.client.rpc.CordaRPCClient -import net.corda.client.rpc.CordaRPCClientConfiguration -import net.corda.client.rpc.GracefulReconnect -import net.corda.client.rpc.internal.ReconnectingCordaRPCOps -import net.corda.client.rpc.notUsed -import net.corda.core.contracts.Amount -import net.corda.core.flows.StateMachineRunId -import net.corda.core.internal.concurrent.transpose -import net.corda.core.messaging.StateMachineUpdate -import net.corda.core.node.services.Vault -import net.corda.core.node.services.vault.PageSpecification -import net.corda.core.node.services.vault.QueryCriteria -import net.corda.core.node.services.vault.builder -import net.corda.core.utilities.NetworkHostAndPort -import net.corda.core.utilities.OpaqueBytes -import net.corda.core.utilities.contextLogger -import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.seconds -import net.corda.finance.contracts.asset.Cash -import net.corda.finance.flows.CashIssueAndPaymentFlow -import net.corda.finance.schemas.CashSchemaV1 -import net.corda.node.services.Permissions -import net.corda.node.services.rpc.RpcReconnectTests.Companion.NUMBER_OF_FLOWS_TO_RUN -import net.corda.testing.core.DUMMY_BANK_A_NAME -import net.corda.testing.core.DUMMY_BANK_B_NAME -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeHandle -import net.corda.testing.driver.OutOfProcess -import net.corda.testing.driver.driver -import net.corda.testing.driver.internal.OutOfProcessImpl -import net.corda.testing.driver.internal.incrementalPortAllocation -import net.corda.testing.node.User -import net.corda.testing.node.internal.FINANCE_CORDAPPS -import org.assertj.core.api.Assertions.assertThat -import org.junit.Test -import java.util.* -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicInteger -import kotlin.concurrent.thread -import kotlin.math.absoluteValue -import kotlin.math.max -import kotlin.test.assertEquals -import kotlin.test.assertTrue -import kotlin.test.currentStackTrace - -/** - * This is a stress test for the rpc reconnection logic, which triggers failures in a probabilistic way. - * - * You can adjust the variable [NUMBER_OF_FLOWS_TO_RUN] to adjust the number of flows to run and the duration of the test. - */ -class RpcReconnectTests { - - companion object { - // this many flows take ~5 minutes - const val NUMBER_OF_FLOWS_TO_RUN = 100 - - private val log = contextLogger() - } - - private val portAllocator = incrementalPortAllocation() - - private lateinit var proxy: RandomFailingProxy - private lateinit var node: NodeHandle - private lateinit var currentAddressPair: AddressPair - - /** - * This test showcases and stress tests the demo [ReconnectingCordaRPCOps]. - * - * Note that during node failure events can be lost and starting flows can become unreliable. - * The only available way to retry failed flows is to attempt a "logical retry" which is also showcased. - * - * This test runs flows in a loop and in the background kills the node or restarts it. - * Also the RPC connection is made through a proxy that introduces random latencies and is also periodically killed. - */ - @Suppress("ComplexMethod") - @Test(timeout=420_000) - fun `test that the RPC client is able to reconnect and proceed after node failure, restart, or connection reset`() { - val nodeRunningTime = { Random().nextInt(12000) + 8000 } - - val demoUser = User("demo", "demo", setOf(Permissions.all())) - - // When this reaches 0 - the test will end. - val flowsCountdownLatch = CountDownLatch(NUMBER_OF_FLOWS_TO_RUN) - // These are the expected progress steps for the CashIssueAndPayFlow. - val expectedProgress = listOf( - "Starting", - "Issuing cash", - "Generating transaction", - "Signing transaction", - "Finalising transaction", - "Broadcasting transaction to participants", - "Paying recipient", - "Generating anonymous identities", - "Generating transaction", - "Signing transaction", - "Finalising transaction", - "Requesting signature by notary service", - "Requesting signature by Notary service", - "Validating response from Notary service", - "Broadcasting transaction to participants", - "Done" - ) - - driver(DriverParameters(cordappsForAllNodes = FINANCE_CORDAPPS, startNodesInProcess = false, inMemoryDB = false)) { - fun startBankA(address: NetworkHostAndPort) = startNode(providedName = DUMMY_BANK_A_NAME, rpcUsers = listOf(demoUser), customOverrides = mapOf("rpcSettings.address" to address.toString())) - fun startProxy(addressPair: AddressPair) = RandomFailingProxy(serverPort = addressPair.proxyAddress.port, remotePort = addressPair.nodeAddress.port).start() - - val addresses = (1..2).map { getRandomAddressPair() } - currentAddressPair = addresses[0] - - proxy = startProxy(currentAddressPair) - val (bankA, bankB) = listOf( - startBankA(currentAddressPair.nodeAddress), - startNode(providedName = DUMMY_BANK_B_NAME, rpcUsers = listOf(demoUser)) - ).transpose().getOrThrow() - node = bankA - - val notary = defaultNotaryIdentity - val baseAmount = Amount.parseCurrency("0 USD") - val issuerRef = OpaqueBytes.of(0x01) - - var numDisconnects = 0 - var numReconnects = 0 - val maxStackOccurrences = AtomicInteger() - - val addressesForRpc = addresses.map { it.proxyAddress } - // DOCSTART rpcReconnectingRPC - val onReconnect = { - numReconnects++ - // We only expect to see a single reconnectOnError in the stack trace. Otherwise we're in danger of stack overflow recursion - maxStackOccurrences.set(max(maxStackOccurrences.get(), currentStackTrace().count { it.methodName == "reconnectOnError" })) - Unit - } - val reconnect = GracefulReconnect(onDisconnect = { numDisconnects++ }, onReconnect = onReconnect) - val config = CordaRPCClientConfiguration.DEFAULT.copy( - connectionRetryInterval = 1.seconds, - connectionRetryIntervalMultiplier = 1.0 - ) - val client = CordaRPCClient(addressesForRpc, configuration = config) - val bankAReconnectingRPCConnection = client.start(demoUser.username, demoUser.password, gracefulReconnect = reconnect) - val bankAReconnectingRpc = bankAReconnectingRPCConnection.proxy as ReconnectingCordaRPCOps - // DOCEND rpcReconnectingRPC - - // Observe the vault and collect the observations. - val vaultEvents = Collections.synchronizedList(mutableListOf>()) - // DOCSTART rpcReconnectingRPCVaultTracking - val vaultFeed = bankAReconnectingRpc.vaultTrackByWithPagingSpec( - Cash.State::class.java, - QueryCriteria.VaultQueryCriteria(), - PageSpecification(1, 1)) - val vaultSubscription = vaultFeed.updates.subscribe { update: Vault.Update -> - log.info("vault update produced ${update.produced.map { it.state.data.amount }} consumed ${update.consumed.map { it.ref }}") - vaultEvents.add(update) - } - // DOCEND rpcReconnectingRPCVaultTracking - - // Observe the stateMachine and collect the observations. - val stateMachineEvents = Collections.synchronizedList(mutableListOf()) - val stateMachineSubscription = bankAReconnectingRpc.stateMachinesFeed().updates.subscribe { update -> - log.info(update.toString()) - stateMachineEvents.add(update) - } - - // While the flows are running, randomly apply a different failure scenario. - val nrRestarts = AtomicInteger() - thread(name = "Node killer") { - while (true) { - if (flowsCountdownLatch.count == 0L) break - - // Let the node run for a random time interval. - nodeRunningTime().also { ms -> - log.info("Running node for ${ms / 1000} s.") - Thread.sleep(ms.toLong()) - } - - if (flowsCountdownLatch.count == 0L) break - when (Random().nextInt().rem(7).absoluteValue) { - 0 -> { - log.info("Forcefully killing node and proxy.") - (node as OutOfProcessImpl).onStopCallback() - (node as OutOfProcess).process.destroyForcibly() - proxy.stop() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy.start() - } - 1 -> { - log.info("Forcefully killing node.") - (node as OutOfProcessImpl).onStopCallback() - (node as OutOfProcess).process.destroyForcibly() - node = startBankA(currentAddressPair.nodeAddress).get() - } - 2 -> { - log.info("Shutting down node.") - node.stop() - proxy.stop() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy.start() - } - 3, 4 -> { - log.info("Killing proxy.") - proxy.stop() - Thread.sleep(Random().nextInt(5000).toLong()) - proxy.start() - } - 5 -> { - log.info("Dropping connection.") - proxy.failConnection() - } - 6 -> { - log.info("Performing failover to a different node") - node.stop() - proxy.stop() - currentAddressPair = (addresses - currentAddressPair).first() - node = startBankA(currentAddressPair.nodeAddress).get() - proxy = startProxy(currentAddressPair) - } - } - nrRestarts.incrementAndGet() - } - } - - // Start nrOfFlowsToRun and provide a logical retry function that checks the vault. - val flowProgressEvents = mutableMapOf>() - for (amount in (1..NUMBER_OF_FLOWS_TO_RUN)) { - // DOCSTART rpcReconnectingRPCFlowStarting - bankAReconnectingRpc.runFlowWithLogicalRetry( - runFlow = { rpc -> - log.info("Starting CashIssueAndPaymentFlow for $amount") - val flowHandle = rpc.startTrackedFlowDynamic( - CashIssueAndPaymentFlow::class.java, - baseAmount.plus(Amount.parseCurrency("$amount USD")), - issuerRef, - bankB.nodeInfo.legalIdentities.first(), - false, - notary - ) - val flowId = flowHandle.id - log.info("Started flow $amount with flowId: $flowId") - flowProgressEvents.addEvent(flowId, null) - - flowHandle.stepsTreeFeed?.updates?.notUsed() - flowHandle.stepsTreeIndexFeed?.updates?.notUsed() - // No reconnecting possible. - flowHandle.progress.subscribe( - { prog -> - flowProgressEvents.addEvent(flowId, prog) - log.info("Progress $flowId : $prog") - }, - { error -> - log.error("Error thrown in the flow progress observer", error) - }) - flowHandle.id - }, - hasFlowStarted = { rpc -> - // Query for a state that is the result of this flow. - val criteria = QueryCriteria.VaultCustomQueryCriteria(builder { CashSchemaV1.PersistentCashState::pennies.equal(amount.toLong() * 100) }, status = Vault.StateStatus.ALL) - val results = rpc.vaultQueryByCriteria(criteria, Cash.State::class.java) - log.info("$amount - Found states ${results.states}") - // The flow has completed if a state is found - results.states.isNotEmpty() - }, - onFlowConfirmed = { - flowsCountdownLatch.countDown() - log.info("Flow started for $amount. Remaining flows: ${flowsCountdownLatch.count}") - } - ) - // DOCEND rpcReconnectingRPCFlowStarting - - Thread.sleep(Random().nextInt(250).toLong()) - } - - log.info("Started all flows") - - // Wait until all flows have been started. - val flowsConfirmed = flowsCountdownLatch.await(10, TimeUnit.MINUTES) - - if (flowsConfirmed) { - log.info("Confirmed all flows have started.") - } else { - log.info("Timed out waiting for confirmation that all flows have started. Remaining flows: ${flowsCountdownLatch.count}") - } - - - // Wait for all events to come in and flows to finish. - Thread.sleep(4000) - - val nrFailures = nrRestarts.get() - log.info("Checking results after $nrFailures restarts.") - - // We should get one disconnect and one reconnect for each failure - assertThat(numDisconnects).isEqualTo(numReconnects) - assertThat(numReconnects).isLessThanOrEqualTo(nrFailures) - assertThat(maxStackOccurrences.get()).isLessThan(2) - - // Query the vault and check that states were created for all flows. - fun readCashStates() = bankAReconnectingRpc - .vaultQueryByWithPagingSpec(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(status = Vault.StateStatus.CONSUMED), PageSpecification(1, 10000)) - .states - - var allCashStates = readCashStates() - var nrRetries = 0 - - // It might be necessary to wait more for all events to arrive when the node is slow. - while (allCashStates.size < NUMBER_OF_FLOWS_TO_RUN && nrRetries++ < 50) { - Thread.sleep(2000) - allCashStates = readCashStates() - } - - val allCash = allCashStates.map { it.state.data.amount.quantity }.toSet() - val missingCash = (1..NUMBER_OF_FLOWS_TO_RUN).filterNot { allCash.contains(it.toLong() * 100) } - log.info("Missing cash states: $missingCash") - - assertEquals(NUMBER_OF_FLOWS_TO_RUN, allCashStates.size, "Not all flows were executed successfully") - - // The progress status for each flow can only miss the last events, because the node might have been killed. - val missingProgressEvents = flowProgressEvents.filterValues { expectedProgress.subList(0, it.size) != it } - assertTrue(missingProgressEvents.isEmpty(), "The flow progress tracker is missing events: $missingProgressEvents") - - // DOCSTART missingVaultEvents - // Check that enough vault events were received. - // This check is fuzzy because events can go missing during node restarts. - // Ideally there should be nrOfFlowsToRun events receive but some might get lost for each restart. - assertThat(vaultEvents!!.size + nrFailures * 3).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN) - // DOCEND missingVaultEvents - - // Check that no flow was triggered twice. - val duplicates = allCashStates.groupBy { it.state.data.amount }.filterValues { it.size > 1 } - assertTrue(duplicates.isEmpty(), "${duplicates.size} flows were retried illegally.") - - log.info("State machine events seen: ${stateMachineEvents!!.size}") - // State machine events are very likely to get lost more often because they seem to be sent with a delay. - assertThat(stateMachineEvents.count { it is StateMachineUpdate.Added }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3) - assertThat(stateMachineEvents.count { it is StateMachineUpdate.Removed }).isGreaterThanOrEqualTo(NUMBER_OF_FLOWS_TO_RUN / 3) - - // Stop the observers. - vaultSubscription.unsubscribe() - stateMachineSubscription.unsubscribe() - bankAReconnectingRPCConnection.close() - } - - proxy.close() - } - - @Synchronized - fun MutableMap>.addEvent(id: StateMachineRunId, progress: String?): Boolean { - return getOrPut(id) { mutableListOf() }.let { if (progress != null) it.add(progress) else false } - } - private fun getRandomAddressPair() = AddressPair(getRandomAddress(), getRandomAddress()) - private fun getRandomAddress() = NetworkHostAndPort("localhost", portAllocator.nextPort()) - - data class AddressPair(val proxyAddress: NetworkHostAndPort, val nodeAddress: NetworkHostAndPort) -} diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt new file mode 100644 index 0000000000..8233cc79df --- /dev/null +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineErrorHandlingTest.kt @@ -0,0 +1,288 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.HospitalizeFlowException +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party +import net.corda.core.internal.list +import net.corda.core.internal.readAllLines +import net.corda.core.messaging.CordaRPCOps +import net.corda.core.messaging.startFlow +import net.corda.core.node.AppServiceHub +import net.corda.core.node.services.CordaService +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.core.utilities.unwrap +import net.corda.node.services.Permissions +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.driver.DriverDSL +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.NodeHandle +import net.corda.testing.driver.NodeParameters +import net.corda.testing.driver.driver +import net.corda.testing.driver.internal.OutOfProcessImpl +import net.corda.testing.node.NotarySpec +import net.corda.testing.node.TestCordapp +import net.corda.testing.node.User +import net.corda.testing.node.internal.InternalDriverDSL +import org.jboss.byteman.agent.submit.ScriptText +import org.jboss.byteman.agent.submit.Submit +import org.junit.Before +import java.time.Duration +import java.util.concurrent.TimeUnit +import kotlin.test.assertEquals + +abstract class StateMachineErrorHandlingTest { + + val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) + var counter = 0 + + @Before + fun setup() { + counter = 0 + } + + internal fun startDriver(notarySpec: NotarySpec = NotarySpec(DUMMY_NOTARY_NAME), dsl: DriverDSL.() -> Unit) { + driver( + DriverParameters( + notarySpecs = listOf(notarySpec), + startNodesInProcess = false, + inMemoryDB = false, + systemProperties = mapOf("co.paralleluniverse.fibers.verifyInstrumentation" to "true") + ) + ) { + dsl() + } + } + + internal fun DriverDSL.createBytemanNode( + providedName: CordaX500Name, + additionalCordapps: Collection = emptyList() + ): Pair { + val port = nextPort() + val nodeHandle = (this as InternalDriverDSL).startNode( + NodeParameters( + providedName = providedName, + rpcUsers = listOf(rpcUser), + additionalCordapps = additionalCordapps + ), + bytemanPort = port + ).getOrThrow() + return nodeHandle to port + } + + internal fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection = emptyList()): NodeHandle { + return startNode( + NodeParameters( + providedName = providedName, + rpcUsers = listOf(rpcUser), + additionalCordapps = additionalCordapps + ) + ).getOrThrow() + } + + internal fun submitBytemanRules(rules: String, port: Int) { + val submit = Submit("localhost", port) + submit.addScripts(listOf(ScriptText("Test script", rules))) + } + + internal fun getBytemanOutput(nodeHandle: NodeHandle): List { + return nodeHandle.baseDirectory + .list() + .first { it.toString().contains("net.corda.node.Corda") && it.toString().contains("stdout.log") } + .readAllLines() + } + + internal fun OutOfProcessImpl.stop(timeout: Duration): Boolean { + return process.run { + destroy() + waitFor(timeout.seconds, TimeUnit.SECONDS) + }.also { onStopCallback() } + } + + @Suppress("LongParameterList") + internal fun CordaRPCOps.assertHospitalCounts( + discharged: Int = 0, + observation: Int = 0, + propagated: Int = 0, + dischargedRetry: Int = 0, + observationRetry: Int = 0, + propagatedRetry: Int = 0 + ) { + val counts = startFlow(StateMachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.getOrThrow(20.seconds) + assertEquals(discharged, counts.discharged) + assertEquals(observation, counts.observation) + assertEquals(propagated, counts.propagated) + assertEquals(dischargedRetry, counts.dischargeRetry) + assertEquals(observationRetry, counts.observationRetry) + assertEquals(propagatedRetry, counts.propagatedRetry) + } + + internal fun CordaRPCOps.assertHospitalCountsAllZero() = assertHospitalCounts() + + internal fun CordaRPCOps.assertNumberOfCheckpoints( + runnable: Int = 0, + failed: Int = 0, + completed: Int = 0, + hospitalized: Int = 0 + ) { + val counts = startFlow(StateMachineErrorHandlingTest::GetNumberOfCheckpointsFlow).returnValue.getOrThrow(20.seconds) + assertEquals(runnable, counts.runnable, "There should be $runnable runnable checkpoints") + assertEquals(failed, counts.failed, "There should be $failed failed checkpoints") + assertEquals(completed, counts.completed, "There should be $completed completed checkpoints") + assertEquals(hospitalized, counts.hospitalized, "There should be $hospitalized hospitalized checkpoints") + } + + internal fun CordaRPCOps.assertNumberOfCheckpointsAllZero() = assertNumberOfCheckpoints() + + @StartableByRPC + @InitiatingFlow + class SendAMessageFlow(private val party: Party) : FlowLogic() { + @Suspendable + override fun call(): String { + val session = initiateFlow(party) + session.send("hello there") + logger.info("Finished my flow") + return "Finished executing test flow - ${this.runId}" + } + } + + @InitiatedBy(SendAMessageFlow::class) + class SendAMessageResponder(private val session: FlowSession) : FlowLogic() { + @Suspendable + override fun call() { + session.receive().unwrap { it } + logger.info("Finished my flow") + } + } + + @StartableByRPC + class ThrowAnErrorFlow : FlowLogic() { + @Suspendable + override fun call(): String { + throwException() + return "cant get here" + } + + private fun throwException() { + logger.info("Throwing exception in flow") + throw IllegalStateException("throwing exception in flow") + } + } + + @StartableByRPC + class ThrowAHospitalizeErrorFlow : FlowLogic() { + @Suspendable + override fun call(): String { + throwException() + return "cant get here" + } + + private fun throwException() { + logger.info("Throwing exception in flow") + throw HospitalizeFlowException("throwing exception in flow") + } + } + + @StartableByRPC + class GetNumberOfCheckpointsFlow : FlowLogic() { + override fun call() = NumberOfCheckpoints( + runnable = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.RUNNABLE), + failed = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.FAILED), + completed = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.COMPLETED), + hospitalized = getNumberOfCheckpointsWithStatus(Checkpoint.FlowStatus.HOSPITALIZED) + ) + + private fun getNumberOfCheckpointsWithStatus(status: Checkpoint.FlowStatus): Int { + return serviceHub.jdbcSession() + .prepareStatement("select count(*) from node_checkpoints where status = ? and flow_id != ?") + .apply { + setInt(1, status.ordinal) + setString(2, runId.uuid.toString()) + } + .use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + }.toInt() + } + } + + @CordaSerializable + data class NumberOfCheckpoints( + val runnable: Int = 0, + val failed: Int = 0, + val completed: Int = 0, + val hospitalized: Int = 0 + ) + + // Internal use for testing only!! + @StartableByRPC + class GetHospitalCountersFlow : FlowLogic() { + override fun call(): HospitalCounts = + HospitalCounts( + serviceHub.cordaService(HospitalCounter::class.java).dischargedCounter, + serviceHub.cordaService(HospitalCounter::class.java).observationCounter, + serviceHub.cordaService(HospitalCounter::class.java).propagatedCounter, + serviceHub.cordaService(HospitalCounter::class.java).dischargeRetryCounter, + serviceHub.cordaService(HospitalCounter::class.java).observationRetryCounter, + serviceHub.cordaService(HospitalCounter::class.java).propagatedRetryCounter + ) + } + + @CordaSerializable + data class HospitalCounts( + val discharged: Int, + val observation: Int, + val propagated: Int, + val dischargeRetry: Int, + val observationRetry: Int, + val propagatedRetry: Int + ) + + @Suppress("UNUSED_PARAMETER") + @CordaService + class HospitalCounter(services: AppServiceHub) : SingletonSerializeAsToken() { + var dischargedCounter: Int = 0 + var observationCounter: Int = 0 + var propagatedCounter: Int = 0 + var dischargeRetryCounter: Int = 0 + var observationRetryCounter: Int = 0 + var propagatedRetryCounter: Int = 0 + + init { + StaffedFlowHospital.onFlowDischarged.add { _, _ -> + dischargedCounter++ + } + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + observationCounter++ + } + StaffedFlowHospital.onFlowErrorPropagated.add { _, _ -> + propagatedCounter++ + } + StaffedFlowHospital.onFlowResuscitated.add { _, _, outcome -> + when (outcome) { + StaffedFlowHospital.Outcome.DISCHARGE -> dischargeRetryCounter++ + StaffedFlowHospital.Outcome.OVERNIGHT_OBSERVATION -> observationRetryCounter++ + StaffedFlowHospital.Outcome.UNTREATABLE -> propagatedRetryCounter++ + } + } + } + } + + internal val actionExecutorClassName: String by lazy { + Class.forName("net.corda.node.services.statemachine.ActionExecutorImpl").name + } + + internal val stateMachineManagerClassName: String by lazy { + Class.forName("net.corda.node.services.statemachine.SingleThreadedStateMachineManager").name + } +} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt similarity index 54% rename from node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt rename to node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt index 98e199afe2..0613fd277e 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineFinalityErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFinalityErrorHandlingTest.kt @@ -1,6 +1,5 @@ package net.corda.node.services.statemachine -import net.corda.client.rpc.CordaRPCClient import net.corda.core.flows.ReceiveFinalityFlow import net.corda.core.internal.ResolveTransactionsFlow import net.corda.core.messaging.startFlow @@ -22,7 +21,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFailsWith @Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped -class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { +class StateMachineFinalityErrorHandlingTest : StateMachineErrorHandlingTest() { /** * Throws an exception when recoding a transaction inside of [ReceiveFinalityFlow] on the responding @@ -33,10 +32,10 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its * send to the responding node and the responding node successfully received it. */ - @Test(timeout=300_000) - fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() { + @Test(timeout = 300_000) + fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) // could not get rule for FinalityDoctor + observation counter to work @@ -67,14 +66,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow( + alice.rpc.startFlow( ::CashIssueAndPaymentFlow, 500.DOLLARS, OpaqueBytes.of(0x01), @@ -83,15 +77,11 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { defaultNotaryIdentity ).returnValue.getOrThrow(30.seconds) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(0, discharge) - assertEquals(1, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - assertEquals(1, charlieClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1) + charlie.rpc.assertHospitalCounts(observation = 1) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + assertEquals(1, charlie.rpc.stateMachinesSnapshot().size) } } @@ -104,10 +94,10 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its * send to the responding node and the responding node successfully received it. */ - @Test(timeout=300_000) - fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() { + @Test(timeout = 300_000) + fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) // could not get rule for FinalityDoctor + observation counter to work @@ -138,14 +128,9 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow( + alice.rpc.startFlow( ::CashIssueAndPaymentFlow, 500.DOLLARS, OpaqueBytes.of(0x01), @@ -154,15 +139,11 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { defaultNotaryIdentity ).returnValue.getOrThrow(30.seconds) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(0, discharge) - assertEquals(1, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - assertEquals(1, charlieClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1) + charlie.rpc.assertHospitalCounts(observation = 1) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + assertEquals(1, charlie.rpc.stateMachinesSnapshot().size) } } @@ -170,22 +151,22 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding * flow's node. * - * The exception is thrown 5 times. + * The exception is thrown 3 times. * * The responding flow is retried 3 times and then completes successfully. * * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the * flow is retried instead of moving straight to observation. */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() { + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -201,38 +182,17 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("finality_flag") && readCounter("counter") < 5 + IF flagged("finality_flag") && readCounter("counter") < 3 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow( + alice.rpc.startFlow( ::CashIssueAndPaymentFlow, 500.DOLLARS, OpaqueBytes.of(0x01), @@ -241,20 +201,14 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { defaultNotaryIdentity ).returnValue.getOrThrow(30.seconds) - val output = getBytemanOutput(charlie) + // This sleep is a bit suspect... + Thread.sleep(1000) - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - assertEquals(0, charlieClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + assertEquals(0, charlie.rpc.stateMachinesSnapshot().size) } } @@ -262,7 +216,7 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding * flow's node. * - * The exception is thrown 7 times. + * The exception is thrown 4 times. * * The responding flow is retried 3 times and is then kept in for observation. * @@ -272,15 +226,15 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the * flow is retried instead of moving straight to observation. */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() { + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() { startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { - val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val (charlie, port) = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -296,39 +250,18 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("finality_flag") && readCounter("counter") < 7 + IF flagged("finality_flag") && readCounter("counter") < 4 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + submitBytemanRules(rules, port) assertFailsWith { - aliceClient.startFlow( + alice.rpc.startFlow( ::CashIssueAndPaymentFlow, 500.DOLLARS, OpaqueBytes.of(0x01), @@ -338,20 +271,14 @@ class StatemachineFinalityErrorHandlingTest : StatemachineErrorHandlingTest() { ).returnValue.getOrThrow(30.seconds) } - val output = getBytemanOutput(charlie) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - assertEquals(1, charlieClient.stateMachinesSnapshot().size) - // 1 for CashIssueAndPaymentFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpoints(runnable = 1) + charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1) + charlie.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + assertEquals(1, charlie.rpc.stateMachinesSnapshot().size) } } } \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt new file mode 100644 index 0000000000..c36d9750f0 --- /dev/null +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineFlowInitErrorHandlingTest.kt @@ -0,0 +1,581 @@ +package net.corda.node.services.statemachine + +import net.corda.core.CordaRuntimeException +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.node.services.api.CheckpointStorage +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.CHARLIE_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.internal.OutOfProcessImpl +import org.junit.Test +import java.sql.Connection +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeoutException +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped +class StateMachineFlowInitErrorHandlingTest : StateMachineErrorHandlingTest() { + + private companion object { + val executor: ExecutorService = Executors.newSingleThreadExecutor() + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint + * (remains in an unstarted state). + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + * + */ + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when calling [FlowStateMachineImpl.processEvent]. + * + * This is not an expected place for an exception to occur, but allows us to test what happens when a random exception is propagated + * up to [FlowStateMachineImpl.run] during flow initialisation. + * + * A "Transaction context is missing" exception is thrown due to where the exception is thrown (no transaction is created so this is + * thrown when leaving [FlowStateMachineImpl.processEventsUntilFlowIsResumed] due to the finally block). + */ + @Test(timeout = 300_000) + fun `unexpected error during flow initialisation throws exception to client`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + val rules = """ + RULE Create Counter + CLASS ${FlowStateMachineImpl::class.java.name} + METHOD processEvent + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception + CLASS ${FlowStateMachineImpl::class.java.name} + METHOD processEvent + AT ENTRY + IF readCounter("counter") < 1 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + assertFailsWith { + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow(30.seconds) + } + + alice.rpc.assertNumberOfCheckpoints(failed = 1) + alice.rpc.assertHospitalCounts(propagated = 1) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint + * (remains in an unstarted state). + * + * A [SQLException] is then thrown when trying to rollback the flow's database transaction. + * + * The [SQLException] should be suppressed and the flow should continue to retry and complete successfully. + */ + @Test(timeout = 300_000) + fun `error during initialisation when trying to rollback the flow's database transaction the flow is able to retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") == 0 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception when rolling back transaction in transition executor + INTERFACE ${Connection::class.java.name} + METHOD rollback + AT ENTRY + IF readCounter("counter") == 1 + DO incrementCounter("counter"); traceln("Throwing exception in transition executor"); throw new java.sql.SQLException("could not reach db", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow(30.seconds) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 1) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint + * (remains in an unstarted state). + * + * A [SQLException] is then thrown when trying to close the flow's database transaction. + * + * The [SQLException] should be suppressed and the flow should continue to retry and complete successfully. + */ + @Test(timeout = 300_000) + fun `error during initialisation when trying to close the flow's database transaction the flow is able to retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") == 0 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception when rolling back transaction in transition executor + INTERFACE ${Connection::class.java.name} + METHOD close + AT ENTRY + IF readCounter("counter") == 1 + DO incrementCounter("counter"); traceln("Throwing exception in transition executor"); throw new java.sql.SQLException("could not reach db", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow(30.seconds) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 1) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint + * (remains in an unstarted state). + * + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + */ + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + executor.execute { + alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) + } + + // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead + Thread.sleep(30.seconds.toMillis()) + + alice.rpc.assertNumberOfCheckpoints(hospitalized = 1) + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + val terminated = (alice as OutOfProcessImpl).stop(60.seconds) + assertTrue(terminated, "The node must be shutdown before it can be restarted") + val (alice2, _) = createBytemanNode(ALICE_NAME) + Thread.sleep(20.seconds.toMillis()) + alice2.rpc.assertNumberOfCheckpointsAllZero() + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has initialised and saved its first checkpoint + * (remains in an unstarted state). + * + * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another + * exception is then thrown during the retry itself. + * + * The flow then retries the retry causing the flow to complete successfully. + */ + @Test(timeout = 300_000) + fun `error during retrying a flow that failed when committing its original checkpoint will retry the flow again and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF !flagged("commit_exception_flag") + DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + + RULE Throw exception on retry + CLASS $stateMachineManagerClassName + METHOD onExternalStartFlow + AT ENTRY + IF flagged("commit_exception_flag") && !flagged("retry_exception_flag") + DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts( + discharged = 1, + dischargedRetry = 1 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event on a responding node before the flow has initialised and + * saved its first checkpoint (remains in an unstarted state). + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + */ + @Test(timeout = 300_000) + fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and complete successfully`() { + startDriver { + val (charlie, port) = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertHospitalCounts(discharged = 3) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event on a responding node before the flow has initialised and + * saved its first checkpoint (remains in an unstarted state). + * + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + */ + @Test(timeout = 300_000) + fun `responding flow - error during transition with CommitTransaction action that occurs during flow initialisation will retry and be kept for observation if error persists`() { + startDriver { + val (charlie, port) = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + executor.execute { + alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) + } + + // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead + Thread.sleep(30.seconds.toMillis()) + + alice.rpc.assertNumberOfCheckpoints(runnable = 1) + charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1) + charlie.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + assertEquals(1, charlie.rpc.stateMachinesSnapshot().size) + val terminated = (charlie as OutOfProcessImpl).stop(60.seconds) + assertTrue(terminated, "The node must be shutdown before it can be restarted") + val (charlie2, _) = createBytemanNode(CHARLIE_NAME) + Thread.sleep(10.seconds.toMillis()) + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie2.rpc.assertNumberOfCheckpointsAllZero() + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state) on a responding node. + * + * The exception is thrown 3 times. + * + * An exception is also thrown from [CheckpointStorage.getCheckpoint]. + * + * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when + * retrying the flow after it failed to commit it's original checkpoint. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test(timeout = 300_000) + fun `responding flow - session init can be retried when there is a transient connection error to the database`() { + startDriver { + val (charlie, port) = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception on getCheckpoint + INTERFACE ${CheckpointStorage::class.java.name} + METHOD getCheckpoint + AT ENTRY + IF true + DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertHospitalCounts( + discharged = 3, + observation = 0 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + assertEquals(0, charlie.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state) on a responding node. + * + * The exception is thrown 4 times. + * + * An exception is also thrown from [CheckpointStorage.getCheckpoint]. + * + * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when + * retrying the flow after it failed to commit it's original checkpoint. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * fails and is kept for in for observation. + */ + @Test(timeout = 300_000) + fun `responding flow - session init can be retried when there is a transient connection error to the database goes to observation if error persists`() { + startDriver { + val (charlie, port) = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception on getCheckpoint + INTERFACE ${CheckpointStorage::class.java.name} + METHOD getCheckpoint + AT ENTRY + IF true + DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + assertFailsWith { + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + } + + charlie.rpc.assertNumberOfCheckpoints(hospitalized = 1) + charlie.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + assertEquals(1, charlie.rpc.stateMachinesSnapshot().size) + } + } +} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt new file mode 100644 index 0000000000..c1af1bce1a --- /dev/null +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineGeneralErrorHandlingTest.kt @@ -0,0 +1,661 @@ +package net.corda.node.services.statemachine + +import net.corda.core.CordaRuntimeException +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.node.services.api.CheckpointStorage +import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.statemachine.transitions.TopLevelTransition +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.CHARLIE_NAME +import net.corda.testing.core.singleIdentity +import org.junit.Test +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeoutException +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped +class StateMachineGeneralErrorHandlingTest : StateMachineErrorHandlingTest() { + + private companion object { + val executor: ExecutorService = Executors.newSingleThreadExecutor() + } + + /** + * Throws an exception when performing an [Action.SendInitial] action. + * + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and is then kept in + * the hospital for observation. + */ + @Test(timeout = 300_000) + fun `error during transition with SendInitial action is retried 3 times and kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendMultiple action + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + assertFailsWith { + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + } + + alice.rpc.assertNumberOfCheckpoints(hospitalized = 1) + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.SendInitial] event. + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test(timeout = 300_000) + fun `error during transition with SendInitial action that does not persist will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendMultiple action + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when executing [DeduplicationHandler.afterDatabaseTransaction] from inside an [Action.AcknowledgeMessages] action. + * + * The exception is thrown every time [DeduplicationHandler.afterDatabaseTransaction] is executed inside of + * [ActionExecutorImpl.executeAcknowledgeMessages] + * + * The exceptions should be swallowed. Therefore there should be no trips to the hospital and no retries. + * The flow should complete successfully as the error is swallowed. + */ + @Test(timeout = 300_000) + fun `error during transition with AcknowledgeMessages action is swallowed and flow completes successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Set flag when inside executeAcknowledgeMessages + CLASS $actionExecutorClassName + METHOD executeAcknowledgeMessages + AT INVOKE ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction() + IF !flagged("exception_flag") + DO flag("exception_flag"); traceln("Setting flag to true") + ENDRULE + + RULE Throw exception when executing ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction when inside executeAcknowledgeMessages + INTERFACE ${DeduplicationHandler::class.java.name} + METHOD afterDatabaseTransaction + AT ENTRY + IF flagged("exception_flag") + DO traceln("Throwing exception"); clear("exception_flag"); traceln("SETTING FLAG TO FALSE"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCountsAllZero() + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event when trying to propagate an error (processing an + * [Event.StartErrorPropagation] event) + * + * The exception is thrown 3 times. + * + * This causes the flow to retry the [Event.StartErrorPropagation] event until it succeeds. This this scenario it is retried 3 times, + * on the final retry the flow successfully propagates the error and completes exceptionally. + */ + @Test(timeout = 300_000) + fun `error during error propagation the flow is able to retry and recover`() { + startDriver { + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ThrowAnErrorFlow::class.java.name} + METHOD throwException + AT ENTRY + IF !flagged("my_flag") + DO traceln("SETTING FLAG TO TRUE"); flag("my_flag") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("my_flag") && readCounter("counter") < 3 + DO traceln("Throwing exception"); incrementCounter("counter"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + assertFailsWith { + alice.rpc.startFlow(StateMachineErrorHandlingTest::ThrowAnErrorFlow).returnValue.getOrThrow(60.seconds) + } + + alice.rpc.assertNumberOfCheckpoints(failed = 1) + alice.rpc.assertHospitalCounts( + propagated = 1, + propagatedRetry = 3 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when replaying a flow that has already successfully created its initial checkpoint. + * + * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another + * exception is then thrown during the retry itself. + * + * The flow is discharged and replayed from the hospital. An exception is then thrown during the retry that causes the flow to be + * retried again. + */ + @Test(timeout = 300_000) + fun `error during flow retry when executing retryFlowFromSafePoint the flow is able to retry and recover`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Set flag when executing first suspend + CLASS ${TopLevelTransition::class.java.name} + METHOD suspendTransition + AT ENTRY + IF !flagged("suspend_flag") + DO flag("suspend_flag"); traceln("Setting suspend flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag") + DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + + RULE Set flag when executing first commit + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && !flagged("commit_flag") + DO flag("commit_flag"); traceln("Setting commit flag to true") + ENDRULE + + RULE Throw exception on retry + CLASS $stateMachineManagerClassName + METHOD addAndStartFlow + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("retry_exception_flag") + DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow(40.seconds) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts( + discharged = 1, + dischargedRetry = 1 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event after the flow has suspended (has moved to a started state). + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + */ + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action that occurs after the first suspend will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + // seems to be restarting the flow from the beginning every time + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when executing first suspend + CLASS ${TopLevelTransition::class.java.name} + METHOD suspendTransition + AT ENTRY + IF !flagged("suspend_flag") + DO flag("suspend_flag"); traceln("Setting suspend flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Set flag when executing first commit + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && !flagged("commit_flag") + DO flag("commit_flag"); traceln("Setting commit flag to true") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing. + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + */ + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + // seems to be restarting the flow from the beginning every time + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 + DO incrementCounter("counter"); clear("remove_checkpoint_flag"); traceln("Throwing exception"); throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws a [ConstraintViolationException] when performing an [Action.CommitTransaction] event when the flow is finishing. + * + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + */ + @Test(timeout = 300_000) + fun `error during transition with CommitTransaction action and ConstraintViolationException that occurs when completing a flow will retry and be kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 4 + DO incrementCounter("counter"); + clear("remove_checkpoint_flag"); + traceln("Throwing exception"); + throw new org.hibernate.exception.ConstraintViolationException("This flow has a terminal condition", new java.sql.SQLException(), "made up constraint") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + assertFailsWith { + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + } + + alice.rpc.assertNumberOfCheckpoints(hospitalized = 1) + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state). + * + * The exception is thrown 3 times. + * + * An exception is also thrown from [CheckpointStorage.getCheckpoint]. + * + * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when + * retrying the flow after it failed to commit it's original checkpoint. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test(timeout = 300_000) + fun `flow can be retried when there is a transient connection error to the database`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception on getCheckpoint + INTERFACE ${CheckpointStorage::class.java.name} + METHOD getCheckpoint + AT ENTRY + IF true + DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 0 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state). + * + * The exception is thrown 4 times. + * + * An exception is also thrown from [CheckpointStorage.getCheckpoint]. + * + * This test is to prevent a regression, where a transient database connection error can be thrown retrieving a flow's checkpoint when + * retrying the flow after it failed to commit it's original checkpoint. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * fails and is kept for in for observation. + */ + @Test(timeout = 300_000) + fun `flow can be retried when there is a transient connection error to the database goes to observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception on getCheckpoint + INTERFACE ${CheckpointStorage::class.java.name} + METHOD getCheckpoint + AT ENTRY + IF true + DO traceln("Throwing exception getting checkpoint"); throw new java.sql.SQLTransientConnectionException("Connection is not available") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + executor.execute { + alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) + } + + // flow is not signaled as started calls to [getOrThrow] will hang, sleeping instead + Thread.sleep(30.seconds.toMillis()) + + alice.rpc.assertNumberOfCheckpoints(hospitalized = 1) + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(1, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing on a responding node. + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test(timeout = 300_000) + fun `responding flow - error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { + startDriver { + val (charlie, port) = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS $actionExecutorClassName + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 + DO incrementCounter("counter"); + clear("remove_checkpoint_flag"); + traceln("Throwing exception"); + throw new java.sql.SQLException("die dammit die", "1") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + alice.rpc.startFlow( + StateMachineErrorHandlingTest::SendAMessageFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( + 30.seconds + ) + + alice.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertNumberOfCheckpointsAllZero() + charlie.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + assertEquals(0, charlie.rpc.stateMachinesSnapshot().size) + } + } +} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt new file mode 100644 index 0000000000..ee5699456d --- /dev/null +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineKillFlowErrorHandlingTest.kt @@ -0,0 +1,181 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.KilledFlowException +import net.corda.core.flows.StartableByRPC +import net.corda.core.messaging.startFlow +import net.corda.core.messaging.startTrackedFlow +import net.corda.core.utilities.ProgressTracker +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.CHARLIE_NAME +import net.corda.testing.core.singleIdentity +import org.junit.Test +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeoutException +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped +class StateMachineKillFlowErrorHandlingTest : StateMachineErrorHandlingTest() { + + /** + * Triggers `killFlow` while the flow is suspended causing a [InterruptedException] to be thrown and passed through the hospital. + * + * The flow terminates and is not retried. + * + * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. + */ + @Test(timeout = 300_000) + fun `error during transition due to killing a flow will terminate the flow`() { + startDriver { + val alice = createNode(ALICE_NAME) + + val flow = alice.rpc.startTrackedFlow(StateMachineKillFlowErrorHandlingTest::SleepFlow) + + var flowKilled = false + flow.progress.subscribe { + if (it == SleepFlow.STARTED.label) { + Thread.sleep(5000) + flowKilled = alice.rpc.killFlow(flow.id) + } + } + + assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } + + assertTrue(flowKilled) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCountsAllZero() + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Triggers `killFlow` during user application code. + * + * The user application code is mimicked by a [Thread.sleep] which is importantly not placed inside the [Suspendable] + * call function. Placing it inside a [Suspendable] function causes quasar to behave unexpectedly. + * + * Although the call to kill the flow is made during user application code. It will not be removed / stop processing + * until the next suspension point is reached within the flow. + * + * The flow terminates and is not retried. + * + * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. + */ + @Test(timeout = 300_000) + fun `flow killed during user code execution stops and removes the flow correctly`() { + startDriver { + val alice = createNode(ALICE_NAME) + + val flow = alice.rpc.startTrackedFlow(StateMachineKillFlowErrorHandlingTest::ThreadSleepFlow) + + var flowKilled = false + flow.progress.subscribe { + if (it == ThreadSleepFlow.STARTED.label) { + Thread.sleep(5000) + flowKilled = alice.rpc.killFlow(flow.id) + } + } + + assertFailsWith { flow.returnValue.getOrThrow(30.seconds) } + + assertTrue(flowKilled) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCountsAllZero() + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + /** + * Triggers `killFlow` after the flow has already been sent to observation. The flow is not running at this point and + * all that remains is its checkpoint in the database. + * + * The flow terminates and is not retried. + * + * Killing the flow does not lead to any passes through the hospital. All the recorded passes through the hospital are + * from the original flow that was put in for observation. + */ + @Test(timeout = 300_000) + fun `flow killed when it is in the flow hospital for observation is removed correctly`() { + startDriver { + val (alice, port) = createBytemanNode(ALICE_NAME) + val charlie = createNode(CHARLIE_NAME) + + val rules = """ + RULE Create Counter + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendMultiple action + CLASS $actionExecutorClassName + METHOD executeSendMultiple + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules, port) + + val flow = alice.rpc.startFlow(StateMachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) + + assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } + + alice.rpc.killFlow(flow.id) + + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts( + discharged = 3, + observation = 1 + ) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) + } + } + + @StartableByRPC + class SleepFlow : FlowLogic() { + + object STARTED : ProgressTracker.Step("I am ready to die") + + override val progressTracker = ProgressTracker(STARTED) + + @Suspendable + override fun call() { + sleep(Duration.of(1, ChronoUnit.SECONDS)) + progressTracker.currentStep = STARTED + sleep(Duration.of(2, ChronoUnit.MINUTES)) + } + } + + @StartableByRPC + class ThreadSleepFlow : FlowLogic() { + + object STARTED : ProgressTracker.Step("I am ready to die") + + override val progressTracker = ProgressTracker(STARTED) + + @Suspendable + override fun call() { + sleep(Duration.of(1, ChronoUnit.SECONDS)) + progressTracker.currentStep = STARTED + logger.info("Starting ${ThreadSleepFlow::class.qualifiedName} application sleep") + sleep() + logger.info("Finished ${ThreadSleepFlow::class.qualifiedName} application sleep") + sleep(Duration.of(2, ChronoUnit.MINUTES)) + } + + // Sleep is moved outside of `@Suspendable` function to prevent issues with Quasar + private fun sleep() { + Thread.sleep(20000) + } + } +} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt similarity index 57% rename from node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt rename to node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt index 161f3c4b39..5a9335136b 100644 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineSubflowErrorHandlingTest.kt +++ b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StateMachineSubFlowErrorHandlingTest.kt @@ -1,7 +1,6 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable -import net.corda.client.rpc.CordaRPCClient import net.corda.core.flows.FlowLogic import net.corda.core.flows.FlowSession import net.corda.core.flows.InitiatedBy @@ -20,13 +19,14 @@ import org.junit.Test import kotlin.test.assertEquals @Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped -class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { +class StateMachineSubFlowErrorHandlingTest : StateMachineErrorHandlingTest() { /** * This test checks that flow calling an initiating subflow will recover correctly. * * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first send to a counterparty. - * The exception is thrown 5 times. + * + * The exception is thrown 3 times. * * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition * succeeds and the flow finishes. @@ -37,15 +37,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify * that 3 retries are attempted before recovering. */ - @Test(timeout=300_000) - fun `initiating subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { + @Test(timeout = 300_000) + fun `initiating subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { startDriver { val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -69,66 +69,34 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("subflow_flag") && flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 5 + IF flagged("subflow_flag") && flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 3 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE RULE Set flag when executing first commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF flagged("subflow_flag") && flagged("suspend_flag") && !flagged("commit_flag") DO flag("commit_flag"); traceln("Setting commit flag to true") ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + alice.rpc.startFlow( + StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( 30.seconds ) - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) } } @@ -136,7 +104,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * This test checks that flow calling an initiating subflow will recover correctly. * * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first receive from a counterparty. - * The exception is thrown 5 times. + * + * The exception is thrown 3 times. * * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition * succeeds and the flow finishes. @@ -147,15 +116,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify * that 3 retries are attempted before recovering. */ - @Test(timeout=300_000) - fun `initiating subflow - error during transition with CommitTransaction action that occurs after the first receive will retry and complete successfully`() { + @Test(timeout = 300_000) + fun `initiating subflow - error during transition with CommitTransaction action that occurs after the first receive will retry and complete successfully`() { startDriver { val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -179,58 +148,26 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("subflow_flag") && flagged("suspend_flag") && readCounter("counter") < 5 + IF flagged("subflow_flag") && flagged("suspend_flag") && readCounter("counter") < 3 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + alice.rpc.startFlow( + StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInitiatingSubflowFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( 30.seconds ) - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) } } @@ -238,7 +175,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * This test checks that flow calling an inline subflow will recover correctly. * * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first send to a counterparty. - * The exception is thrown 5 times. + * + * The exception is thrown 3 times. * * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition * succeeds and the flow finishes. @@ -249,15 +187,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify * that 3 retries are attempted before recovering. */ - @Test(timeout=300_000) - fun `inline subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { + @Test(timeout = 300_000) + fun `inline subflow - error during transition with CommitTransaction action that occurs during the first send will retry and complete successfully`() { startDriver { val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -273,58 +211,26 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("subflow_flag") && readCounter("counter") < 5 + IF flagged("subflow_flag") && readCounter("counter") < 3 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + alice.rpc.startFlow( + StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( 30.seconds ) - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) } } @@ -332,7 +238,8 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * This test checks that flow calling an inline subflow will recover correctly. * * Throws an exception when performing an [Action.CommitTransaction] event during the subflow's first receive from a counterparty. - * The exception is thrown 5 times. + * + * The exception is thrown 3 times. * * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition * succeeds and the flow finishes. @@ -343,15 +250,15 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify * that 3 retries are attempted before recovering. */ - @Test(timeout=300_000) - fun `inline subflow - error during transition with CommitTransaction action that occurs during the first receive will retry and complete successfully`() { + @Test(timeout = 300_000) + fun `inline subflow - error during transition with CommitTransaction action that occurs during the first receive will retry and complete successfully`() { startDriver { val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) + val (alice, port) = createBytemanNode(ALICE_NAME) val rules = """ RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF createCounter("counter", $counter) @@ -367,66 +274,34 @@ class StatemachineSubflowErrorHandlingTest : StatemachineErrorHandlingTest() { ENDRULE RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY - IF flagged("subflow_flag") && flagged("commit_flag") && readCounter("counter") < 5 + IF flagged("subflow_flag") && flagged("commit_flag") && readCounter("counter") < 3 DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") ENDRULE RULE Set flag when executing first commit - CLASS ${ActionExecutorImpl::class.java.name} + CLASS $actionExecutorClassName METHOD executeCommitTransaction AT ENTRY IF flagged("subflow_flag") && !flagged("commit_flag") DO flag("commit_flag"); traceln("Setting commit flag to true") ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE """.trimIndent() - submitBytemanRules(rules) + submitBytemanRules(rules, port) - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineSubflowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + alice.rpc.startFlow( + StateMachineSubFlowErrorHandlingTest::SendAMessageInAnInlineSubflowFlow, + charlie.nodeInfo.singleIdentity() + ).returnValue.getOrThrow( 30.seconds ) - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) + alice.rpc.assertNumberOfCheckpointsAllZero() + alice.rpc.assertHospitalCounts(discharged = 3) + assertEquals(0, alice.rpc.stateMachinesSnapshot().size) } } diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt deleted file mode 100644 index 20b85b98c1..0000000000 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt +++ /dev/null @@ -1,166 +0,0 @@ -package net.corda.node.services.statemachine - -import co.paralleluniverse.fibers.Suspendable -import net.corda.core.flows.FlowLogic -import net.corda.core.flows.FlowSession -import net.corda.core.flows.InitiatedBy -import net.corda.core.flows.InitiatingFlow -import net.corda.core.flows.StartableByRPC -import net.corda.core.identity.CordaX500Name -import net.corda.core.identity.Party -import net.corda.core.internal.list -import net.corda.core.internal.readAllLines -import net.corda.core.node.AppServiceHub -import net.corda.core.node.services.CordaService -import net.corda.core.serialization.CordaSerializable -import net.corda.core.serialization.SingletonSerializeAsToken -import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.unwrap -import net.corda.node.services.Permissions -import net.corda.testing.core.DUMMY_NOTARY_NAME -import net.corda.testing.driver.DriverDSL -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeHandle -import net.corda.testing.driver.NodeParameters -import net.corda.testing.driver.driver -import net.corda.testing.node.NotarySpec -import net.corda.testing.node.TestCordapp -import net.corda.testing.node.User -import net.corda.testing.node.internal.InternalDriverDSL -import org.jboss.byteman.agent.submit.ScriptText -import org.jboss.byteman.agent.submit.Submit -import org.junit.Before - -abstract class StatemachineErrorHandlingTest { - - val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) - var counter = 0 - - @Before - fun setup() { - counter = 0 - } - - internal fun startDriver(notarySpec: NotarySpec = NotarySpec(DUMMY_NOTARY_NAME), dsl: DriverDSL.() -> Unit) { - driver( - DriverParameters( - notarySpecs = listOf(notarySpec), - startNodesInProcess = false, - inMemoryDB = false, - systemProperties = mapOf("co.paralleluniverse.fibers.verifyInstrumentation" to "true") - ) - ) { - dsl() - } - } - - internal fun DriverDSL.createBytemanNode( - providedName: CordaX500Name, - additionalCordapps: Collection = emptyList() - ): NodeHandle { - return (this as InternalDriverDSL).startNode( - NodeParameters( - providedName = providedName, - rpcUsers = listOf(rpcUser), - additionalCordapps = additionalCordapps - ), - bytemanPort = 12000 - ).getOrThrow() - } - - internal fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection = emptyList()): NodeHandle { - return startNode( - NodeParameters( - providedName = providedName, - rpcUsers = listOf(rpcUser), - additionalCordapps = additionalCordapps - ) - ).getOrThrow() - } - - internal fun submitBytemanRules(rules: String) { - val submit = Submit("localhost", 12000) - submit.addScripts(listOf(ScriptText("Test script", rules))) - } - - internal fun getBytemanOutput(nodeHandle: NodeHandle): List { - return nodeHandle.baseDirectory - .list() - .filter { it.toString().contains("net.corda.node.Corda") && it.toString().contains("stdout.log") } - .flatMap { it.readAllLines() } - } - - @StartableByRPC - @InitiatingFlow - class SendAMessageFlow(private val party: Party) : FlowLogic() { - @Suspendable - override fun call(): String { - val session = initiateFlow(party) - session.send("hello there") - return "Finished executing test flow - ${this.runId}" - } - } - - @InitiatedBy(SendAMessageFlow::class) - class SendAMessageResponder(private val session: FlowSession) : FlowLogic() { - @Suspendable - override fun call() { - session.receive().unwrap { it } - } - } - - @StartableByRPC - class GetNumberOfUncompletedCheckpointsFlow : FlowLogic() { - override fun call(): Long { - val sqlStatement = "select count(*) from node_checkpoints where status not in (${Checkpoint.FlowStatus.COMPLETED.ordinal})" - return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> - ps.executeQuery().use { rs -> - rs.next() - rs.getLong(1) - } - } - } - } - - @StartableByRPC - class GetNumberOfHospitalizedCheckpointsFlow : FlowLogic() { - override fun call(): Long { - val sqlStatement = "select count(*) from node_checkpoints where status in (${Checkpoint.FlowStatus.HOSPITALIZED.ordinal})" - return serviceHub.jdbcSession().prepareStatement(sqlStatement).use { ps -> - ps.executeQuery().use { rs -> - rs.next() - rs.getLong(1) - } - } - } - } - - // Internal use for testing only!! - @StartableByRPC - class GetHospitalCountersFlow : FlowLogic() { - override fun call(): HospitalCounts = - HospitalCounts( - serviceHub.cordaService(HospitalCounter::class.java).dischargeCounter, - serviceHub.cordaService(HospitalCounter::class.java).observationCounter - ) - } - - @CordaSerializable - data class HospitalCounts(val discharge: Int, val observation: Int) - - @Suppress("UNUSED_PARAMETER") - @CordaService - class HospitalCounter(services: AppServiceHub) : SingletonSerializeAsToken() { - var observationCounter: Int = 0 - var dischargeCounter: Int = 0 - - init { - StaffedFlowHospital.onFlowDischarged.add { _, _ -> - ++dischargeCounter - } - StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> - ++observationCounter - } - } - } -} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt deleted file mode 100644 index 349ab5e5a8..0000000000 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineGeneralErrorHandlingTest.kt +++ /dev/null @@ -1,1281 +0,0 @@ -package net.corda.node.services.statemachine - -import net.corda.client.rpc.CordaRPCClient -import net.corda.core.messaging.startFlow -import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.seconds -import net.corda.node.services.messaging.DeduplicationHandler -import net.corda.node.services.statemachine.transitions.TopLevelTransition -import net.corda.testing.core.ALICE_NAME -import net.corda.testing.core.CHARLIE_NAME -import net.corda.testing.core.singleIdentity -import org.junit.Ignore -import org.junit.Test -import java.util.concurrent.TimeoutException -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith - -@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped -class StatemachineGeneralErrorHandlingTest : StatemachineErrorHandlingTest() { - - /** - * Throws an exception when performing an [Action.SendInitial] action. - * The exception is thrown 4 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and is then kept in - * the hospital for observation. - */ - @Test(timeout=300_000) - fun `error during transition with SendInitial action is retried 3 times and kept for observation if error persists`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeSendMultiple action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF readCounter("counter") < 4 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.SendInitial] event. - * The exception is thrown 3 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - */ - @Test(timeout=300_000) - fun `error during transition with SendInitial action that does not persist will retry and complete successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeSendMultiple action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF readCounter("counter") < 3 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when executing [DeduplicationHandler.afterDatabaseTransaction] from - * inside an [Action.AcknowledgeMessages] action. - * The exception is thrown every time [DeduplicationHandler.afterDatabaseTransaction] is executed - * inside of [ActionExecutorImpl.executeAcknowledgeMessages] - * - * The exceptions should be swallowed. Therefore there should be no trips to the hospital and no retries. - * The flow should complete successfully as the error is swallowed. - */ - @Test(timeout=300_000) - fun `error during transition with AcknowledgeMessages action is swallowed and flow completes successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Set flag when inside executeAcknowledgeMessages - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeAcknowledgeMessages - AT INVOKE ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction() - IF !flagged("exception_flag") - DO flag("exception_flag"); traceln("Setting flag to true") - ENDRULE - - RULE Throw exception when executing ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction when inside executeAcknowledgeMessages - INTERFACE ${DeduplicationHandler::class.java.name} - METHOD afterDatabaseTransaction - AT ENTRY - IF flagged("exception_flag") - DO traceln("Throwing exception"); clear("exception_flag"); traceln("SETTING FLAG TO FALSE"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(0, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted - * state). - * The exception is thrown 5 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - * - * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). - * - * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition - * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to - * verify that 3 retries are attempted before recovering. - */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and complete successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF readCounter("counter") < 5 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted - * state). - * The exception is thrown 7 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. - * - * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). - * - * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition - * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to - * verify that 3 retries are attempted before recovering. - * - * CORDA-3352 - it is currently hanging after putting the flow in for observation - */ - @Test(timeout=300_000) -@Ignore - fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and be kept for observation if error persists`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF readCounter("counter") < 7 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event after the flow has suspended (has moved to a started state). - * The exception is thrown 5 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - * - * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. - * - * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition - * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to - * verify that 3 retries are attempted before recovering. - */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action that occurs after the first suspend will retry and complete successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - // seems to be restarting the flow from the beginning every time - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Set flag when executing first suspend - CLASS ${TopLevelTransition::class.java.name} - METHOD suspendTransition - AT ENTRY - IF !flagged("suspend_flag") - DO flag("suspend_flag"); traceln("Setting suspend flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 5 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Set flag when executing first commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && !flagged("commit_flag") - DO flag("commit_flag"); traceln("Setting commit flag to true") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing. - * The exception is thrown 3 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - * - * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. - */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - // seems to be restarting the flow from the beginning every time - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Set flag when adding action to remove checkpoint - CLASS ${TopLevelTransition::class.java.name} - METHOD flowFinishTransition - AT ENTRY - IF !flagged("remove_checkpoint_flag") - DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction when removing checkpoint - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 - DO incrementCounter("counter"); clear("remove_checkpoint_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when replaying a flow that has already successfully created its initial checkpoint. - * - * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another - * exception is then thrown during the retry itself. - * - * The flow is discharged and replayed from the hospital once. After failing during the replay, the flow is forced into overnight - * observation. It is not ran again after this point - */ - @Test(timeout=300_000) - fun `error during retry of a flow will force the flow into overnight observation`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Set flag when executing first suspend - CLASS ${TopLevelTransition::class.java.name} - METHOD suspendTransition - AT ENTRY - IF !flagged("suspend_flag") - DO flag("suspend_flag"); traceln("Setting suspend flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag") - DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Set flag when executing first commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && !flagged("commit_flag") - DO flag("commit_flag"); traceln("Setting commit flag to true") - ENDRULE - - RULE Throw exception on retry - CLASS ${SingleThreadedStateMachineManager::class.java.name} - METHOD addAndStartFlow - AT ENTRY - IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("retry_exception_flag") - DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(1, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when replaying a flow that has already successfully created its initial checkpoint. - * - * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another - * exception is then thrown during the database commit that comes as part of retrying a flow. - * - * The flow is discharged and replayed from the hospital once. When the database commit failure occurs as part of retrying the - * flow, the starting and completion of the retried flow is affected. In other words, the error occurs as part of the replay, but the - * flow will still finish successfully. This is due to the even being scheduled as part of the retry and the failure in the database - * commit occurs after this point. As the flow is already scheduled, the failure has not affect on it. - */ - @Test(timeout=300_000) - fun `error during commit transaction action when retrying a flow will retry the flow again and complete successfully`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Set flag when executing first suspend - CLASS ${TopLevelTransition::class.java.name} - METHOD suspendTransition - AT ENTRY - IF !flagged("suspend_flag") - DO flag("suspend_flag"); traceln("Setting suspend flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag") - DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Set flag when executing first commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && !flagged("commit_flag") - DO flag("commit_flag"); traceln("Setting commit flag to true") - ENDRULE - - RULE Throw exception on retry - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("suspend_flag") && flagged("commit_exception_flag") && !flagged("retry_exception_flag") - DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(1, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when replaying a flow that has not made its initial checkpoint. - * - * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another - * exception is then thrown during the retry itself. - * - * The flow is discharged and replayed from the hospital once. After failing during the replay, the flow is forced into overnight - * observation. It is not ran again after this point - * - * CORDA-3352 - it is currently hanging after putting the flow in for observation - * - */ - @Test(timeout=300_000) -@Ignore - fun `error during retrying a flow that failed when committing its original checkpoint will force the flow into overnight observation`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Throw exception on executeCommitTransaction action after first suspend + commit - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF !flagged("commit_exception_flag") - DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Throw exception on retry - CLASS ${SingleThreadedStateMachineManager::class.java.name} - METHOD onExternalStartFlow - AT ENTRY - IF flagged("commit_exception_flag") && !flagged("retry_exception_flag") - DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(1, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws a [ConstraintViolationException] when performing an [Action.CommitTransaction] event when the flow is finishing. - * The exception is thrown 4 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. - * - * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. - */ - @Test(timeout=300_000) - fun `error during transition with CommitTransaction action and ConstraintViolationException that occurs when completing a flow will retry and be kept for observation if error persists`() { - startDriver { - val charlie = createNode(CHARLIE_NAME) - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Set flag when adding action to remove checkpoint - CLASS ${TopLevelTransition::class.java.name} - METHOD flowFinishTransition - AT ENTRY - IF !flagged("remove_checkpoint_flag") - DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction when removing checkpoint - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("remove_checkpoint_flag") && readCounter("counter") < 4 - DO incrementCounter("counter"); - clear("remove_checkpoint_flag"); - traceln("Throwing exception"); - throw new org.hibernate.exception.ConstraintViolationException("This flow has a terminal condition", new java.sql.SQLException(), "made up constraint") - ENDRULE - - RULE Entering duplicate insert staff member - CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached duplicate insert staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment not my speciality counter - CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} - METHOD consult - AT READ NOT_MY_SPECIALTY - IF true - DO traceln("Byteman test - not my speciality") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - not my speciality") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - // 1 for errored flow and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event on a responding flow. The failure prevents the node from saving - * its original checkpoint. - * - * The exception is thrown 5 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - * - * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). - * - * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition - * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify - * that 3 retries are attempted before recovering. - */ - @Test(timeout=300_000) - fun `responding flow - error during transition with CommitTransaction action that occurs during the beginning of execution will retry and complete successfully`() { - startDriver { - val charlie = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF readCounter("counter") < 5 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(charlie) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - assertEquals(0, charlieClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event on a responding flow. The failure prevents the node from saving - * its original checkpoint. - * - * The exception is thrown 5 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. - * - * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). - * - * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition - * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify - * that 3 retries are attempted before recovering. - * - * The final asserts for checking the checkpoints on the nodes are correct since the responding node can replay the flow starting events - * from artemis. Therefore, the checkpoint is missing due the failures from saving the original checkpoint. But, the node will still be - * able to recover when the node is restarted (by using the events). The initiating flow maintains the checkpoint as it is waiting for - * the responding flow to recover and finish its flow. - */ - @Test(timeout=300_000) - fun `responding flow - error during transition with CommitTransaction action that occurs during the beginning of execution will retry and be kept for observation if error persists`() { - startDriver { - val charlie = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeCommitTransaction action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF readCounter("counter") < 7 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - assertFailsWith { - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - } - - val output = getBytemanOutput(charlie) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(1, aliceClient.stateMachinesSnapshot().size) - assertEquals(1, charlieClient.stateMachinesSnapshot().size) - // 1 for the flow that is waiting for the errored counterparty flow to finish and 1 for GetNumberOfCheckpointsFlow - assertEquals(2, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for GetNumberOfCheckpointsFlow - // a hospitalized flow is saved as the original checkpoint kept failing to commit - // the flow will recover since artemis will keep the events and replay them on node restart - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfHospitalizedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing on a responding node. - * - * The exception is thrown 3 times. - * - * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition - * succeeds and the flow finishes. - */ - @Test(timeout=300_000) - fun `responding flow - error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { - startDriver { - val charlie = createBytemanNode(CHARLIE_NAME) - val alice = createNode(ALICE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Set flag when adding action to remove checkpoint - CLASS ${TopLevelTransition::class.java.name} - METHOD flowFinishTransition - AT ENTRY - IF !flagged("remove_checkpoint_flag") - DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") - ENDRULE - - RULE Throw exception on executeCommitTransaction when removing checkpoint - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeCommitTransaction - AT ENTRY - IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 - DO incrementCounter("counter"); - clear("remove_checkpoint_flag"); - traceln("Throwing exception"); - throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - val charlieClient = - CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( - 30.seconds - ) - - val output = getBytemanOutput(charlie) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = charlieClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - assertEquals(0, charlieClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, charlieClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } -} \ No newline at end of file diff --git a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt b/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt deleted file mode 100644 index 0d0c8f7177..0000000000 --- a/node/src/integration-test-slow/kotlin/net/corda/node/services/statemachine/StatemachineKillFlowErrorHandlingTest.kt +++ /dev/null @@ -1,321 +0,0 @@ -package net.corda.node.services.statemachine - -import co.paralleluniverse.fibers.Suspendable -import net.corda.client.rpc.CordaRPCClient -import net.corda.core.flows.FlowLogic -import net.corda.core.flows.KilledFlowException -import net.corda.core.flows.StartableByRPC -import net.corda.core.messaging.startFlow -import net.corda.core.messaging.startTrackedFlow -import net.corda.core.utilities.ProgressTracker -import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.seconds -import net.corda.testing.core.ALICE_NAME -import net.corda.testing.core.CHARLIE_NAME -import net.corda.testing.core.singleIdentity -import org.junit.Test -import java.time.Duration -import java.time.temporal.ChronoUnit -import java.util.concurrent.TimeoutException -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped -class StatemachineKillFlowErrorHandlingTest : StatemachineErrorHandlingTest() { - - /** - * Triggers `killFlow` while the flow is suspended causing a [InterruptedException] to be thrown and passed through the hospital. - * - * The flow terminates and is not retried. - * - * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. - */ - @Test(timeout=300_000) - fun `error during transition due to killing a flow will terminate the flow`() { - startDriver { - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - - RULE Increment terminal counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ TERMINAL - IF true - DO traceln("Byteman test - terminal") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - val flow = aliceClient.startTrackedFlow(StatemachineKillFlowErrorHandlingTest::SleepFlow) - - var flowKilled = false - flow.progress.subscribe { - if (it == SleepFlow.STARTED.label) { - Thread.sleep(5000) - flowKilled = aliceClient.killFlow(flow.id) - } - } - - assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } - - val output = getBytemanOutput(alice) - - assertTrue(flowKilled) - // Check the stdout for the lines generated by byteman - assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(0, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Triggers `killFlow` during user application code. - * - * The user application code is mimicked by a [Thread.sleep] which is importantly not placed inside the [Suspendable] - * call function. Placing it inside a [Suspendable] function causes quasar to behave unexpectedly. - * - * Although the call to kill the flow is made during user application code. It will not be removed / stop processing - * until the next suspension point is reached within the flow. - * - * The flow terminates and is not retried. - * - * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. - */ - @Test(timeout=300_000) - fun `flow killed during user code execution stops and removes the flow correctly`() { - startDriver { - val alice = createBytemanNode(ALICE_NAME) - - val rules = """ - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - - RULE Increment terminal counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ TERMINAL - IF true - DO traceln("Byteman test - terminal") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - val flow = aliceClient.startTrackedFlow(StatemachineKillFlowErrorHandlingTest::ThreadSleepFlow) - - var flowKilled = false - flow.progress.subscribe { - if (it == ThreadSleepFlow.STARTED.label) { - Thread.sleep(5000) - flowKilled = aliceClient.killFlow(flow.id) - } - } - - assertFailsWith { flow.returnValue.getOrThrow(30.seconds) } - - val output = getBytemanOutput(alice) - - assertTrue(flowKilled) - // Check the stdout for the lines generated by byteman - assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) - val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size - println(numberOfTerminalDiagnoses) - assertEquals(0, numberOfTerminalDiagnoses) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(0, discharge) - assertEquals(0, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - /** - * Triggers `killFlow` after the flow has already been sent to observation. The flow is not running at this point and - * all that remains is its checkpoint in the database. - * - * The flow terminates and is not retried. - * - * Killing the flow does not lead to any passes through the hospital. All the recorded passes through the hospital are - * from the original flow that was put in for observation. - */ - @Test(timeout=300_000) - fun `flow killed when it is in the flow hospital for observation is removed correctly`() { - startDriver { - val alice = createBytemanNode(ALICE_NAME) - val charlie = createNode(CHARLIE_NAME) - - val rules = """ - RULE Create Counter - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF createCounter("counter", $counter) - DO traceln("Counter created") - ENDRULE - - RULE Throw exception on executeSendMultiple action - CLASS ${ActionExecutorImpl::class.java.name} - METHOD executeSendMultiple - AT ENTRY - IF readCounter("counter") < 4 - DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") - ENDRULE - - RULE Entering internal error staff member - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT ENTRY - IF true - DO traceln("Reached internal transition error staff member") - ENDRULE - - RULE Increment discharge counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ DISCHARGE - IF true - DO traceln("Byteman test - discharging") - ENDRULE - - RULE Increment observation counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ OVERNIGHT_OBSERVATION - IF true - DO traceln("Byteman test - overnight observation") - ENDRULE - - RULE Increment terminal counter - CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} - METHOD consult - AT READ TERMINAL - IF true - DO traceln("Byteman test - terminal") - ENDRULE - """.trimIndent() - - submitBytemanRules(rules) - - val aliceClient = - CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy - - val flow = aliceClient.startFlow(StatemachineErrorHandlingTest::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) - - assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } - - aliceClient.killFlow(flow.id) - - val output = getBytemanOutput(alice) - - // Check the stdout for the lines generated by byteman - assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) - assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) - val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size - assertEquals(0, numberOfTerminalDiagnoses) - val (discharge, observation) = aliceClient.startFlow(StatemachineErrorHandlingTest::GetHospitalCountersFlow).returnValue.get() - assertEquals(3, discharge) - assertEquals(1, observation) - assertEquals(0, aliceClient.stateMachinesSnapshot().size) - // 1 for GetNumberOfCheckpointsFlow - assertEquals(1, aliceClient.startFlow(StatemachineErrorHandlingTest::GetNumberOfUncompletedCheckpointsFlow).returnValue.get()) - } - } - - @StartableByRPC - class SleepFlow : FlowLogic() { - - object STARTED : ProgressTracker.Step("I am ready to die") - - override val progressTracker = ProgressTracker(STARTED) - - @Suspendable - override fun call() { - sleep(Duration.of(1, ChronoUnit.SECONDS)) - progressTracker.currentStep = STARTED - sleep(Duration.of(2, ChronoUnit.MINUTES)) - } - } - - @StartableByRPC - class ThreadSleepFlow : FlowLogic() { - - object STARTED : ProgressTracker.Step("I am ready to die") - - override val progressTracker = ProgressTracker(STARTED) - - @Suspendable - override fun call() { - sleep(Duration.of(1, ChronoUnit.SECONDS)) - progressTracker.currentStep = STARTED - logger.info("Starting ${ThreadSleepFlow::class.qualifiedName} application sleep") - sleep() - logger.info("Finished ${ThreadSleepFlow::class.qualifiedName} application sleep") - sleep(Duration.of(2, ChronoUnit.MINUTES)) - } - - // Sleep is moved outside of `@Suspendable` function to prevent issues with Quasar - private fun sleep() { - Thread.sleep(20000) - } - } -} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/DataObject.kt b/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/DataObject.kt new file mode 100644 index 0000000000..6384bd3900 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/DataObject.kt @@ -0,0 +1,14 @@ +package net.corda.contracts.serialization.generics + +import net.corda.core.serialization.CordaSerializable + +@CordaSerializable +data class DataObject(val value: Long) : Comparable { + override fun toString(): String { + return "$value data points" + } + + override fun compareTo(other: DataObject): Int { + return value.compareTo(other.value) + } +} diff --git a/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/GenericTypeContract.kt b/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/GenericTypeContract.kt new file mode 100644 index 0000000000..38a236b28f --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/contracts/serialization/generics/GenericTypeContract.kt @@ -0,0 +1,47 @@ +package net.corda.contracts.serialization.generics + +import net.corda.core.contracts.CommandData +import net.corda.core.contracts.Contract +import net.corda.core.contracts.ContractState +import net.corda.core.contracts.requireThat +import net.corda.core.identity.AbstractParty +import net.corda.core.transactions.LedgerTransaction +import java.util.Optional + +@Suppress("unused") +class GenericTypeContract : Contract { + override fun verify(tx: LedgerTransaction) { + val states = tx.outputsOfType() + requireThat { + "Requires at least one data state" using states.isNotEmpty() + } + val purchases = tx.commandsOfType() + requireThat { + "Requires at least one purchase" using purchases.isNotEmpty() + } + for (purchase in purchases) { + requireThat { + "Purchase has a price" using purchase.value.price.isPresent + } + } + } + + @Suppress("CanBeParameter", "MemberVisibilityCanBePrivate") + class State(val owner: AbstractParty, val data: DataObject?) : ContractState { + override val participants: List = listOf(owner) + + @Override + override fun toString(): String { + return data.toString() + } + } + + /** + * The [price] field is the important feature of the [Purchase] + * class because its type is [Optional] with a CorDapp-specific + * generic type parameter. It does not matter that the [price] + * is not used; it only matters that the [Purchase] command + * must be serialized as part of building a new transaction. + */ + class Purchase(val price: Optional) : CommandData +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/flows/serialization/generics/GenericTypeFlow.kt b/node/src/integration-test/kotlin/net/corda/flows/serialization/generics/GenericTypeFlow.kt new file mode 100644 index 0000000000..b5dc4abe29 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/flows/serialization/generics/GenericTypeFlow.kt @@ -0,0 +1,27 @@ +package net.corda.flows.serialization.generics + +import co.paralleluniverse.fibers.Suspendable +import net.corda.contracts.serialization.generics.DataObject +import net.corda.contracts.serialization.generics.GenericTypeContract.Purchase +import net.corda.contracts.serialization.generics.GenericTypeContract.State +import net.corda.core.contracts.Command +import net.corda.core.crypto.SecureHash +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.StartableByRPC +import net.corda.core.transactions.TransactionBuilder +import java.util.Optional + +@StartableByRPC +class GenericTypeFlow(private val purchase: DataObject?) : FlowLogic() { + @Suspendable + override fun call(): SecureHash { + val notary = serviceHub.networkMapCache.notaryIdentities[0] + val stx = serviceHub.signInitialTransaction( + TransactionBuilder(notary) + .addOutputState(State(ourIdentity, purchase)) + .addCommand(Command(Purchase(Optional.ofNullable(purchase)), ourIdentity.owningKey)) + ) + stx.verify(serviceHub, checkSufficientSignatures = false) + return stx.id + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt b/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt index 70293b6074..e2795b83b3 100644 --- a/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/AuthDBTests.kt @@ -15,6 +15,7 @@ import net.corda.node.services.Permissions import net.corda.node.services.config.PasswordEncryption import net.corda.testing.core.ALICE_NAME import net.corda.testing.node.internal.NodeBasedTest +import net.corda.testing.node.internal.cordappForClasses import org.apache.activemq.artemis.api.core.ActiveMQSecurityException import org.apache.shiro.authc.credential.DefaultPasswordService import org.junit.After @@ -32,7 +33,7 @@ import kotlin.test.assertFailsWith * check authentication/authorization of RPC connections. */ @RunWith(Parameterized::class) -class AuthDBTests : NodeBasedTest() { +class AuthDBTests : NodeBasedTest(cordappPackages = CORDAPPS) { private lateinit var node: NodeWithInfo private lateinit var client: CordaRPCClient private lateinit var db: UsersDB @@ -43,6 +44,9 @@ class AuthDBTests : NodeBasedTest() { @JvmStatic @Parameterized.Parameters(name = "password encryption format = {0}") fun encFormats() = arrayOf(PasswordEncryption.NONE, PasswordEncryption.SHIRO_1_CRYPT) + + @Suppress("SpreadOperator") + private val CORDAPPS = setOf(cordappForClasses(*AuthDBTests::class.nestedClasses.map { it.java }.toTypedArray())) } @Suppress("MemberVisibilityCanBePrivate") diff --git a/node/src/integration-test/kotlin/net/corda/node/ContractWithGenericTypeTest.kt b/node/src/integration-test/kotlin/net/corda/node/ContractWithGenericTypeTest.kt new file mode 100644 index 0000000000..d23c137dda --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/ContractWithGenericTypeTest.kt @@ -0,0 +1,78 @@ +package net.corda.node + +import net.corda.client.rpc.CordaRPCClient +import net.corda.contracts.serialization.generics.DataObject +import net.corda.core.contracts.TransactionVerificationException.ContractRejection +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.loggerFor +import net.corda.flows.serialization.generics.GenericTypeFlow +import net.corda.node.services.Permissions +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.driver.internal.incrementalPortAllocation +import net.corda.testing.node.NotarySpec +import net.corda.testing.node.User +import net.corda.testing.node.internal.cordappWithPackages +import org.assertj.core.api.Assertions.assertThat +import org.junit.Test +import org.junit.jupiter.api.assertThrows + +@Suppress("FunctionName") +class ContractWithGenericTypeTest { + companion object { + const val DATA_VALUE = 5000L + + @JvmField + val logger = loggerFor() + + @JvmField + val user = User("u", "p", setOf(Permissions.all())) + + fun parameters(): DriverParameters { + return DriverParameters( + portAllocation = incrementalPortAllocation(), + startNodesInProcess = false, + notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, validating = true)), + cordappsForAllNodes = listOf( + cordappWithPackages("net.corda.flows.serialization.generics").signed(), + cordappWithPackages("net.corda.contracts.serialization.generics").signed() + ) + ) + } + } + + @Test(timeout = 300_000) + fun `flow with value of generic type`() { + driver(parameters()) { + val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val txID = CordaRPCClient(hostAndPort = alice.rpcAddress) + .start(user.username, user.password) + .use { client -> + client.proxy.startFlow(::GenericTypeFlow, DataObject(DATA_VALUE)) + .returnValue + .getOrThrow() + } + logger.info("TX-ID=$txID") + } + } + + @Test(timeout = 300_000) + fun `flow without value of generic type`() { + driver(parameters()) { + val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val ex = assertThrows { + CordaRPCClient(hostAndPort = alice.rpcAddress) + .start(user.username, user.password) + .use { client -> + client.proxy.startFlow(::GenericTypeFlow, null) + .returnValue + .getOrThrow() + } + } + assertThat(ex).hasMessageContaining("Contract verification failed: Failed requirement: Purchase has a price,") + } + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt b/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt deleted file mode 100644 index 5854e61fdd..0000000000 --- a/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt +++ /dev/null @@ -1,114 +0,0 @@ -package net.corda.node - -import net.corda.core.utilities.getOrThrow -import net.corda.node.logging.logFile -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeParameters -import net.corda.testing.driver.driver -import net.corda.testing.driver.internal.incrementalPortAllocation -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.Test -import org.junit.Assert.assertTrue - - -class NodeConfigParsingTests { - - @Test(timeout=300_000) - fun `config is overriden by underscore variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("corda_sshd_port" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator, - cordappsForAllNodes = emptyList())) { - val hasSsh = startNode().get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assertTrue(hasSsh) - } - } - - @Test(timeout=300_000) - fun `config is overriden by case insensitive underscore variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("CORDA_sshd_port" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator, - cordappsForAllNodes = emptyList())) { - val hasSsh = startNode().get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assertTrue(hasSsh) - } - } - - @Test(timeout=300_000) - fun `config is overriden by case insensitive dot variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("CORDA.sshd.port" to sshPort.toString(), - "corda.devMode" to true.toString()), - startNodesInProcess = false, - portAllocation = portAllocator, - cordappsForAllNodes = emptyList())) { - val hasSsh = startNode(NodeParameters()).get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assertTrue(hasSsh) - } - } - - @Test(timeout=300_000) - fun `shadowing is forbidden`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf( - "CORDA_sshd_port" to sshPort.toString(), - "corda.sshd.port" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator, - notarySpecs = emptyList())) { - - assertThatThrownBy { - startNode().getOrThrow() - } - } - } - - @Test(timeout=300_000) - fun `bad keys are ignored and warned for`() { - val portAllocator = incrementalPortAllocation() - driver(DriverParameters( - environmentVariables = mapOf( - "corda_bad_key" to "2077"), - startNodesInProcess = false, - portAllocation = portAllocator, - notarySpecs = emptyList(), - cordappsForAllNodes = emptyList())) { - - val hasWarning = startNode() - .getOrThrow() - .logFile() - .readLines() - .any { - it.contains("(property or environment variable) cannot be mapped to an existing Corda") - } - assertTrue(hasWarning) - } - } -} diff --git a/node/src/integration-test/kotlin/net/corda/node/NodeRPCTests.kt b/node/src/integration-test/kotlin/net/corda/node/NodeRPCTests.kt index 5e42f01c9b..2fb9e35196 100644 --- a/node/src/integration-test/kotlin/net/corda/node/NodeRPCTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/NodeRPCTests.kt @@ -11,7 +11,7 @@ import kotlin.test.assertEquals import kotlin.test.assertTrue class NodeRPCTests { - private val CORDA_VERSION_REGEX = "\\d+(\\.\\d+)?(-\\w+)?".toRegex() + private val CORDA_VERSION_REGEX = "\\d+(\\.\\d+)?(\\.\\d+)?(-\\w+)?".toRegex() private val CORDA_VENDOR = "Corda Open Source" private val CORDAPPS = listOf(FINANCE_CONTRACTS_CORDAPP, FINANCE_WORKFLOWS_CORDAPP) private val CORDAPP_TYPES = setOf("Contract CorDapp", "Workflow CorDapp") diff --git a/node/src/integration-test/kotlin/net/corda/node/services/DeterministicContractWithGenericTypeTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/DeterministicContractWithGenericTypeTest.kt new file mode 100644 index 0000000000..c3c440eaf6 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/DeterministicContractWithGenericTypeTest.kt @@ -0,0 +1,85 @@ +package net.corda.node.services + +import net.corda.client.rpc.CordaRPCClient +import net.corda.contracts.serialization.generics.DataObject +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.loggerFor +import net.corda.flows.serialization.generics.GenericTypeFlow +import net.corda.node.DeterministicSourcesRule +import net.corda.node.internal.djvm.DeterministicVerificationException +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.driver.internal.incrementalPortAllocation +import net.corda.testing.node.NotarySpec +import net.corda.testing.node.User +import net.corda.testing.node.internal.cordappWithPackages +import org.assertj.core.api.Assertions.assertThat +import org.junit.ClassRule +import org.junit.Test +import org.junit.jupiter.api.assertThrows + +@Suppress("FunctionName") +class DeterministicContractWithGenericTypeTest { + companion object { + const val DATA_VALUE = 5000L + + @JvmField + val logger = loggerFor() + + @JvmField + val user = User("u", "p", setOf(Permissions.all())) + + @ClassRule + @JvmField + val djvmSources = DeterministicSourcesRule() + + fun parameters(): DriverParameters { + return DriverParameters( + portAllocation = incrementalPortAllocation(), + startNodesInProcess = false, + notarySpecs = listOf(NotarySpec(DUMMY_NOTARY_NAME, validating = true)), + cordappsForAllNodes = listOf( + cordappWithPackages("net.corda.flows.serialization.generics").signed(), + cordappWithPackages("net.corda.contracts.serialization.generics").signed() + ), + djvmBootstrapSource = djvmSources.bootstrap, + djvmCordaSource = djvmSources.corda + ) + } + } + + @Test(timeout = 300_000) + fun `test DJVM can deserialise command with generic type`() { + driver(parameters()) { + val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val txID = CordaRPCClient(hostAndPort = alice.rpcAddress) + .start(user.username, user.password) + .use { client -> + client.proxy.startFlow(::GenericTypeFlow, DataObject(DATA_VALUE)) + .returnValue + .getOrThrow() + } + logger.info("TX-ID=$txID") + } + } + + @Test(timeout = 300_000) + fun `test DJVM can deserialise command without value of generic type`() { + driver(parameters()) { + val alice = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val ex = assertThrows { + CordaRPCClient(hostAndPort = alice.rpcAddress) + .start(user.username, user.password) + .use { client -> + client.proxy.startFlow(::GenericTypeFlow, null) + .returnValue + .getOrThrow() + } + } + assertThat(ex).hasMessageContaining("Contract verification failed: Failed requirement: Purchase has a price,") + } + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt new file mode 100644 index 0000000000..2f704bf630 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/config/NodeConfigParsingTests.kt @@ -0,0 +1,32 @@ +package net.corda.node.services.config + +import net.corda.core.utilities.getOrThrow +import net.corda.node.logging.logFile +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.driver.internal.incrementalPortAllocation +import org.junit.Assert.assertTrue +import org.junit.Test + +class NodeConfigParsingTests { + @Test(timeout = 300_000) + fun `bad keys are ignored and warned for`() { + val portAllocator = incrementalPortAllocation() + driver(DriverParameters( + environmentVariables = mapOf( + "corda_bad_key" to "2077"), + startNodesInProcess = false, + portAllocation = portAllocator, + notarySpecs = emptyList())) { + + val hasWarning = startNode() + .getOrThrow() + .logFile() + .readLines() + .any { + it.contains("(property or environment variable) cannot be mapped to an existing Corda") + } + assertTrue(hasWarning) + } + } +} diff --git a/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt new file mode 100644 index 0000000000..ad417530d5 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/messaging/MessagingSendAllTest.kt @@ -0,0 +1,72 @@ +package net.corda.node.services.messaging + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.flows.Destination +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.internal.concurrent.transpose +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.unwrap +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import org.junit.Test +import kotlin.test.assertEquals + +class MessagingSendAllTest { + + @Test(timeout=300_000) + fun `flow can exchange messages with multiple sessions to the same party in parallel`() { + driver(DriverParameters(startNodesInProcess = true)) { + val (alice, bob) = listOf( + startNode(providedName = ALICE_NAME), + startNode(providedName = BOB_NAME) + ).transpose().getOrThrow() + + val bobIdentity = bob.nodeInfo.singleIdentity() + val messages = listOf( + bobIdentity to "hey bob 1", + bobIdentity to "hey bob 2" + ) + + alice.rpc.startFlow(::SenderFlow, messages).returnValue.getOrThrow() + } + } + + @StartableByRPC + @InitiatingFlow + class SenderFlow(private val parties: List>): FlowLogic() { + @Suspendable + override fun call(): String { + val messagesPerSession = parties.toList().map { (party, messageType) -> + val session = initiateFlow(party) + Pair(session, messageType) + }.toMap() + + sendAllMap(messagesPerSession) + val messages = receiveAll(String::class.java, messagesPerSession.keys.toList()) + + messages.map { it.unwrap { payload -> assertEquals("pong", payload) } } + + return "ok" + } + } + + @InitiatedBy(SenderFlow::class) + class RecipientFlow(private val otherPartySession: FlowSession): FlowLogic() { + @Suspendable + override fun call(): String { + otherPartySession.receive().unwrap { it } + otherPartySession.send("pong") + + return "ok" + } + } + +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt index 261532c1e3..1e52c27ed6 100644 --- a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityAsNodeTest.kt @@ -1,6 +1,7 @@ package net.corda.services.messaging import net.corda.core.crypto.Crypto +import net.corda.core.crypto.toStringShort import net.corda.core.identity.CordaX500Name import net.corda.core.internal.createDirectories import net.corda.core.internal.exists @@ -14,6 +15,9 @@ import net.corda.nodeapi.internal.crypto.CertificateType import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.loadDevCaTrustStore import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.nodeapi.internal.ArtemisMessagingComponent +import net.corda.services.messaging.SimpleAMQPClient.Companion.sendAndVerify +import net.corda.testing.core.singleIdentity import org.apache.activemq.artemis.api.config.ActiveMQDefaultConfiguration import org.apache.activemq.artemis.api.core.ActiveMQClusterSecurityException import org.apache.activemq.artemis.api.core.ActiveMQNotConnectedException @@ -24,6 +28,8 @@ import org.bouncycastle.asn1.x509.GeneralSubtree import org.bouncycastle.asn1.x509.NameConstraints import org.junit.Test import java.nio.file.Files +import javax.jms.JMSSecurityException +import kotlin.test.assertEquals /** * Runs the security tests with the attacker pretending to be a node on the network. @@ -39,7 +45,7 @@ class MQSecurityAsNodeTest : P2PMQSecurityTest() { @Test(timeout=300_000) fun `send message to RPC requests address`() { - assertSendAttackFails(RPCApi.RPC_SERVER_QUEUE_NAME) + assertProducerQueueCreationAttackFails(RPCApi.RPC_SERVER_QUEUE_NAME) } @Test(timeout=300_000) @@ -117,4 +123,53 @@ class MQSecurityAsNodeTest : P2PMQSecurityTest() { attacker.start(PEER_USER, PEER_USER) } } + + override fun `send message to notifications address`() { + assertProducerQueueCreationAttackFails(ArtemisMessagingComponent.NOTIFICATIONS_ADDRESS) + } + + @Test(timeout=300_000) + fun `send message on core protocol`() { + val attacker = clientTo(alice.node.configuration.p2pAddress) + attacker.start(PEER_USER, PEER_USER) + val message = attacker.createMessage() + assertEquals(true, attacker.producer.isBlockOnNonDurableSend) + assertThatExceptionOfType(ActiveMQSecurityException::class.java).isThrownBy { + attacker.producer.send("${ArtemisMessagingComponent.P2P_PREFIX}${alice.info.singleIdentity().owningKey.toStringShort()}", message) + }.withMessageContaining("CoreMessage").withMessageContaining("AMQPMessage") + } + + @Test(timeout = 300_000) + fun `send AMQP message with correct validated user in header`() { + val attacker = amqpClientTo(alice.node.configuration.p2pAddress) + val session = attacker.start(PEER_USER, PEER_USER) + val message = session.createMessage() + message.setStringProperty("_AMQ_VALIDATED_USER", "O=MegaCorp, L=London, C=GB") + val queue = session.createQueue("${ArtemisMessagingComponent.P2P_PREFIX}${alice.info.singleIdentity().owningKey.toStringShort()}") + val producer = session.createProducer(queue) + producer.sendAndVerify(message) + } + + @Test(timeout = 300_000) + fun `send AMQP message with incorrect validated user in header`() { + val attacker = amqpClientTo(alice.node.configuration.p2pAddress) + val session = attacker.start(PEER_USER, PEER_USER) + val message = session.createMessage() + message.setStringProperty("_AMQ_VALIDATED_USER", "O=Bob, L=New York, C=US") + val queue = session.createQueue("${ArtemisMessagingComponent.P2P_PREFIX}${alice.info.singleIdentity().owningKey.toStringShort()}") + val producer = session.createProducer(queue) + assertThatExceptionOfType(JMSSecurityException::class.java).isThrownBy { + producer.sendAndVerify(message) + }.withMessageContaining("_AMQ_VALIDATED_USER mismatch") + } + + @Test(timeout = 300_000) + fun `send AMQP message without header`() { + val attacker = amqpClientTo(alice.node.configuration.p2pAddress) + val session = attacker.start(PEER_USER, PEER_USER) + val message = session.createMessage() + val queue = session.createQueue("${ArtemisMessagingComponent.P2P_PREFIX}${alice.info.singleIdentity().owningKey.toStringShort()}") + val producer = session.createProducer(queue) + producer.sendAndVerify(message) + } } diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityTest.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityTest.kt index 2586a67ced..82c9804b8f 100644 --- a/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityTest.kt +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/MQSecurityTest.kt @@ -45,7 +45,7 @@ abstract class MQSecurityTest : NodeBasedTest() { private val rpcUser = User("user1", "pass", permissions = emptySet()) lateinit var alice: NodeWithInfo lateinit var attacker: SimpleMQClient - private val clients = ArrayList() + private val runOnStop = ArrayList<() -> Any?>() @Before override fun setUp() { @@ -62,8 +62,8 @@ abstract class MQSecurityTest : NodeBasedTest() { abstract fun startAttacker(attacker: SimpleMQClient) @After - fun stopClients() { - clients.forEach { it.stop() } + fun tearDown() { + runOnStop.forEach { it() } } @Test(timeout=300_000) @@ -79,7 +79,7 @@ abstract class MQSecurityTest : NodeBasedTest() { } @Test(timeout=300_000) - fun `send message to notifications address`() { + open fun `send message to notifications address`() { assertSendAttackFails(NOTIFICATIONS_ADDRESS) } @@ -97,18 +97,21 @@ abstract class MQSecurityTest : NodeBasedTest() { fun clientTo(target: NetworkHostAndPort, sslConfiguration: MutualSslConfiguration? = configureTestSSL(CordaX500Name("MegaCorp", "London", "GB"))): SimpleMQClient { val client = SimpleMQClient(target, sslConfiguration) - clients += client + runOnStop += client::stop + return client + } + + fun amqpClientTo(target: NetworkHostAndPort, + sslConfiguration: MutualSslConfiguration = configureTestSSL(CordaX500Name("MegaCorp", "London", "GB")) + ): SimpleAMQPClient { + val client = SimpleAMQPClient(target, sslConfiguration) + runOnStop += client::stop return client } private val rpcConnections = mutableListOf() private fun loginToRPC(target: NetworkHostAndPort, rpcUser: User): CordaRPCOps { - return CordaRPCClient(target).start(rpcUser.username, rpcUser.password).also { rpcConnections.add(it) }.proxy - } - - @After - fun closeRPCConnections() { - rpcConnections.forEach { it.forceClose() } + return CordaRPCClient(target).start(rpcUser.username, rpcUser.password).also { runOnStop += it::forceClose }.proxy } fun loginToRPCAndGetClientQueue(): String { @@ -152,7 +155,7 @@ abstract class MQSecurityTest : NodeBasedTest() { } } - fun assertSendAttackFails(address: String) { + open fun assertSendAttackFails(address: String) { val message = attacker.createMessage() assertEquals(true, attacker.producer.isBlockOnNonDurableSend) assertAttackFails(address, "SEND") { diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMQSecurityTest.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMQSecurityTest.kt index 240ad1007d..1773b03380 100644 --- a/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMQSecurityTest.kt +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMQSecurityTest.kt @@ -3,17 +3,43 @@ package net.corda.services.messaging import net.corda.core.crypto.generateKeyPair import net.corda.core.crypto.toStringShort import net.corda.nodeapi.RPCApi +import net.corda.nodeapi.internal.ArtemisMessagingComponent import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.P2P_PREFIX import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEERS_PREFIX +import net.corda.services.messaging.SimpleAMQPClient.Companion.sendAndVerify import net.corda.testing.core.BOB_NAME import net.corda.testing.core.singleIdentity +import org.assertj.core.api.Assertions.assertThatExceptionOfType import org.junit.Test +import javax.jms.JMSException /** * Runs a series of MQ-related attacks against a node. Subclasses need to call [startAttacker] to connect * the attacker to [alice]. */ abstract class P2PMQSecurityTest : MQSecurityTest() { + override fun assertSendAttackFails(address: String) { + val attacker = amqpClientTo(alice.node.configuration.p2pAddress) + val session = attacker.start(ArtemisMessagingComponent.PEER_USER, ArtemisMessagingComponent.PEER_USER) + val message = session.createMessage() + message.setStringProperty("_AMQ_VALIDATED_USER", "O=MegaCorp, L=London, C=GB") + val queue = session.createQueue(address) + assertThatExceptionOfType(JMSException::class.java).isThrownBy { + session.createProducer(queue).sendAndVerify(message) + }.withMessageContaining(address).withMessageContaining("SEND") + } + + fun assertProducerQueueCreationAttackFails(address: String) { + val attacker = amqpClientTo(alice.node.configuration.p2pAddress) + val session = attacker.start(ArtemisMessagingComponent.PEER_USER, ArtemisMessagingComponent.PEER_USER) + val message = session.createMessage() + message.setStringProperty("_AMQ_VALIDATED_USER", "O=MegaCorp, L=London, C=GB") + val queue = session.createQueue(address) + assertThatExceptionOfType(JMSException::class.java).isThrownBy { + session.createProducer(queue) + }.withMessageContaining(address).withMessageContaining("CREATE_DURABLE_QUEUE") + } + @Test(timeout=300_000) fun `consume message from P2P queue`() { assertConsumeAttackFails("$P2P_PREFIX${alice.info.singleIdentity().owningKey.toStringShort()}") diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/SimpleAMQPClient.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/SimpleAMQPClient.kt new file mode 100644 index 0000000000..bb3c86e9de --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/SimpleAMQPClient.kt @@ -0,0 +1,141 @@ +package net.corda.services.messaging + +import net.corda.core.internal.concurrent.openFuture +import net.corda.core.utilities.NetworkHostAndPort +import net.corda.nodeapi.internal.config.MutualSslConfiguration +import org.apache.qpid.jms.JmsConnectionFactory +import org.apache.qpid.jms.meta.JmsConnectionInfo +import org.apache.qpid.jms.provider.Provider +import org.apache.qpid.jms.provider.ProviderFuture +import org.apache.qpid.jms.provider.amqp.AmqpProvider +import org.apache.qpid.jms.provider.amqp.AmqpSaslAuthenticator +import org.apache.qpid.jms.sasl.PlainMechanism +import org.apache.qpid.jms.transports.TransportOptions +import org.apache.qpid.jms.transports.netty.NettyTcpTransport +import org.apache.qpid.proton.engine.Sasl +import org.apache.qpid.proton.engine.SaslListener +import org.apache.qpid.proton.engine.Transport +import java.net.URI +import java.security.SecureRandom +import java.util.concurrent.ExecutionException +import java.util.concurrent.TimeUnit +import javax.jms.CompletionListener +import javax.jms.Connection +import javax.jms.Message +import javax.jms.MessageProducer +import javax.jms.Session +import javax.net.ssl.KeyManagerFactory +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManagerFactory + +/** + * Simple AMQP client connecting to broker using JMS. + */ +class SimpleAMQPClient(private val target: NetworkHostAndPort, private val config: MutualSslConfiguration) { + companion object { + /** + * Send message and wait for completion. + * @throws Exception on failure + */ + fun MessageProducer.sendAndVerify(message: Message) { + val request = openFuture() + send(message, object : CompletionListener { + override fun onException(message: Message, exception: Exception) { + request.setException(exception) + } + + override fun onCompletion(message: Message) { + request.set(Unit) + } + }) + try { + request.get(10, TimeUnit.SECONDS) + } catch (e: ExecutionException) { + throw e.cause!! + } + } + } + + private lateinit var connection: Connection + + private fun sslContext(): SSLContext { + val keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()).apply { + init(config.keyStore.get().value.internal, config.keyStore.entryPassword.toCharArray()) + } + val trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()).apply { + init(config.trustStore.get().value.internal) + } + val sslContext = SSLContext.getInstance("TLS") + val keyManagers = keyManagerFactory.keyManagers + val trustManagers = trustManagerFactory.trustManagers + sslContext.init(keyManagers, trustManagers, SecureRandom()) + return sslContext + } + + fun start(username: String, password: String): Session { + val connectionFactory = TestJmsConnectionFactory("amqps://${target.host}:${target.port}", username, password) + connectionFactory.setSslContext(sslContext()) + connection = connectionFactory.createConnection() + connection.start() + return connection.createSession(false, Session.AUTO_ACKNOWLEDGE) + } + + fun stop() { + try { + connection.close() + } catch (e: Exception) { + // connection might not have initialised. + } + } + + private class TestJmsConnectionFactory(uri: String, private val user: String, private val pwd: String) : JmsConnectionFactory(uri) { + override fun createProvider(remoteURI: URI): Provider { + val transportOptions = TransportOptions().apply { + // Disable SNI check for server certificate + isVerifyHost = false + } + val transport = NettyTcpTransport(remoteURI, transportOptions, true) + + // Manually override SASL negotiations to accept failure in SASL-OUTCOME, which is produced by node Artemis server + return object : AmqpProvider(remoteURI, transport) { + override fun connect(connectionInfo: JmsConnectionInfo?) { + super.connect(connectionInfo) + val sasl = protonTransport.sasl() + sasl.client() + sasl.setRemoteHostname(remoteURI.host) + val authenticator = AmqpSaslAuthenticator { + PlainMechanism().apply { + username = user + password = pwd + } + } + val saslRequest = ProviderFuture() + sasl.setListener(object : SaslListener { + override fun onSaslMechanisms(sasl: Sasl, transport: Transport) { + authenticator.handleSaslMechanisms(sasl, transport) + } + + override fun onSaslChallenge(sasl: Sasl, transport: Transport) { + authenticator.handleSaslChallenge(sasl, transport) + } + + override fun onSaslOutcome(sasl: Sasl, transport: Transport) { + authenticator.handleSaslOutcome(sasl, transport) + saslRequest.onSuccess() + } + + override fun onSaslInit(sasl: Sasl, transport: Transport) { + } + + override fun onSaslResponse(sasl: Sasl, transport: Transport) { + } + }) + pumpToProtonTransport() + saslRequest.sync() + } + }.apply { + isSaslLayer = false + } + } + } +} diff --git a/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt b/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt index e8f2feeedc..d73e3583d5 100644 --- a/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt +++ b/node/src/main/kotlin/net/corda/node/NodeCmdLineOptions.kt @@ -34,7 +34,7 @@ open class SharedNodeCmdLineOptions { description = ["The path to the config file. By default this is node.conf in the base directory."] ) private var _configFile: Path? = null - val configFile: Path get() = _configFile ?: (baseDirectory / "node.conf") + val configFile: Path get() = if (_configFile != null) baseDirectory.resolve(_configFile) else (baseDirectory / "node.conf") @Option( names = ["--on-unknown-config-keys"], diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 6460a851ab..7b0c884d50 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -59,6 +59,8 @@ import net.corda.core.schemas.MappedSchema import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.toFuture import net.corda.core.transactions.LedgerTransaction import net.corda.core.utilities.NetworkHostAndPort @@ -358,6 +360,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } else { BasicVerifierFactoryService() } + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory).tokenize() val contractUpgradeService = ContractUpgradeServiceImpl(cacheFactory).tokenize() val auditService = DummyAuditService().tokenize() @Suppress("LeakingThis") @@ -699,11 +702,22 @@ abstract class AbstractNode(val configuration: NodeConfiguration, val myNotaryIdentity = configuration.notary?.let { if (it.serviceLegalName != null) { - val (notaryIdentity, notaryIdentityKeyPair) = loadNotaryClusterIdentity(it.serviceLegalName) + val (notaryIdentity, notaryIdentityKeyPair) = loadNotaryServiceIdentity(it.serviceLegalName) keyPairs += notaryIdentityKeyPair notaryIdentity } else { - // In case of a single notary service myNotaryIdentity will be the node's single identity. + // The only case where the myNotaryIdentity will be the node's legal identity is for existing single notary services running + // an older version. Current single notary services (V4.6+) sign requests using a separate notary service identity so the + // notary identity will be different from the node's legal identity. + + // This check is here to ensure that a user does not accidentally/intentionally remove the serviceLegalName configuration + // parameter after a notary has been registered. If that was possible then notary would start and sign incoming requests + // with the node's legal identity key, corrupting the data. + check (!cryptoService.containsKey(DISTRIBUTED_NOTARY_KEY_ALIAS)) { + "The notary service key exists in the key store but no notary service legal name has been configured. " + + "Either include the relevant 'notary.serviceLegalName' configuration or validate this key is not necessary " + + "and remove from the key store." + } identity } } @@ -1147,8 +1161,12 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } } - /** Loads pre-generated notary service cluster identity. */ - private fun loadNotaryClusterIdentity(serviceLegalName: CordaX500Name): Pair { + /** + * Loads notary service identity. In the case of the experimental RAFT and BFT notary clusters, this loads the pre-generated + * cluster identity that all worker nodes share. In the case of a simple single notary, this loads the notary service identity + * that is generated during initial registration and is used to sign notarisation requests. + * */ + private fun loadNotaryServiceIdentity(serviceLegalName: CordaX500Name): Pair { val privateKeyAlias = "$DISTRIBUTED_NOTARY_KEY_ALIAS" val compositeKeyAlias = "$DISTRIBUTED_NOTARY_COMPOSITE_KEY_ALIAS" @@ -1264,6 +1282,8 @@ abstract class AbstractNode(val configuration: NodeConfiguration, private lateinit var _myInfo: NodeInfo override val myInfo: NodeInfo get() = _myInfo + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache get() = this@AbstractNode.attachmentsClassLoaderCache + private lateinit var _networkParameters: NetworkParameters override val networkParameters: NetworkParameters get() = _networkParameters @@ -1463,11 +1483,12 @@ fun CordaPersistence.startHikariPool( NodeDatabaseErrors.MISSING_DRIVER) ex is OutstandingDatabaseChangesException -> throw (DatabaseIncompatibleException(ex.message)) else -> { - LoggerFactory.getLogger("CordaPersistence extension").error("Could not create the DataSource", ex) + val msg = ex.message ?: ex::class.java.canonicalName throw CouldNotCreateDataSourceException( "Could not create the DataSource: ${ex.message}", NodeDatabaseErrors.FAILED_STARTUP, - cause = ex) + cause = ex, + parameters = listOf(msg)) } } } diff --git a/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt b/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt index 571c97b82c..6d058aaf37 100644 --- a/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt +++ b/node/src/main/kotlin/net/corda/node/internal/CordaRPCOpsImpl.kt @@ -1,5 +1,6 @@ package net.corda.node.internal +import net.corda.client.rpc.RPCException import net.corda.client.rpc.notUsed import net.corda.common.logging.CordaVersion import net.corda.core.CordaRuntimeException @@ -263,7 +264,8 @@ internal class CordaRPCOpsImpl( } override fun openAttachment(id: SecureHash): InputStream { - return services.attachments.openAttachment(id)!!.open() + return services.attachments.openAttachment(id)?.open() ?: + throw RPCException("Unable to open attachment with id: $id") } override fun uploadAttachment(jar: InputStream): SecureHash { diff --git a/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt b/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt index 0bab5cb88e..ce964ab97c 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NetworkParametersReader.kt @@ -86,6 +86,7 @@ class NetworkParametersReader(private val trustRoot: X509Certificate, logger.info("No network-parameters file found. Expecting network parameters to be available from the network map.") networkMapClient ?: throw Error.NetworkMapNotConfigured() val signedParams = networkMapClient.getNetworkParameters(parametersHash) + signedParams.verifiedNetworkParametersCert(trustRoot) signedParams.serialize().open().copyTo(baseDirectory / NETWORK_PARAMS_FILE_NAME) return signedParams } diff --git a/node/src/main/kotlin/net/corda/node/internal/artemis/UserValidationPlugin.kt b/node/src/main/kotlin/net/corda/node/internal/artemis/UserValidationPlugin.kt new file mode 100644 index 0000000000..963f5169a6 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/internal/artemis/UserValidationPlugin.kt @@ -0,0 +1,47 @@ +package net.corda.node.internal.artemis + +import net.corda.core.utilities.contextLogger +import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.PEER_USER +import org.apache.activemq.artemis.api.core.ActiveMQSecurityException +import org.apache.activemq.artemis.api.core.Message +import org.apache.activemq.artemis.core.server.ServerSession +import org.apache.activemq.artemis.core.server.plugin.ActiveMQServerPlugin +import org.apache.activemq.artemis.core.transaction.Transaction +import org.apache.activemq.artemis.protocol.amqp.broker.AMQPMessage + +/** + * Plugin to verify the user in the AMQP message header against the user in the authenticated session. + * + * In core protocol, Artemis Server automatically overwrites the _AMQ_VALIDATED_USER field in message header according to authentication + * of the session. However, this is not done for AMQP protocol, which is used by Corda. Hence, _AMQ_VALIDATED_USER in AMQP packet is + * delivered in the same form, as it was produced by counterpart. To prevent manipulations of this field by other peers, we should check + * message header against user in authenticated session. + * + * Note that AMQP message is immutable, so changing the header means rebuilding the whole message, which is expensive. Instead, the + * preferred option is to throw an exception. + */ +class UserValidationPlugin : ActiveMQServerPlugin { + companion object { + private val log = contextLogger() + } + + override fun beforeSend(session: ServerSession, tx: Transaction?, message: Message, direct: Boolean, noAutoCreateQueue: Boolean) { + try { + if (session.username == PEER_USER) { + if (message !is AMQPMessage) { + throw ActiveMQSecurityException("Invalid message type: expected [${AMQPMessage::class.java.name}], got [${message.javaClass.name}]") + } + val user = message.getStringProperty(Message.HDR_VALIDATED_USER) + if (user != null && user != session.validatedUser) { + throw ActiveMQSecurityException("_AMQ_VALIDATED_USER mismatch: expected [${session.validatedUser}], got [${user}]") + } + } + } catch (e: ActiveMQSecurityException) { + throw e + } catch (e: Throwable) { + // Artemis swallows any exception except ActiveMQException + log.error("Message validation failed", e) + throw ActiveMQSecurityException("Message validation failed") + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt b/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt index d0ea54969e..881de2c7df 100644 --- a/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt +++ b/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt @@ -62,11 +62,11 @@ class InitialRegistration(val baseDirectory: Path, private val networkRootTrustS val versionInfo = startup.getVersionInfo() println("\n" + - "******************************************************************\n" + - "* *\n" + - "* Registering as a new participant with a Corda network *\n" + - "* *\n" + - "******************************************************************\n") + "*******************************************************************\n" + + "* *\n" + + "* Registering as a new participant with a Corda network *\n" + + "* *\n" + + "*******************************************************************\n") NodeRegistrationHelper(NodeRegistrationConfiguration(conf), HTTPNetworkRegistrationService( diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt index 70bb911106..0e2538e8ac 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt @@ -37,6 +37,7 @@ class MigrationNamedCacheFactory(private val metricRegistry: MetricRegistry?, "NodeAttachmentService_contractAttachmentVersions" -> caffeine.maximumSize(defaultCacheSize) "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name.") } } diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt index ec8b5d315d..0186b9659c 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt @@ -15,6 +15,8 @@ import net.corda.core.node.services.NetworkParametersService import net.corda.core.node.services.TransactionStorage import net.corda.core.serialization.deserialize import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.ContractUpgradeLedgerTransaction import net.corda.core.transactions.NotaryChangeLedgerTransaction import net.corda.core.transactions.WireTransaction @@ -62,6 +64,8 @@ class MigrationServicesForResolution( cacheFactory ) + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) + private fun defaultNetworkParameters(): NetworkParameters { logger.warn("Using a dummy set of network parameters for migration.") val clock = Clock.systemUTC() @@ -124,7 +128,8 @@ class MigrationServicesForResolution( networkParameters, tx.id, attachmentTrustCalculator::calculate, - cordappLoader.appClassLoader) { + cordappLoader.appClassLoader, + attachmentsClassLoaderCache) { deserialiseComponentGroup(tx.componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true) } states.filterIndexed {index, _ -> stateIndices.contains(index)}.toList() diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index 0bac15c171..1edc2491a8 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -65,4 +65,6 @@ interface CheckpointStorage { * This method does not fetch [Checkpoint.Serialized.serializedFlowState] to save memory. */ fun getPausedCheckpoints(): Stream> + + fun updateStatus(runId: StateMachineRunId, flowStatus: Checkpoint.FlowStatus) } diff --git a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt index 8a5eec792f..d4e0239537 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt @@ -6,6 +6,7 @@ import com.typesafe.config.ConfigParseOptions import net.corda.cliutils.CordaSystemUtils import net.corda.common.configuration.parsing.internal.Configuration import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.createDirectories import net.corda.core.internal.div import net.corda.core.internal.exists @@ -36,13 +37,36 @@ object ConfigHelper { private const val UPPERCASE_PROPERTY_PREFIX = "CORDA." private val log = LoggerFactory.getLogger(javaClass) + + val DEFAULT_CONFIG_FILENAME = "node.conf" + @Suppress("LongParameterList") fun loadConfig(baseDirectory: Path, - configFile: Path = baseDirectory / "node.conf", + configFile: Path = baseDirectory / DEFAULT_CONFIG_FILENAME, allowMissingConfig: Boolean = false, - configOverrides: Config = ConfigFactory.empty()): Config { + configOverrides: Config = ConfigFactory.empty()): Config + = loadConfig(baseDirectory, + configFile = configFile, + allowMissingConfig = allowMissingConfig, + configOverrides = configOverrides, + rawSystemOverrides = ConfigFactory.systemProperties(), + rawEnvironmentOverrides = ConfigFactory.systemEnvironment()) + + /** + * Internal equivalent of [loadConfig] which allows the system and environment + * overrides to be provided from a test. + */ + @Suppress("LongParameterList") + @VisibleForTesting + internal fun loadConfig(baseDirectory: Path, + configFile: Path, + allowMissingConfig: Boolean, + configOverrides: Config, + rawSystemOverrides: Config, + rawEnvironmentOverrides: Config + ): Config { val parseOptions = ConfigParseOptions.defaults() - val defaultConfig = ConfigFactory.parseResources("reference.conf", parseOptions.setAllowMissing(false)) + val defaultConfig = ConfigFactory.parseResources("corda-reference.conf", parseOptions.setAllowMissing(false)) val appConfig = ConfigFactory.parseFile(configFile.toFile(), parseOptions.setAllowMissing(allowMissingConfig)) // Detect the underlying OS. If mac or windows non-server then we assume we're running in devMode. Unless specified otherwise. @@ -55,8 +79,8 @@ object ConfigHelper { "flowExternalOperationThreadPoolSize" to min(coreCount, FLOW_EXTERNAL_OPERATION_THREAD_POOL_SIZE_MAX).toString() ) - val systemOverrides = ConfigFactory.systemProperties().cordaEntriesOnly() - val environmentOverrides = ConfigFactory.systemEnvironment().cordaEntriesOnly() + val systemOverrides = rawSystemOverrides.cordaEntriesOnly() + val environmentOverrides = rawEnvironmentOverrides.cordaEntriesOnly() val finalConfig = configOf( // Add substitution values here "baseDirectory" to baseDirectory.toString()) @@ -91,8 +115,8 @@ object ConfigHelper { .mapKeys { val original = it.key as String - // Reject environment variable that are in all caps - // since these cannot be properties. + // Reject environment variable that are in all caps + // since these cannot be properties. if (original == original.toUpperCase()){ return@mapKeys original } diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt index f2dc3f16cb..a12989e169 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt @@ -151,7 +151,7 @@ fun NodeConfiguration.shouldInitCrashShell() = shouldStartLocalShell() || should data class NotaryConfig( /** Specifies whether the notary validates transactions or not. */ val validating: Boolean, - /** The legal name of cluster in case of a distributed notary service. */ + /** The legal name of the notary service identity. */ val serviceLegalName: CordaX500Name? = null, /** The name of the notary service class to load. */ val className: String? = null, diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt index ec2aafb1d8..277d51742b 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt @@ -149,6 +149,8 @@ class ArtemisMessagingServer(private val config: NodeConfiguration, isJMXManagementEnabled = true isJMXUseBrokerName = true } + // Validate user in AMQP message header against authenticated session + registerBrokerPlugin(UserValidationPlugin()) }.configureAddressSecurity() diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt b/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt index eead9f5698..0734c958e1 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/MessagingExecutor.kt @@ -54,8 +54,8 @@ class MessagingExecutor( } @Synchronized - fun sendAll(messages: Map) { - messages.forEach { recipients, message -> send(message, recipients) } + fun sendAll(messages: List>) { + messages.forEach { (recipients, message) -> send(message, recipients) } } @Synchronized diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index 341996a980..38cbf1d833 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -50,8 +50,8 @@ class DBCheckpointStorage( private const val HMAC_SIZE_BYTES = 16 @VisibleForTesting - const val MAX_STACKTRACE_LENGTH = 4000 - private const val MAX_EXC_MSG_LENGTH = 4000 + const val MAX_STACKTRACE_LENGTH = 2000 + private const val MAX_EXC_MSG_LENGTH = 2000 private const val MAX_EXC_TYPE_LENGTH = 256 private const val MAX_FLOW_NAME_LENGTH = 128 private const val MAX_PROGRESS_STEP_LENGTH = 256 @@ -461,6 +461,7 @@ class DBCheckpointStorage( return session.createQuery(delete).executeUpdate() } + @Throws(SQLException::class) override fun getCheckpoint(id: StateMachineRunId): Checkpoint.Serialized? { return getDBCheckpoint(id)?.toSerializedCheckpoint() } @@ -498,6 +499,11 @@ class DBCheckpointStorage( } } + override fun updateStatus(runId: StateMachineRunId, flowStatus: FlowStatus) { + val update = "Update ${NODE_DATABASE_PREFIX}checkpoints set status = ${flowStatus.ordinal} where flow_id = '${runId.uuid}'" + currentDBSession().createNativeQuery(update).executeUpdate() + } + private fun createDBFlowMetadata(flowId: String, checkpoint: Checkpoint): DBFlowMetadata { val context = checkpoint.checkpointState.invocationContext val flowInfo = checkpoint.checkpointState.subFlowStack.first() diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt index 51aadc69cf..6b17fe0870 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/Action.kt @@ -17,7 +17,7 @@ sealed class Action { /** * Track a transaction hash and notify the state machine once the corresponding transaction has committed. */ - data class TrackTransaction(val hash: SecureHash) : Action() + data class TrackTransaction(val hash: SecureHash, val currentState: StateMachineState) : Action() /** * Send an initial session message to [destination]. @@ -140,7 +140,11 @@ sealed class Action { /** * Execute the specified [operation]. */ - data class ExecuteAsyncOperation(val deduplicationId: String, val operation: FlowAsyncOperation<*>) : Action() + data class ExecuteAsyncOperation( + val deduplicationId: String, + val operation: FlowAsyncOperation<*>, + val currentState: StateMachineState + ) : Action() /** * Release soft locks associated with given ID (currently the flow ID). diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutor.kt index 7c2bd77fd8..8e4fb07582 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutor.kt @@ -1,6 +1,7 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable +import java.sql.SQLException /** * An executor of a single [Action]. @@ -10,5 +11,6 @@ interface ActionExecutor { * Execute [action] by [fiber]. */ @Suspendable + @Throws(SQLException::class) fun executeAction(fiber: FlowFiber, action: Action) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index c3ddadd716..435ae5d6f3 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -3,7 +3,6 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable import com.codahale.metrics.Gauge import com.codahale.metrics.Reservoir -import net.corda.core.internal.concurrent.thenMatch import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.internal.checkpointSerialize @@ -14,17 +13,18 @@ import net.corda.node.services.api.ServiceHubInternal import net.corda.nodeapi.internal.persistence.contextDatabase import net.corda.nodeapi.internal.persistence.contextTransaction import net.corda.nodeapi.internal.persistence.contextTransactionOrNull -import java.time.Duration +import java.sql.SQLException /** * This is the bottom execution engine of flow side-effects. */ -class ActionExecutorImpl( - private val services: ServiceHubInternal, - private val checkpointStorage: CheckpointStorage, - private val flowMessaging: FlowMessaging, - private val stateMachineManager: StateMachineManagerInternal, - private val checkpointSerializationContext: CheckpointSerializationContext +internal class ActionExecutorImpl( + private val services: ServiceHubInternal, + private val checkpointStorage: CheckpointStorage, + private val flowMessaging: FlowMessaging, + private val stateMachineManager: StateMachineManagerInternal, + private val actionFutureExecutor: ActionFutureExecutor, + private val checkpointSerializationContext: CheckpointSerializationContext ) : ActionExecutor { private companion object { @@ -75,14 +75,7 @@ class ActionExecutorImpl( @Suspendable private fun executeTrackTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { - services.validatedTransactions.trackTransactionWithNoWarning(action.hash).thenMatch( - success = { transaction -> - fiber.scheduleEvent(Event.TransactionCommitted(transaction)) - }, - failure = { exception -> - fiber.scheduleEvent(Event.Error(exception)) - } - ) + actionFutureExecutor.awaitTransaction(fiber, action) } @Suspendable @@ -156,13 +149,8 @@ class ActionExecutorImpl( fiber.scheduleEvent(action.event) } - @Suspendable private fun executeSleepUntil(fiber: FlowFiber, action: Action.SleepUntil) { - stateMachineManager.scheduleFlowSleep( - fiber, - action.currentState, - Duration.between(services.clock.instant(), action.time) - ) + actionFutureExecutor.sleep(fiber, action) } @Suspendable @@ -208,6 +196,7 @@ class ActionExecutorImpl( } @Suspendable + @Throws(SQLException::class) private fun executeCreateTransaction() { if (contextTransactionOrNull != null) { throw IllegalStateException("Refusing to create a second transaction") @@ -224,6 +213,7 @@ class ActionExecutorImpl( } @Suspendable + @Throws(SQLException::class) private fun executeCommitTransaction() { try { contextTransaction.commit() @@ -233,19 +223,11 @@ class ActionExecutorImpl( } } - @Suppress("TooGenericExceptionCaught") // this is fully intentional here, see comment in the catch clause + @Suppress("TooGenericExceptionCaught") @Suspendable private fun executeAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { try { - val operationFuture = action.operation.execute(action.deduplicationId) - operationFuture.thenMatch( - success = { result -> - fiber.scheduleEvent(Event.AsyncOperationCompletion(result)) - }, - failure = { exception -> - fiber.scheduleEvent(Event.AsyncOperationThrows(exception)) - } - ) + actionFutureExecutor.awaitAsyncOperation(fiber, action) } catch (e: Exception) { // Catch and wrap any unexpected exceptions from the async operation // Wrapping the exception allows it to be better handled by the flow hospital diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt new file mode 100644 index 0000000000..40ee343707 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionFutureExecutor.kt @@ -0,0 +1,99 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.core.internal.concurrent.thenMatch +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.node.services.api.ServiceHubInternal +import java.time.Duration +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit + +internal class ActionFutureExecutor( + private val innerState: StateMachineInnerState, + private val services: ServiceHubInternal, + private val scheduledExecutor: ScheduledExecutorService +) { + + private companion object { + val log = contextLogger() + } + + /** + * Put a flow to sleep for the duration specified in [action]. + * + * @param fiber The [FlowFiber] that will be woken up after sleeping + * @param action The [Action.SleepUntil] to create a future from + */ + fun sleep(fiber: FlowFiber, action: Action.SleepUntil) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + val duration = Duration.between(services.clock.instant(), action.time) + log.debug { "Putting flow ${instance.runId} to sleep for $duration" } + val future = scheduledExecutor.schedule( + { + log.debug { "Scheduling flow wake up event for flow ${instance.runId}" } + scheduleWakeUpEvent(instance, Event.WakeUpFromSleep) + }, + duration.toMillis(), TimeUnit.MILLISECONDS + ) + action.currentState.future = future + } + + /** + * Suspend a flow until its async operation specified in [action] is completed. + * + * @param fiber The [FlowFiber] to resume after completing the async operation + * @param action The [Action.ExecuteAsyncOperation] to create a future from + */ + @Suspendable + fun awaitAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + log.debug { "Suspending flow ${instance.runId} until its async operation has completed" } + val future = action.operation.execute(action.deduplicationId) + future.thenMatch( + success = { result -> scheduleWakeUpEvent(instance, Event.AsyncOperationCompletion(result)) }, + failure = { exception -> scheduleWakeUpEvent(instance, Event.AsyncOperationThrows(exception)) } + ) + action.currentState.future = future + } + + /** + * Suspend a flow until the transaction specified in [action] is committed. + * + * @param fiber The [FlowFiber] to resume after the committing the specified transaction + * @param action [Action.TrackTransaction] contains the transaction hash to wait for + */ + @Suspendable + fun awaitTransaction(fiber: FlowFiber, action: Action.TrackTransaction) { + cancelFutureIfRunning(fiber, action.currentState) + val instance = fiber.instanceId + log.debug { "Suspending flow ${instance.runId} until transaction ${action.hash} is committed" } + val future = services.validatedTransactions.trackTransactionWithNoWarning(action.hash) + future.thenMatch( + success = { transaction -> scheduleWakeUpEvent(instance, Event.TransactionCommitted(transaction)) }, + failure = { exception -> scheduleWakeUpEvent(instance, Event.Error(exception)) } + ) + action.currentState.future = future + } + + private fun cancelFutureIfRunning(fiber: FlowFiber, currentState: StateMachineState) { + // No other future should be running, cancel it if there is + currentState.future?.run { + log.debug { "Cancelling existing future for flow ${fiber.id}" } + if (!isDone) cancel(true) + } + } + + private fun scheduleWakeUpEvent(instance: StateMachineInstanceId, event: Event) { + innerState.withLock { + flows[instance.runId]?.let { flow -> + // Only schedule a wake up event if the fiber the flow is executing on has not changed + if (flow.fiber.instanceId == instance) { + flow.fiber.scheduleEvent(event) + } + } + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt index fc80c17dfb..c9dc734460 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/Event.kt @@ -41,7 +41,7 @@ sealed class Event { * Signal that an error has happened. This may be due to an uncaught exception in the flow or some external error. * @param exception the exception itself. */ - data class Error(val exception: Throwable) : Event() + data class Error(val exception: Throwable, val rollback: Boolean = true) : Event() /** * Signal that a ledger transaction has committed. This is an event completing a [FlowIORequest.WaitForLedgerCommit] diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt new file mode 100644 index 0000000000..0dc5f28791 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowDefaultUncaughtExceptionHandler.kt @@ -0,0 +1,67 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.strands.Strand +import net.corda.core.flows.StateMachineRunId +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.node.services.api.CheckpointStorage +import net.corda.node.utilities.errorAndTerminate +import net.corda.nodeapi.internal.persistence.CordaPersistence +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit + +class FlowDefaultUncaughtExceptionHandler( + private val flowHospital: StaffedFlowHospital, + private val checkpointStorage: CheckpointStorage, + private val database: CordaPersistence, + private val scheduledExecutor: ScheduledExecutorService +) : Strand.UncaughtExceptionHandler { + + private companion object { + val log = contextLogger() + const val RESCHEDULE_DELAY = 30L + } + + override fun uncaughtException(fiber: Strand, throwable: Throwable) { + val id = (fiber as FlowStateMachineImpl<*>).id + if (throwable is VirtualMachineError) { + errorAndTerminate( + "Caught unrecoverable error from flow $id. Forcibly terminating the JVM, this might leave resources open, and most likely will.", + throwable + ) + } else { + fiber.logger.warn("Caught exception from flow $id", throwable) + setFlowToHospitalized(fiber, throwable) + } + } + + private fun setFlowToHospitalized(fiber: FlowStateMachineImpl<*>, throwable: Throwable) { + val id = fiber.id + if (!fiber.resultFuture.isDone) { + fiber.transientState.let { state -> + if (state != null) { + fiber.logger.warn("Forcing flow $id into overnight observation") + flowHospital.forceIntoOvernightObservation(state.value, listOf(throwable)) + val hospitalizedCheckpoint = state.value.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) + val hospitalizedState = state.value.copy(checkpoint = hospitalizedCheckpoint) + fiber.transientState = TransientReference(hospitalizedState) + } else { + fiber.logger.warn("The fiber's transient state is not set, cannot force flow $id into in-memory overnight observation, status will still be updated in database") + } + } + scheduledExecutor.schedule({ setFlowToHospitalizedRescheduleOnFailure(id) }, 0, TimeUnit.SECONDS) + } + } + + @Suppress("TooGenericExceptionCaught") + private fun setFlowToHospitalizedRescheduleOnFailure(id: StateMachineRunId) { + try { + log.debug { "Updating the status of flow $id to hospitalized after uncaught exception" } + database.transaction { checkpointStorage.updateStatus(id, Checkpoint.FlowStatus.HOSPITALIZED) } + log.debug { "Updated the status of flow $id to hospitalized after uncaught exception" } + } catch (e: Exception) { + log.info("Failed to update the status of flow $id to hospitalized after uncaught exception, rescheduling", e) + scheduledExecutor.schedule({ setFlowToHospitalizedRescheduleOnFailure(id) }, RESCHEDULE_DELAY, TimeUnit.SECONDS) + } + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt deleted file mode 100644 index 4ff9df43c9..0000000000 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowSleepScheduler.kt +++ /dev/null @@ -1,78 +0,0 @@ -package net.corda.node.services.statemachine - -import net.corda.core.internal.FlowIORequest -import net.corda.core.utilities.contextLogger -import net.corda.core.utilities.debug -import java.time.Duration -import java.util.concurrent.Future -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit - -internal class FlowSleepScheduler(private val smm: StateMachineManagerInternal, private val scheduledExecutor: ScheduledExecutorService) { - - private companion object { - val log = contextLogger() - } - - /** - * Put a flow to sleep for a specified duration. - * - * @param fiber The [FlowFiber] that will be woken up after sleeping - * @param currentState The current [StateMachineState] - * @param duration How long to sleep for - */ - fun sleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) { - // No other future should be running, cancel it if there is - currentState.future?.run { - log.debug { "Cancelling the existing future for flow ${fiber.id}" } - cancelIfRunning() - } - currentState.future = setAlarmClock(fiber, duration) - } - - /** - * Schedule a wake up event. - * - * @param fiber The [FlowFiber] to schedule a wake up event for - */ - fun scheduleWakeUp(fiber: FlowFiber) { - fiber.scheduleEvent(Event.WakeUpFromSleep) - } - - /** - * Cancel a sleeping flow's future. Note, this does not cause the flow to wake up. - * - * @param currentState The current [StateMachineState] - */ - fun cancel(currentState: StateMachineState) { - (currentState.checkpoint.flowState as? FlowState.Started)?.let { flowState -> - if (currentState.isWaitingForFuture && flowState.flowIORequest is FlowIORequest.Sleep) { - (currentState.future as? ScheduledFuture)?.run { - log.debug { "Cancelling the sleep scheduled future for flow ${currentState.flowLogic.runId}" } - cancelIfRunning() - currentState.future = null - } - } - - } - } - - private fun Future<*>.cancelIfRunning() { - if (!isDone) cancel(true) - } - - private fun setAlarmClock(fiber: FlowFiber, duration: Duration): ScheduledFuture { - val instance = fiber.instanceId - log.debug { "Putting flow to sleep for $duration" } - return scheduledExecutor.schedule( - { - log.debug { "Scheduling flow wake up event for flow ${instance.runId}" } - // This passes back into the SMM to check that the fiber that went to sleep is the same fiber that is now being scheduled - // with the wake up event - smm.scheduleFlowWakeUp(instance) - }, - duration.toMillis(), TimeUnit.MILLISECONDS - ) - } -} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index 408d8a12b5..5277d89638 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -284,12 +284,15 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, openThreadLocalWormhole() setLoggingContext() - initialiseFlow() logger.debug { "Calling flow: $logic" } val startTime = System.nanoTime() + var initialised = false val resultOrError = try { + initialiseFlow() + initialised = true + // This sets the Cordapp classloader on the contextClassLoader of the current thread. // Needed because in previous versions of the finance app we used Thread.contextClassLoader to resolve services defined in cordapps. Thread.currentThread().contextClassLoader = (serviceHub.cordappProvider as CordappProviderImpl).cordappLoader.appClassLoader @@ -310,14 +313,14 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, Event.FlowFinish(resultOrError.value, softLocksId) } is Try.Failure -> { - Event.Error(resultOrError.exception) + Event.Error(resultOrError.exception, initialised) } } // Immediately process the last event. This is to make sure the transition can assume that it has an open // database transaction. val continuation = processEventImmediately( finalEvent, - isDbTransactionOpenOnEntry = true, + isDbTransactionOpenOnEntry = initialised, isDbTransactionOpenOnExit = false ) if (continuation == FlowContinuation.ProcessEvents) { @@ -335,8 +338,8 @@ class FlowStateMachineImpl(override val id: StateMachineRunId, @Suspendable private fun initialiseFlow() { processEventsUntilFlowIsResumed( - isDbTransactionOpenOnEntry = false, - isDbTransactionOpenOnExit = true + isDbTransactionOpenOnEntry = false, + isDbTransactionOpenOnExit = true ) } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowTimeoutScheduler.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowTimeoutScheduler.kt new file mode 100644 index 0000000000..5ba6d700e3 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowTimeoutScheduler.kt @@ -0,0 +1,101 @@ +package net.corda.node.services.statemachine + +import net.corda.core.flows.StateMachineRunId +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.node.services.api.ServiceHubInternal +import java.util.concurrent.Future +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.TimeUnit + +internal class FlowTimeoutScheduler( + private val innerState: StateMachineInnerState, + private val scheduledExecutor: ScheduledExecutorService, + private val serviceHub: ServiceHubInternal +) { + + private companion object { + val log = contextLogger() + } + + /** + * Schedules the flow [flowId] to be retried if it does not finish within the timeout period + * specified in the config. + * + * @param flowId The id of the flow that the timeout is scheduled for + */ + fun timeout(flowId: StateMachineRunId) { + timeout(flowId) { flow, retryCount -> + val scheduledFuture = scheduleTimeoutException(flow, calculateDefaultTimeoutSeconds(retryCount)) + ScheduledTimeout(scheduledFuture, retryCount + 1) + } + } + + /** + * Cancel a flow's timeout future. + * + * @param flowId The flow's id + */ + fun cancel(flowId: StateMachineRunId) { + innerState.withLock { + timedFlows[flowId]?.let { (future, _) -> + future.cancelIfRunning() + timedFlows.remove(flowId) + } + } + } + + /** + * Resets a flow's timeout with the input timeout duration, only if it is longer than the default flow timeout configuration. + * + * @param flowId The flow's id + * @param timeoutSeconds The custom timeout + */ + fun resetCustomTimeout(flowId: StateMachineRunId, timeoutSeconds: Long) { + if (timeoutSeconds < serviceHub.configuration.flowTimeout.timeout.seconds) { + log.debug { "Ignoring request to set time-out on timed flow $flowId to $timeoutSeconds seconds which is shorter than default of ${serviceHub.configuration.flowTimeout.timeout.seconds} seconds." } + return + } + log.debug { "Processing request to set time-out on timed flow $flowId to $timeoutSeconds seconds." } + timeout(flowId) { flow, retryCount -> + val scheduledFuture = scheduleTimeoutException(flow, timeoutSeconds) + ScheduledTimeout(scheduledFuture, retryCount) + } + } + + private inline fun timeout(flowId: StateMachineRunId, timeout: (flow: Flow<*>, retryCount: Int) -> ScheduledTimeout) { + innerState.withLock { + val flow = flows[flowId] + if (flow != null) { + val retryCount = timedFlows[flowId]?.let { (future, retryCount) -> + future.cancelIfRunning() + retryCount + } ?: 0 + timedFlows[flowId] = timeout(flow, retryCount) + } else { + log.warn("Unable to schedule timeout for flow $flowId – flow not found.") + } + } + } + + /** Schedules a [FlowTimeoutException] to be fired in order to restart the flow. */ + private fun scheduleTimeoutException(flow: Flow<*>, delay: Long): ScheduledFuture<*> { + return scheduledExecutor.schedule({ + val event = Event.Error(FlowTimeoutException()) + flow.fiber.scheduleEvent(event) + }, delay, TimeUnit.SECONDS) + } + + private fun calculateDefaultTimeoutSeconds(retryCount: Int): Long { + return serviceHub.configuration.flowTimeout.run { + val timeoutDelaySeconds = + timeout.seconds * Math.pow(backoffBase, Integer.min(retryCount, maxRestartCount).toDouble()).toLong() + maxOf(1L, ((1.0 + Math.random()) * timeoutDelaySeconds / 2).toLong()) + } + } + + private fun Future<*>.cancelIfRunning() { + if (!isDone) cancel(true) + } +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index 8145df136c..1d07a75d02 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -13,10 +13,8 @@ import net.corda.core.flows.FlowLogic import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.Party import net.corda.core.internal.FlowStateMachine -import net.corda.core.internal.ThreadBox import net.corda.core.internal.bufferUntilSubscribed import net.corda.core.internal.castIfPossible -import net.corda.core.internal.concurrent.OpenFuture import net.corda.core.internal.concurrent.map import net.corda.core.internal.concurrent.mapError import net.corda.core.internal.concurrent.openFuture @@ -40,7 +38,6 @@ import net.corda.node.services.statemachine.interceptors.FiberDeserializationChe import net.corda.node.services.statemachine.interceptors.HospitalisingInterceptor import net.corda.node.services.statemachine.interceptors.PrintingInterceptor import net.corda.node.utilities.AffinityExecutor -import net.corda.node.utilities.errorAndTerminate import net.corda.node.utilities.injectOldProgressTracker import net.corda.node.utilities.isEnabledTimedFlow import net.corda.nodeapi.internal.persistence.CordaPersistence @@ -49,17 +46,12 @@ import net.corda.serialization.internal.CheckpointSerializeAsTokenContextImpl import net.corda.serialization.internal.withTokenContext import org.apache.activemq.artemis.utils.ReusableLatch import rx.Observable -import rx.subjects.PublishSubject -import java.lang.Integer.min import java.security.SecureRandom -import java.time.Duration +import java.util.ArrayList import java.util.HashSet -import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import java.util.concurrent.ScheduledFuture -import java.util.concurrent.TimeUnit import javax.annotation.concurrent.ThreadSafe import kotlin.collections.component1 import kotlin.collections.component2 @@ -71,7 +63,7 @@ import kotlin.streams.toList * thread actually starts them via [deliverExternalEvent]. */ @ThreadSafe -class SingleThreadedStateMachineManager( +internal class SingleThreadedStateMachineManager( val serviceHub: ServiceHubInternal, private val checkpointStorage: CheckpointStorage, val executor: ExecutorService, @@ -84,38 +76,19 @@ class SingleThreadedStateMachineManager( private val logger = contextLogger() } - private data class ScheduledTimeout( - /** Will fire a [FlowTimeoutException] indicating to the flow hospital to restart the flow. */ - val scheduledFuture: ScheduledFuture<*>, - /** Specifies the number of times this flow has been retried. */ - val retryCount: Int = 0 - ) - - // A list of all the state machines being managed by this class. We expose snapshots of it via the stateMachines - // property. - private class InnerState { - val changesPublisher = PublishSubject.create()!! - /** True if we're shutting down, so don't resume anything. */ - var stopping = false - val flows = HashMap>() - val pausedFlows = HashMap() - val startedFutures = HashMap>() - /** Flows scheduled to be retried if not finished within the specified timeout period. */ - val timedFlows = HashMap() - } - - private val mutex = ThreadBox(InnerState()) + private val innerState = StateMachineInnerStateImpl() private val scheduler = FiberExecutorScheduler("Same thread scheduler", executor) private val scheduledFutureExecutor = Executors.newSingleThreadScheduledExecutor( ThreadFactoryBuilder().setNameFormat("flow-scheduled-future-thread").setDaemon(true).build() ) - // How many Fibers are running and not suspended. If zero and stopping is true, then we are halted. + // How many Fibers are running (this includes suspended flows). If zero and stopping is true, then we are halted. private val liveFibers = ReusableLatch() // Monitoring support. private val metrics = serviceHub.monitoringService.metrics private val sessionToFlow = ConcurrentHashMap() private val flowMessaging: FlowMessaging = FlowMessagingImpl(serviceHub) - private val flowSleepScheduler = FlowSleepScheduler(this, scheduledFutureExecutor) + private val actionFutureExecutor = ActionFutureExecutor(innerState, serviceHub, scheduledFutureExecutor) + private val flowTimeoutScheduler = FlowTimeoutScheduler(innerState, scheduledFutureExecutor, serviceHub) private val fiberDeserializationChecker = if (serviceHub.configuration.shouldCheckCheckpoints()) FiberDeserializationChecker() else null private val ourSenderUUID = serviceHub.networkService.ourSenderUUID @@ -126,7 +99,7 @@ class SingleThreadedStateMachineManager( private val transitionExecutor = makeTransitionExecutor() override val allStateMachines: List> - get() = mutex.locked { flows.values.map { it.fiber.logic } } + get() = innerState.withLock { flows.values.map { it.fiber.logic } } private val totalStartedFlows = metrics.counter("Flows.Started") private val totalFinishedFlows = metrics.counter("Flows.Finished") @@ -137,7 +110,7 @@ class SingleThreadedStateMachineManager( * * We use assignment here so that multiple subscribers share the same wrapped Observable. */ - override val changes: Observable = mutex.content.changesPublisher + override val changes: Observable = innerState.changesPublisher override fun start(tokenizableServices: List, startMode: StateMachineManager.StartMode): CordaFuture { checkQuasarJavaAgentPresence() @@ -157,29 +130,25 @@ class SingleThreadedStateMachineManager( StateMachineManager.StartMode.Safe -> markAllFlowsAsPaused() } this.flowCreator = FlowCreator( - checkpointSerializationContext, - checkpointStorage, - scheduler, - database, - transitionExecutor, - actionExecutor, - secureRandom, - serviceHub, - unfinishedFibers, - ::resetCustomTimeout) + checkpointSerializationContext, + checkpointStorage, + scheduler, + database, + transitionExecutor, + actionExecutor, + secureRandom, + serviceHub, + unfinishedFibers, + flowTimeoutScheduler::resetCustomTimeout + ) val fibers = restoreFlowsFromCheckpoints() - metrics.register("Flows.InFlight", Gauge { mutex.content.flows.size }) - Fiber.setDefaultUncaughtExceptionHandler { fiber, throwable -> - if (throwable is VirtualMachineError) { - errorAndTerminate("Caught unrecoverable error from flow. Forcibly terminating the JVM, this might leave resources open, and most likely will.", throwable) - } else { - (fiber as FlowStateMachineImpl<*>).logger.warn("Caught exception from flow", throwable) - } - } + metrics.register("Flows.InFlight", Gauge { innerState.flows.size }) + + setFlowDefaultUncaughtExceptionHandler() val pausedFlows = restoreNonResidentFlowsFromPausedCheckpoints() - mutex.locked { + innerState.withLock { this.pausedFlows.putAll(pausedFlows) for ((id, flow) in pausedFlows) { val checkpoint = flow.checkpoint @@ -199,10 +168,21 @@ class SingleThreadedStateMachineManager( } } - override fun snapshot(): Set> = mutex.content.flows.values.map { it.fiber }.toSet() + private fun setFlowDefaultUncaughtExceptionHandler() { + Fiber.setDefaultUncaughtExceptionHandler( + FlowDefaultUncaughtExceptionHandler( + flowHospital, + checkpointStorage, + database, + scheduledFutureExecutor + ) + ) + } + + override fun snapshot(): Set> = innerState.flows.values.map { it.fiber }.toSet() override fun > findStateMachines(flowClass: Class): List>> { - return mutex.locked { + return innerState.withLock { flows.values.mapNotNull { flowClass.castIfPossible(it.fiber.logic)?.let { it to it.stateMachine.resultFuture } } @@ -217,7 +197,7 @@ class SingleThreadedStateMachineManager( */ override fun stop(allowedUnsuspendedFiberCount: Int) { require(allowedUnsuspendedFiberCount >= 0){"allowedUnsuspendedFiberCount must be greater than or equal to zero"} - mutex.locked { + innerState.withLock { if (stopping) throw IllegalStateException("Already stopping!") stopping = true for ((_, flow) in flows) { @@ -241,7 +221,7 @@ class SingleThreadedStateMachineManager( * calls to [allStateMachines] */ override fun track(): DataFeed>, StateMachineManager.Change> { - return mutex.locked { + return innerState.withMutex { database.transaction { DataFeed(flows.values.map { it.fiber.logic }, changesPublisher.bufferUntilSubscribed().wrapWithDatabaseTransaction(database)) } @@ -266,7 +246,7 @@ class SingleThreadedStateMachineManager( } override fun killFlow(id: StateMachineRunId): Boolean { - val killFlowResult = mutex.locked { + val killFlowResult = innerState.withLock { val flow = flows[id] if (flow != null) { logger.info("Killing flow $id known to this node.") @@ -281,7 +261,7 @@ class SingleThreadedStateMachineManager( unfinishedFibers.countDown() val state = flow.fiber.transientState - return@locked if (state != null) { + return@withLock if (state != null) { state.value.isKilled = true flow.fiber.scheduleEvent(Event.DoRemainingWork) true @@ -333,9 +313,9 @@ class SingleThreadedStateMachineManager( } override fun removeFlow(flowId: StateMachineRunId, removalReason: FlowRemovalReason, lastState: StateMachineState) { - mutex.locked { - cancelTimeoutIfScheduled(flowId) - cancelFlowSleep(lastState) + innerState.withLock { + flowTimeoutScheduler.cancel(flowId) + lastState.cancelFutureIfRunning() val flow = flows.remove(flowId) if (flow != null) { decrementLiveFibers() @@ -352,7 +332,7 @@ class SingleThreadedStateMachineManager( } override fun signalFlowHasStarted(flowId: StateMachineRunId) { - mutex.locked { + innerState.withLock { startedFutures.remove(flowId)?.set(Unit) flows[flowId]?.let { flow -> changesPublisher.onNext(StateMachineManager.Change.Add(flow.fiber.logic)) @@ -378,7 +358,7 @@ class SingleThreadedStateMachineManager( return checkpointStorage.getCheckpointsToRun().use { it.mapNotNull { (id, serializedCheckpoint) -> // If a flow is added before start() then don't attempt to restore it - mutex.locked { if (id in flows) return@mapNotNull null } + innerState.withLock { if (id in flows) return@mapNotNull null } val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, id) ?: return@mapNotNull null flowCreator.createFlowFromCheckpoint(id, checkpoint) }.toList() @@ -403,71 +383,59 @@ class SingleThreadedStateMachineManager( @Suppress("TooGenericExceptionCaught", "ComplexMethod", "MaxLineLength") // this is fully intentional here, see comment in the catch clause override fun retryFlowFromSafePoint(currentState: StateMachineState) { - cancelFlowSleep(currentState) + currentState.cancelFutureIfRunning() // Get set of external events val flowId = currentState.flowLogic.runId - try { - val oldFlowLeftOver = mutex.locked { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue - if (oldFlowLeftOver == null) { - logger.error("Unable to find flow for flow $flowId. Something is very wrong. The flow will not retry.") + val oldFlowLeftOver = innerState.withLock { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue + if (oldFlowLeftOver == null) { + logger.error("Unable to find flow for flow $flowId. Something is very wrong. The flow will not retry.") + return + } + val flow = if (currentState.isAnyCheckpointPersisted) { + // We intentionally grab the checkpoint from storage rather than relying on the one referenced by currentState. This is so that + // we mirror exactly what happens when restarting the node. + val serializedCheckpoint = database.transaction { checkpointStorage.getCheckpoint(flowId) } + if (serializedCheckpoint == null) { + logger.error("Unable to find database checkpoint for flow $flowId. Something is very wrong. The flow will not retry.") return } - val flow = if (currentState.isAnyCheckpointPersisted) { - // We intentionally grab the checkpoint from storage rather than relying on the one referenced by currentState. This is so that - // we mirror exactly what happens when restarting the node. - val serializedCheckpoint = checkpointStorage.getCheckpoint(flowId) - if (serializedCheckpoint == null) { - logger.error("Unable to find database checkpoint for flow $flowId. Something is very wrong. The flow will not retry.") - return - } - val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) ?: return - // Resurrect flow - flowCreator.createFlowFromCheckpoint(flowId, checkpoint) ?: return - } else { - // Just flow initiation message - null + val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) ?: return + // Resurrect flow + flowCreator.createFlowFromCheckpoint(flowId, checkpoint) ?: return + } else { + // Just flow initiation message + null + } + innerState.withLock { + if (stopping) { + return } - mutex.locked { - if (stopping) { - return - } - // Remove any sessions the old flow has. - for (sessionId in getFlowSessionIds(currentState.checkpoint)) { - sessionToFlow.remove(sessionId) - } - if (flow != null) { - injectOldProgressTracker(currentState.flowLogic.progressTracker, flow.fiber.logic) - addAndStartFlow(flowId, flow) - } - // Deliver all the external events from the old flow instance. - val unprocessedExternalEvents = mutableListOf() - do { - val event = oldFlowLeftOver.tryReceive() - if (event is Event.GeneratedByExternalEvent) { - unprocessedExternalEvents += event.deduplicationHandler.externalCause - } - } while (event != null) - val externalEvents = currentState.pendingDeduplicationHandlers.map { it.externalCause } + unprocessedExternalEvents - for (externalEvent in externalEvents) { - deliverExternalEvent(externalEvent) - } + // Remove any sessions the old flow has. + for (sessionId in getFlowSessionIds(currentState.checkpoint)) { + sessionToFlow.remove(sessionId) + } + if (flow != null) { + injectOldProgressTracker(currentState.flowLogic.progressTracker, flow.fiber.logic) + addAndStartFlow(flowId, flow) + } + // Deliver all the external events from the old flow instance. + val unprocessedExternalEvents = mutableListOf() + do { + val event = oldFlowLeftOver.tryReceive() + if (event is Event.GeneratedByExternalEvent) { + unprocessedExternalEvents += event.deduplicationHandler.externalCause + } + } while (event != null) + val externalEvents = currentState.pendingDeduplicationHandlers.map { it.externalCause } + unprocessedExternalEvents + for (externalEvent in externalEvents) { + deliverExternalEvent(externalEvent) } - } catch (e: Exception) { - // Failed to retry - manually put the flow in for observation rather than - // relying on the [HospitalisingInterceptor] to do so - val exceptions = (currentState.checkpoint.errorState as? ErrorState.Errored) - ?.errors - ?.map { it.exception } - ?.plus(e) ?: emptyList() - logger.info("Failed to retry flow $flowId, keeping in for observation and aborting") - flowHospital.forceIntoOvernightObservation(flowId, exceptions) - throw e } } override fun deliverExternalEvent(event: ExternalEvent) { - mutex.locked { + innerState.withLock { if (!stopping) { when (event) { is ExternalEvent.ExternalMessageEvent -> onSessionMessage(event) @@ -527,7 +495,7 @@ class SingleThreadedStateMachineManager( } } else { val event = Event.DeliverSessionMessage(sessionMessage, deduplicationHandler, sender) - mutex.locked { + innerState.withLock { flows[flowId]?.run { fiber.scheduleEvent(event) } // If flow is not running add it to the list of external events to be processed if/when the flow resumes. ?: pausedFlows[flowId]?.run { addExternalEvent(event) } @@ -623,18 +591,20 @@ class SingleThreadedStateMachineManager( deduplicationHandler: DeduplicationHandler? ): CordaFuture> { - val flowAlreadyExists = mutex.locked { flows[flowId] != null } - - val existingCheckpoint = if (flowAlreadyExists) { + val existingFlow = innerState.withLock { flows[flowId] } + val existingCheckpoint = if (existingFlow != null && existingFlow.fiber.transientState?.value?.isAnyCheckpointPersisted == true) { // Load the flow's checkpoint // The checkpoint will be missing if the flow failed before persisting the original checkpoint // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) - checkpointStorage.getCheckpoint(flowId)?.let { serializedCheckpoint -> + val existingCheckpoint = database.transaction { checkpointStorage.getCheckpoint(flowId) } + existingCheckpoint?.let { serializedCheckpoint -> val checkpoint = tryDeserializeCheckpoint(serializedCheckpoint, flowId) if (checkpoint == null) { return openFuture>().mapError { - IllegalStateException("Unable to deserialize database checkpoint for flow $flowId. " + - "Something is very wrong. The flow will not retry.") + IllegalStateException( + "Unable to deserialize database checkpoint for flow $flowId. " + + "Something is very wrong. The flow will not retry." + ) } } else { checkpoint @@ -647,7 +617,7 @@ class SingleThreadedStateMachineManager( val flow = flowCreator.createFlowFromLogic(flowId, invocationContext, flowLogic, flowStart, ourIdentity, existingCheckpoint, deduplicationHandler, ourSenderUUID) val startedFuture = openFuture() - mutex.locked { + innerState.withLock { startedFutures[flowId] = startedFuture } totalStartedFlows.inc() @@ -656,108 +626,11 @@ class SingleThreadedStateMachineManager( } override fun scheduleFlowTimeout(flowId: StateMachineRunId) { - mutex.locked { scheduleTimeout(flowId) } + flowTimeoutScheduler.timeout(flowId) } override fun cancelFlowTimeout(flowId: StateMachineRunId) { - mutex.locked { cancelTimeoutIfScheduled(flowId) } - } - - override fun scheduleFlowSleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) { - flowSleepScheduler.sleep(fiber, currentState, duration) - } - - override fun scheduleFlowWakeUp(instanceId: StateMachineInstanceId) { - mutex.locked { - flows[instanceId.runId]?.let { flow -> - // Only schedule a wake up event if the fiber the flow is executing on has not changed - if (flow.fiber.instanceId == instanceId) { - flowSleepScheduler.scheduleWakeUp(flow.fiber) - } - } - } - } - - private fun cancelFlowSleep(currentState: StateMachineState) { - flowSleepScheduler.cancel(currentState) - } - - /** - * Schedules the flow [flowId] to be retried if it does not finish within the timeout period - * specified in the config. - * - * Assumes lock is taken on the [InnerState]. - */ - private fun InnerState.scheduleTimeout(flowId: StateMachineRunId) { - val flow = flows[flowId] - if (flow != null) { - val scheduledTimeout = timedFlows[flowId] - val retryCount = if (scheduledTimeout != null) { - val timeoutFuture = scheduledTimeout.scheduledFuture - if (!timeoutFuture.isDone) scheduledTimeout.scheduledFuture.cancel(true) - scheduledTimeout.retryCount - } else 0 - val scheduledFuture = scheduleTimeoutException(flow, calculateDefaultTimeoutSeconds(retryCount)) - timedFlows[flowId] = ScheduledTimeout(scheduledFuture, retryCount + 1) - } else { - logger.warn("Unable to schedule timeout for flow $flowId – flow not found.") - } - } - - private fun resetCustomTimeout(flowId: StateMachineRunId, timeoutSeconds: Long) { - if (timeoutSeconds < serviceHub.configuration.flowTimeout.timeout.seconds) { - logger.debug { "Ignoring request to set time-out on timed flow $flowId to $timeoutSeconds seconds which is shorter than default of ${serviceHub.configuration.flowTimeout.timeout.seconds} seconds." } - return - } - logger.debug { "Processing request to set time-out on timed flow $flowId to $timeoutSeconds seconds." } - mutex.locked { - resetCustomTimeout(flowId, timeoutSeconds) - } - } - - private fun InnerState.resetCustomTimeout(flowId: StateMachineRunId, timeoutSeconds: Long) { - val flow = flows[flowId] - if (flow != null) { - val scheduledTimeout = timedFlows[flowId] - val retryCount = if (scheduledTimeout != null) { - val timeoutFuture = scheduledTimeout.scheduledFuture - if (!timeoutFuture.isDone) scheduledTimeout.scheduledFuture.cancel(true) - scheduledTimeout.retryCount - } else 0 - val scheduledFuture = scheduleTimeoutException(flow, timeoutSeconds) - timedFlows[flowId] = ScheduledTimeout(scheduledFuture, retryCount) - } else { - logger.warn("Unable to schedule timeout for flow $flowId – flow not found.") - } - } - - /** Schedules a [FlowTimeoutException] to be fired in order to restart the flow. */ - private fun scheduleTimeoutException(flow: Flow<*>, delay: Long): ScheduledFuture<*> { - return with(serviceHub.configuration.flowTimeout) { - scheduledFutureExecutor.schedule({ - val event = Event.Error(FlowTimeoutException()) - flow.fiber.scheduleEvent(event) - }, delay, TimeUnit.SECONDS) - } - } - - private fun calculateDefaultTimeoutSeconds(retryCount: Int): Long { - return with(serviceHub.configuration.flowTimeout) { - val timeoutDelaySeconds = timeout.seconds * Math.pow(backoffBase, min(retryCount, maxRestartCount).toDouble()).toLong() - maxOf(1L, ((1.0 + Math.random()) * timeoutDelaySeconds / 2).toLong()) - } - } - - /** - * Cancels any scheduled flow timeout for [flowId]. - * - * Assumes lock is taken on the [InnerState]. - */ - private fun InnerState.cancelTimeoutIfScheduled(flowId: StateMachineRunId) { - timedFlows[flowId]?.let { (future, _) -> - if (!future.isDone) future.cancel(true) - timedFlows.remove(flowId) - } + flowTimeoutScheduler.cancel(flowId) } private fun tryDeserializeCheckpoint(serializedCheckpoint: Checkpoint.Serialized, flowId: StateMachineRunId): Checkpoint? { @@ -774,7 +647,7 @@ class SingleThreadedStateMachineManager( for (sessionId in getFlowSessionIds(checkpoint)) { sessionToFlow[sessionId] = id } - mutex.locked { + innerState.withLock { if (stopping) { startedFutures[id]?.setException(IllegalStateException("Will not start flow as SMM is stopping")) logger.trace("Not resuming as SMM is stopping.") @@ -787,7 +660,7 @@ class SingleThreadedStateMachineManager( oldFlow.resultFuture.captureLater(flow.resultFuture) } val flowLogic = flow.fiber.logic - if (flowLogic.isEnabledTimedFlow()) scheduleTimeout(id) + if (flowLogic.isEnabledTimedFlow()) flowTimeoutScheduler.timeout(id) flow.fiber.scheduleEvent(Event.DoRemainingWork) startOrResume(checkpoint, flow) } @@ -817,11 +690,12 @@ class SingleThreadedStateMachineManager( private fun makeActionExecutor(checkpointSerializationContext: CheckpointSerializationContext): ActionExecutor { return ActionExecutorImpl( - serviceHub, - checkpointStorage, - flowMessaging, - this, - checkpointSerializationContext + serviceHub, + checkpointStorage, + flowMessaging, + this, + actionFutureExecutor, + checkpointSerializationContext ) } @@ -847,7 +721,7 @@ class SingleThreadedStateMachineManager( return StaffedFlowHospital(flowMessaging, serviceHub.clock, ourSenderUUID) } - private fun InnerState.removeFlowOrderly( + private fun StateMachineInnerState.removeFlowOrderly( flow: Flow<*>, removalReason: FlowRemovalReason.OrderlyFinish, lastState: StateMachineState @@ -863,7 +737,7 @@ class SingleThreadedStateMachineManager( changesPublisher.onNext(StateMachineManager.Change.Removed(lastState.flowLogic, Try.Success(removalReason.flowReturnValue))) } - private fun InnerState.removeFlowError( + private fun StateMachineInnerState.removeFlowError( flow: Flow<*>, removalReason: FlowRemovalReason.ErrorFinish, lastState: StateMachineState @@ -874,6 +748,8 @@ class SingleThreadedStateMachineManager( (exception as? FlowException)?.originalErrorId = flowError.errorId flow.resultFuture.setException(exception) lastState.flowLogic.progressTracker?.endWithError(exception) + // Complete the started future, needed when the flow fails during flow init (before completing an [UnstartedFlowTransition]) + startedFutures.remove(flow.fiber.id)?.set(Unit) changesPublisher.onNext(StateMachineManager.Change.Removed(lastState.flowLogic, Try.Failure(exception))) } @@ -901,4 +777,12 @@ class SingleThreadedStateMachineManager( } } } + + private fun StateMachineState.cancelFutureIfRunning() { + future?.run { + logger.debug { "Cancelling future for flow ${flowLogic.runId}" } + if (!isDone) cancel(true) + future = null + } + } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt index af7d197d50..4d6e73bfbe 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt @@ -32,12 +32,14 @@ import java.time.Instant import java.util.* import java.util.concurrent.ConcurrentHashMap import javax.persistence.PersistenceException +import kotlin.collections.HashMap import kotlin.concurrent.timerTask import kotlin.math.pow /** * This hospital consults "staff" to see if they can automatically diagnose and treat flows. */ +@Suppress("TooManyFunctions") class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val clock: Clock, private val ourSenderUUID: String) : Closeable { @@ -52,15 +54,24 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, DatabaseEndocrinologist, TransitionErrorGeneralPractitioner, SedationNurse, - NotaryDoctor + NotaryDoctor, + ResuscitationSpecialist ) + private const val MAX_BACKOFF_TIME = 110.0 // Totals to 2 minutes when calculating the backoff time + @VisibleForTesting val onFlowKeptForOvernightObservation = mutableListOf<(id: StateMachineRunId, by: List) -> Unit>() @VisibleForTesting val onFlowDischarged = mutableListOf<(id: StateMachineRunId, by: List) -> Unit>() + @VisibleForTesting + val onFlowErrorPropagated = mutableListOf<(id: StateMachineRunId, by: List) -> Unit>() + + @VisibleForTesting + val onFlowResuscitated = mutableListOf<(id: StateMachineRunId, by: List, outcome: Outcome) -> Unit>() + @VisibleForTesting val onFlowAdmitted = mutableListOf<(id: StateMachineRunId) -> Unit>() } @@ -146,6 +157,8 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, val payload = RejectSessionMessage(message, secureRandom.nextLong()) val replyError = ExistingSessionMessage(sessionMessage.initiatorSessionId, payload) + log.info("Sending session initiation error back to $sender", error) + flowMessaging.sendSessionMessage(sender, replyError, SenderDeduplicationId(DeduplicationId.createRandom(secureRandom), ourSenderUUID)) event.deduplicationHandler.afterDatabaseTransaction() } @@ -165,39 +178,38 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, } /** - * Forces the flow to be kept in for overnight observation by the hospital. A flow must already exist inside the hospital - * and have existing medical records for it to be moved to overnight observation. If it does not meet these criteria then - * an [IllegalArgumentException] will be thrown. + * Forces the flow to be kept in for overnight observation by the hospital. * - * @param id The [StateMachineRunId] of the flow that you are trying to force into observation + * @param currentState The [StateMachineState] of the flow that is being forced into observation * @param errors The errors to include in the new medical record */ - fun forceIntoOvernightObservation(id: StateMachineRunId, errors: List) { + fun forceIntoOvernightObservation(currentState: StateMachineState, errors: List) { mutex.locked { - // If a flow does not meet the criteria below, then it has moved into an invalid state or the function is being - // called from an incorrect location. The assertions below should error out the flow if they are not true. - requireNotNull(flowsInHospital[id]) { "Flow must already be in the hospital before forcing into overnight observation" } - val history = requireNotNull(flowPatients[id]) { "Flow must already have history before forcing into overnight observation" } - // Use the last staff member that last discharged the flow as the current staff member - val record = history.records.last().copy( + val id = currentState.flowLogic.runId + val medicalHistory = flowPatients.computeIfAbsent(id) { FlowMedicalHistory() } + val record = MedicalRecord.Flow( time = clock.instant(), + flowId = id, + suspendCount = currentState.checkpoint.checkpointState.numberOfSuspends, errors = errors, + by = listOf(TransitionErrorGeneralPractitioner), outcome = Outcome.OVERNIGHT_OBSERVATION ) + + medicalHistory.records += record + onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(id, record.by.map { it.toString() }) } - history.records += record recordsPublisher.onNext(record) } } /** - * Request treatment for the [flowFiber]. A flow can only be added to the hospital if they are not already being - * treated. + * Request treatment for the [flowFiber]. */ fun requestTreatment(flowFiber: FlowFiber, currentState: StateMachineState, errors: List) { - // Only treat flows that are not already in the hospital - if (!currentState.isRemoved && flowsInHospital.putIfAbsent(flowFiber.id, flowFiber) == null) { + if (!currentState.isRemoved) { + flowsInHospital[flowFiber.id] = flowFiber admit(flowFiber, currentState, errors) } } @@ -217,20 +229,30 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, Diagnosis.DISCHARGE -> { val backOff = calculateBackOffForChronicCondition(report, medicalHistory, currentState) log.info("Flow error discharged from hospital (delay ${backOff.seconds}s) by ${report.by} (error was ${report.error.message})") - onFlowDischarged.forEach { hook -> hook.invoke(flowFiber.id, report.by.map{it.toString()}) } + onFlowDischarged.forEach { hook -> hook.invoke(flowFiber.id, report.by.map { it.toString() }) } Triple(Outcome.DISCHARGE, Event.RetryFlowFromSafePoint, backOff) } Diagnosis.OVERNIGHT_OBSERVATION -> { log.info("Flow error kept for overnight observation by ${report.by} (error was ${report.error.message})") // We don't schedule a next event for the flow - it will automatically retry from its checkpoint on node restart - onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(flowFiber.id, report.by.map{it.toString()}) } + onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(flowFiber.id, report.by.map { it.toString() }) } Triple(Outcome.OVERNIGHT_OBSERVATION, Event.OvernightObservation, 0.seconds) } Diagnosis.NOT_MY_SPECIALTY, Diagnosis.TERMINAL -> { // None of the staff care for these errors, or someone decided it is a terminal condition, so we let them propagate log.info("Flow error allowed to propagate", report.error) + onFlowErrorPropagated.forEach { hook -> hook.invoke(flowFiber.id, report.by.map { it.toString() }) } Triple(Outcome.UNTREATABLE, Event.StartErrorPropagation, 0.seconds) } + Diagnosis.RESUSCITATE -> { + // reschedule the last outcome as it failed to process it + // do a 0.seconds backoff in dev mode? / when coming from the driver? make it configurable? + val backOff = calculateBackOffForResuscitation(medicalHistory, currentState) + val outcome = medicalHistory.records.last().outcome + log.info("Flow error to be resuscitated, rescheduling previous outcome - $outcome (delay ${backOff.seconds}s) by ${report.by} (error was ${report.error.message})") + onFlowResuscitated.forEach { hook -> hook.invoke(flowFiber.id, report.by.map { it.toString() }, outcome) } + Triple(outcome, outcome.event, backOff) + } } val numberOfSuspends = currentState.checkpoint.checkpointState.numberOfSuspends @@ -249,18 +271,29 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, } } - private fun calculateBackOffForChronicCondition(report: ConsultationReport, medicalHistory: FlowMedicalHistory, currentState: StateMachineState): Duration { - return report.by.firstOrNull { it is Chronic }?.let { chronicStaff -> - return medicalHistory.timesDischargedForTheSameThing(chronicStaff, currentState).let { - if (it == 0) { - 0.seconds - } else { - maxOf(10, (10 + (Math.random()) * (10 * 1.5.pow(it)) / 2).toInt()).seconds - } - } + private fun calculateBackOffForChronicCondition( + report: ConsultationReport, + medicalHistory: FlowMedicalHistory, + currentState: StateMachineState + ): Duration { + return report.by.firstOrNull { it is Chronic }?.let { staff -> + calculateBackOff(medicalHistory.timesDischargedForTheSameThing(staff, currentState)) } ?: 0.seconds } + private fun calculateBackOffForResuscitation( + medicalHistory: FlowMedicalHistory, + currentState: StateMachineState + ): Duration = calculateBackOff(medicalHistory.timesResuscitated(currentState)) + + private fun calculateBackOff(timesDiagnosisGiven: Int): Duration { + return if (timesDiagnosisGiven == 0) { + 0.seconds + } else { + maxOf(10, (10 + (Math.random()) * minOf(MAX_BACKOFF_TIME, (10 * 1.5.pow(timesDiagnosisGiven)) / 2)).toInt()).seconds + } + } + private fun consultStaff(flowFiber: FlowFiber, currentState: StateMachineState, errors: List, @@ -322,6 +355,11 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, return records.count { it.outcome == Outcome.DISCHARGE && by in it.by && it.suspendCount == lastAdmittanceSuspendCount } } + fun timesResuscitated(currentState: StateMachineState): Int { + val lastAdmittanceSuspendCount = currentState.checkpoint.checkpointState.numberOfSuspends + return records.count { ResuscitationSpecialist in it.by && it.suspendCount == lastAdmittanceSuspendCount } + } + override fun toString(): String = "${this.javaClass.simpleName}(records = $records)" } @@ -355,10 +393,16 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, } } - enum class Outcome { DISCHARGE, OVERNIGHT_OBSERVATION, UNTREATABLE } + enum class Outcome(val event: Event) { + DISCHARGE(Event.RetryFlowFromSafePoint), + OVERNIGHT_OBSERVATION(Event.OvernightObservation), + UNTREATABLE(Event.StartErrorPropagation) + } /** The order of the enum values are in priority order. */ enum class Diagnosis { + /** Retry the last outcome/diagnosis **/ + RESUSCITATE, /** The flow should not see other staff members */ TERMINAL, /** Retry from last safe point. */ @@ -373,6 +417,11 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, fun consult(flowFiber: FlowFiber, currentState: StateMachineState, newError: Throwable, history: FlowMedicalHistory): Diagnosis } + /** + * The [Chronic] interface relates to [Staff] that return diagnoses that can be constantly be diagnosed if the flow keeps returning to + * the hospital. [Chronic] diagnoses apply a backoff before scheduling a new [Event], this prevents a flow from constantly retrying + * without a chance for the underlying issue to resolve itself. + */ interface Chronic /** @@ -543,10 +592,10 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, newError.mentionsThrowable(AsyncOperationTransitionException::class.java) -> Diagnosis.NOT_MY_SPECIALTY history.notDischargedForTheSameThingMoreThan(2, this, currentState) -> Diagnosis.DISCHARGE else -> Diagnosis.OVERNIGHT_OBSERVATION - } + }.also { logDiagnosis(it, newError, flowFiber, history) } } else { Diagnosis.NOT_MY_SPECIALTY - }.also { logDiagnosis(it, newError, flowFiber, history) } + } } private fun logDiagnosis(diagnosis: Diagnosis, newError: Throwable, flowFiber: FlowFiber, history: FlowMedicalHistory) { @@ -597,6 +646,25 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, return Diagnosis.NOT_MY_SPECIALTY } } + + /** + * Handles errors coming from the processing of errors events ([Event.StartErrorPropagation] and [Event.RetryFlowFromSafePoint]), + * returning a [Diagnosis.RESUSCITATE] diagnosis + */ + object ResuscitationSpecialist : Staff { + override fun consult( + flowFiber: FlowFiber, + currentState: StateMachineState, + newError: Throwable, + history: FlowMedicalHistory + ): Diagnosis { + return if (newError is ErrorStateTransitionException) { + Diagnosis.RESUSCITATE + } else { + Diagnosis.NOT_MY_SPECIALTY + } + } + } } private fun Throwable?.mentionsThrowable(exceptionType: Class, errorMessage: String? = null): Boolean { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineInnerState.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineInnerState.kt new file mode 100644 index 0000000000..0252e21e80 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineInnerState.kt @@ -0,0 +1,44 @@ +package net.corda.node.services.statemachine + +import net.corda.core.flows.StateMachineRunId +import net.corda.core.internal.concurrent.OpenFuture +import net.corda.node.services.statemachine.StateMachineManager.Change +import rx.subjects.PublishSubject +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.locks.Lock +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock + +internal interface StateMachineInnerState { + val lock: Lock + val flows: MutableMap> + val pausedFlows: MutableMap + val startedFutures: MutableMap> + val changesPublisher: PublishSubject + /** Flows scheduled to be retried if not finished within the specified timeout period. */ + val timedFlows: MutableMap + + fun withMutex(block: StateMachineInnerState.() -> R): R +} + +internal class StateMachineInnerStateImpl : StateMachineInnerState { + /** True if we're shutting down, so don't resume anything. */ + var stopping = false + override val lock = ReentrantLock() + override val changesPublisher = PublishSubject.create()!! + override val flows = HashMap>() + override val pausedFlows = HashMap() + override val startedFutures = HashMap>() + override val timedFlows = HashMap() + + override fun withMutex(block: StateMachineInnerState.() -> R): R = lock.withLock { block(this) } +} + +internal inline fun T.withLock(block: T.() -> R): R = lock.withLock { block(this) } + +internal data class ScheduledTimeout( + /** Will fire a [FlowTimeoutException] indicating to the flow hospital to restart the flow. */ + val scheduledFuture: ScheduledFuture<*>, + /** Specifies the number of times this flow has been retried. */ + val retryCount: Int = 0 +) \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt index c6aebbdf62..6c5050962b 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt @@ -10,7 +10,6 @@ import net.corda.core.utilities.Try import net.corda.node.services.messaging.DeduplicationHandler import net.corda.node.services.messaging.ReceivedMessage import rx.Observable -import java.time.Duration /** * A StateMachineManager is responsible for coordination and persistence of multiple [FlowStateMachine] objects. @@ -102,7 +101,7 @@ interface StateMachineManager { // These must be idempotent! A later failure in the state transition may error the flow state, and a replay may call // these functions again -interface StateMachineManagerInternal { +internal interface StateMachineManagerInternal { fun signalFlowHasStarted(flowId: StateMachineRunId) fun addSessionBinding(flowId: StateMachineRunId, sessionId: SessionId) fun removeSessionBindings(sessionIds: Set) @@ -110,8 +109,6 @@ interface StateMachineManagerInternal { fun retryFlowFromSafePoint(currentState: StateMachineState) fun scheduleFlowTimeout(flowId: StateMachineRunId) fun cancelFlowTimeout(flowId: StateMachineRunId) - fun scheduleFlowSleep(fiber: FlowFiber, currentState: StateMachineState, duration: Duration) - fun scheduleFlowWakeUp(instanceId: StateMachineInstanceId) } /** diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt index e32014ab18..2e37261c04 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt @@ -16,3 +16,5 @@ class StateTransitionException( } class AsyncOperationTransitionException(exception: Exception) : CordaException(exception.message, exception) + +class ErrorStateTransitionException(val exception: Exception) : CordaException(exception.message, exception) \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt index 7d9b518c05..8b22573421 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt @@ -9,6 +9,7 @@ import net.corda.nodeapi.internal.persistence.DatabaseTransactionException import net.corda.nodeapi.internal.persistence.contextDatabase import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import java.security.SecureRandom +import java.sql.SQLException import javax.persistence.OptimisticLockException /** @@ -19,8 +20,8 @@ import javax.persistence.OptimisticLockException * completely aborted to avoid error loops. */ class TransitionExecutorImpl( - val secureRandom: SecureRandom, - val database: CordaPersistence + val secureRandom: SecureRandom, + val database: CordaPersistence ) : TransitionExecutor { private companion object { @@ -30,36 +31,44 @@ class TransitionExecutorImpl( @Suppress("NestedBlockDepth", "ReturnCount") @Suspendable override fun executeTransition( - fiber: FlowFiber, - previousState: StateMachineState, - event: Event, - transition: TransitionResult, - actionExecutor: ActionExecutor + fiber: FlowFiber, + previousState: StateMachineState, + event: Event, + transition: TransitionResult, + actionExecutor: ActionExecutor ): Pair { contextDatabase = database for (action in transition.actions) { try { actionExecutor.executeAction(fiber, action) } catch (exception: Exception) { - contextTransactionOrNull?.run { - rollback() - close() - } + rollbackTransactionOnError() if (transition.newState.checkpoint.errorState is ErrorState.Errored) { - // If we errored while transitioning to an error state then we cannot record the additional - // error as that may result in an infinite loop, e.g. error propagation fails -> record error -> propagate fails again. - // Instead we just keep around the old error state and wait for a new schedule, perhaps - // triggered from a flow hospital - log.warn("Error while executing $action during transition to errored state, aborting transition", exception) - // CORDA-3354 - Go to the hospital with the new error that has occurred - // while already in a error state (as this error could be for a different reason) - return Pair(FlowContinuation.Abort, previousState.copy(isFlowResumed = false)) + log.warn("Error while executing $action, with error event $event, updating errored state", exception) + + val newState = previousState.copy( + checkpoint = previousState.checkpoint.copy( + errorState = previousState.checkpoint.errorState.addErrors( + listOf( + FlowError( + secureRandom.nextLong(), + ErrorStateTransitionException(exception) + ) + ) + ) + ), + isFlowResumed = false + ) + + return Pair(FlowContinuation.ProcessEvents, newState) } else { // Otherwise error the state manually keeping the old flow state and schedule a DoRemainingWork // to trigger error propagation - if(previousState.isRemoved && exception is OptimisticLockException) { - log.debug("Flow has been killed and the following error is likely due to the flow's checkpoint being deleted. " + - "Occurred while executing $action, with event $event", exception) + if (log.isDebugEnabled && previousState.isRemoved && exception is OptimisticLockException) { + log.debug( + "Flow has been killed and the following error is likely due to the flow's checkpoint being deleted. " + + "Occurred while executing $action, with event $event", exception + ) } else { log.info("Error while executing $action, with event $event, erroring state", exception) } @@ -77,12 +86,12 @@ class TransitionExecutorImpl( } val newState = previousState.copy( - checkpoint = previousState.checkpoint.copy( - errorState = previousState.checkpoint.errorState.addErrors( - listOf(FlowError(secureRandom.nextLong(), stateTransitionOrDatabaseTransactionException)) - ) - ), - isFlowResumed = false + checkpoint = previousState.checkpoint.copy( + errorState = previousState.checkpoint.errorState.addErrors( + listOf(FlowError(secureRandom.nextLong(), stateTransitionOrDatabaseTransactionException)) + ) + ), + isFlowResumed = false ) fiber.scheduleEvent(Event.DoRemainingWork) return Pair(FlowContinuation.ProcessEvents, newState) @@ -91,4 +100,25 @@ class TransitionExecutorImpl( } return Pair(transition.continuation, transition.newState) } + + private fun rollbackTransactionOnError() { + contextTransactionOrNull?.run { + try { + rollback() + } catch (rollbackException: SQLException) { + log.info( + "Error rolling back database transaction from a previous error, continuing error handling for the original error", + rollbackException + ) + } + try { + close() + } catch (rollbackException: SQLException) { + log.info( + "Error closing database transaction from a previous error, continuing error handling for the original error", + rollbackException + ) + } + } + } } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt index 36026d671a..562bbf6ea8 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt @@ -17,17 +17,17 @@ import net.corda.node.services.statemachine.transitions.TransitionResult * transition. */ class HospitalisingInterceptor( - private val flowHospital: StaffedFlowHospital, - private val delegate: TransitionExecutor + private val flowHospital: StaffedFlowHospital, + private val delegate: TransitionExecutor ) : TransitionExecutor { @Suspendable override fun executeTransition( - fiber: FlowFiber, - previousState: StateMachineState, - event: Event, - transition: TransitionResult, - actionExecutor: ActionExecutor + fiber: FlowFiber, + previousState: StateMachineState, + event: Event, + transition: TransitionResult, + actionExecutor: ActionExecutor ): Pair { // If the fiber's previous state was clean then remove it from the hospital @@ -38,8 +38,8 @@ class HospitalisingInterceptor( val (continuation, nextState) = delegate.executeTransition(fiber, previousState, event, transition, actionExecutor) - if (nextState.checkpoint.errorState is ErrorState.Errored && previousState.checkpoint.errorState is ErrorState.Clean) { - val exceptionsToHandle = nextState.checkpoint.errorState.errors.map { it.exception } + if (canEnterHospital(previousState, nextState)) { + val exceptionsToHandle = (nextState.checkpoint.errorState as ErrorState.Errored).errors.map { it.exception } flowHospital.requestTreatment(fiber, previousState, exceptionsToHandle) } if (nextState.isRemoved) { @@ -48,6 +48,11 @@ class HospitalisingInterceptor( return Pair(continuation, nextState) } + private fun canEnterHospital(previousState: StateMachineState, nextState: StateMachineState): Boolean { + return nextState.checkpoint.errorState is ErrorState.Errored + && (previousState.checkpoint.errorState as? ErrorState.Errored)?.errors != nextState.checkpoint.errorState.errors + } + private fun removeFlow(id: StateMachineRunId) { flowHospital.leave(id) flowHospital.removeMedicalHistory(id) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt index 21b06c6e40..7d56967c24 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/DoRemainingWorkTransition.kt @@ -1,6 +1,8 @@ package net.corda.node.services.statemachine.transitions -import net.corda.node.services.statemachine.* +import net.corda.node.services.statemachine.ErrorState +import net.corda.node.services.statemachine.FlowState +import net.corda.node.services.statemachine.StateMachineState /** * This transition checks the current state of the flow and determines whether anything needs to be done. diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt index 904ab3f06a..96b6557829 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/StartedFlowTransition.kt @@ -105,11 +105,12 @@ class StartedFlowTransition( // This ensures that the [WaitForLedgerCommit] request is not executed multiple times if extra // [DoRemainingWork] events are pushed onto the fiber's event queue before the flow has really woken up return if (!startingState.isWaitingForFuture) { + val state = startingState.copy(isWaitingForFuture = true) TransitionResult( - newState = startingState.copy(isWaitingForFuture = true), + newState = state, actions = listOf( Action.CreateTransaction, - Action.TrackTransaction(flowIORequest.hash), + Action.TrackTransaction(flowIORequest.hash, state), Action.CommitTransaction ) ) @@ -432,8 +433,8 @@ class StartedFlowTransition( // The `numberOfSuspends` is added to the deduplication ID in case an async // operation is executed multiple times within the same flow. val deduplicationId = context.id.toString() + ":" + currentState.checkpoint.checkpointState.numberOfSuspends.toString() - actions.add(Action.ExecuteAsyncOperation(deduplicationId, flowIORequest.operation)) currentState = currentState.copy(isWaitingForFuture = true) + actions += Action.ExecuteAsyncOperation(deduplicationId, flowIORequest.operation, currentState) FlowContinuation.ProcessEvents } } else { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt index 037d408928..1b7d79dfec 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TopLevelTransition.kt @@ -3,7 +3,9 @@ package net.corda.node.services.statemachine.transitions import net.corda.core.crypto.SecureHash import net.corda.core.flows.InitiatingFlow import net.corda.core.internal.FlowIORequest +import net.corda.core.serialization.deserialize import net.corda.core.utilities.Try +import net.corda.node.services.messaging.DeduplicationHandler import net.corda.node.services.statemachine.Action import net.corda.node.services.statemachine.Checkpoint import net.corda.node.services.statemachine.DeduplicationId @@ -11,12 +13,15 @@ import net.corda.node.services.statemachine.EndSessionMessage import net.corda.node.services.statemachine.ErrorState import net.corda.node.services.statemachine.Event import net.corda.node.services.statemachine.ExistingSessionMessage +import net.corda.node.services.statemachine.ExternalEvent import net.corda.node.services.statemachine.FlowRemovalReason import net.corda.node.services.statemachine.FlowSessionImpl import net.corda.node.services.statemachine.FlowState +import net.corda.node.services.statemachine.InitialSessionMessage import net.corda.node.services.statemachine.InitiatedSessionState import net.corda.node.services.statemachine.SenderDeduplicationId import net.corda.node.services.statemachine.SessionId +import net.corda.node.services.statemachine.SessionMessage import net.corda.node.services.statemachine.SessionState import net.corda.node.services.statemachine.StateMachineState import net.corda.node.services.statemachine.SubFlow @@ -62,7 +67,7 @@ class TopLevelTransition( private fun errorTransition(event: Event.Error): TransitionResult { return builder { - freshErrorTransition(event.exception) + freshErrorTransition(event.exception, event.rollback) FlowContinuation.ProcessEvents } } @@ -314,24 +319,40 @@ class TopLevelTransition( private fun retryFlowFromSafePointTransition(startingState: StateMachineState): TransitionResult { return builder { // Need to create a flow from the prior checkpoint or flow initiation. - actions.add(Action.CreateTransaction) actions.add(Action.RetryFlowFromSafePoint(startingState)) - actions.add(Action.CommitTransaction) FlowContinuation.Abort } } private fun overnightObservationTransition(): TransitionResult { return builder { + val flowStartEvents = currentState.pendingDeduplicationHandlers.filter(::isFlowStartEvent) val newCheckpoint = startingState.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED) - actions.add(Action.CreateTransaction) - actions.add(Action.PersistCheckpoint(context.id, newCheckpoint, isCheckpointUpdate = currentState.isAnyCheckpointPersisted)) - actions.add(Action.CommitTransaction) - currentState = currentState.copy(checkpoint = newCheckpoint) + actions += Action.CreateTransaction + actions += Action.PersistDeduplicationFacts(flowStartEvents) + actions += Action.PersistCheckpoint(context.id, newCheckpoint, isCheckpointUpdate = currentState.isAnyCheckpointPersisted) + actions += Action.CommitTransaction + actions += Action.AcknowledgeMessages(flowStartEvents) + currentState = currentState.copy( + checkpoint = startingState.checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED), + pendingDeduplicationHandlers = currentState.pendingDeduplicationHandlers - flowStartEvents + ) FlowContinuation.ProcessEvents } } + private fun isFlowStartEvent(handler: DeduplicationHandler): Boolean { + return handler.externalCause.run { isSessionInit() || isFlowStart() } + } + + private fun ExternalEvent.isSessionInit(): Boolean { + return this is ExternalEvent.ExternalMessageEvent && this.receivedMessage.data.deserialize() is InitialSessionMessage + } + + private fun ExternalEvent.isFlowStart(): Boolean { + return this is ExternalEvent.ExternalStartFlowEvent<*> + } + private fun wakeUpFromSleepTransition(): TransitionResult { return builder { resumeFlowLogic(Unit) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt index bfcd317768..5e6ca3adbb 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/transitions/TransitionBuilder.kt @@ -28,12 +28,12 @@ class TransitionBuilder(val context: TransitionContext, initialState: StateMachi * * @param error the error. */ - fun freshErrorTransition(error: Throwable) { + fun freshErrorTransition(error: Throwable, rollback: Boolean = true) { val flowError = FlowError( errorId = (error as? IdentifiableException)?.errorId ?: context.secureRandom.nextLong(), exception = error ) - errorTransition(flowError) + errorTransition(flowError, rollback) } /** @@ -42,7 +42,7 @@ class TransitionBuilder(val context: TransitionContext, initialState: StateMachi * * @param error the error. */ - fun errorsTransition(errors: List) { + fun errorsTransition(errors: List, rollback: Boolean) { currentState = currentState.copy( checkpoint = currentState.checkpoint.copy( errorState = currentState.checkpoint.errorState.addErrors(errors) @@ -50,10 +50,10 @@ class TransitionBuilder(val context: TransitionContext, initialState: StateMachi isFlowResumed = false ) actions.clear() - actions.addAll(arrayOf( - Action.RollbackTransaction, - Action.ScheduleEvent(Event.DoRemainingWork) - )) + if(rollback) { + actions += Action.RollbackTransaction + } + actions += Action.ScheduleEvent(Event.DoRemainingWork) } /** @@ -62,8 +62,8 @@ class TransitionBuilder(val context: TransitionContext, initialState: StateMachi * * @param error the error. */ - fun errorTransition(error: FlowError) { - errorsTransition(listOf(error)) + fun errorTransition(error: FlowError, rollback: Boolean) { + errorsTransition(listOf(error), rollback) } fun resumeFlowLogic(result: Any?): FlowContinuation { diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index 6dfd3b6e04..ad7d80059d 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -62,7 +62,7 @@ class NodeVaultService( companion object { private val log = contextLogger() - val MAX_SQL_IN_CLAUSE_SET = 16 + const val DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE = 16 /** * Establish whether a given state is relevant to a node, given the node's public keys. @@ -870,7 +870,7 @@ private fun CriteriaBuilder.executeUpdate( var updatedRows = 0 it.asSequence() .map { stateRef -> PersistentStateRef(stateRef.txhash.bytes.toHexString(), stateRef.index) } - .chunked(NodeVaultService.MAX_SQL_IN_CLAUSE_SET) + .chunked(NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE) .forEach { persistentStateRefs -> updatedRows += doUpdate(persistentStateRefs) } diff --git a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt index 4a514e0172..4d6dd05b19 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt @@ -63,6 +63,7 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi name == "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) name == "PublicKeyToOwningIdentityCache_cache" -> caffeine.maximumSize(defaultCacheSize) name == "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + name == "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultAttachmentsClassLoaderCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name. Did you add a new cache?") } } @@ -85,4 +86,6 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi } open protected val defaultCacheSize = 1024L -} \ No newline at end of file + private val defaultAttachmentsClassLoaderCacheSize = defaultCacheSize / CACHE_SIZE_DENOMINATOR +} +private const val CACHE_SIZE_DENOMINATOR = 4L \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/utilities/StateMachineManagerUtils.kt b/node/src/main/kotlin/net/corda/node/utilities/StateMachineManagerUtils.kt index 25ec223836..99aeebc6d8 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/StateMachineManagerUtils.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/StateMachineManagerUtils.kt @@ -14,7 +14,7 @@ import java.lang.reflect.Field * If the new tracker contains any child trackers from sub-flows, we need to attach those to the old tracker as well. */ //TODO: instead of replacing the progress tracker after constructing the flow logic, we should inject it during fiber deserialization -fun StateMachineManagerInternal.injectOldProgressTracker(oldTracker: ProgressTracker?, newFlowLogic: FlowLogic<*>) { +internal fun StateMachineManagerInternal.injectOldProgressTracker(oldTracker: ProgressTracker?, newFlowLogic: FlowLogic<*>) { if (oldTracker != null) { val newTracker = newFlowLogic.progressTracker if (newTracker != null) { diff --git a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt index 45aa089f9e..8d2558ca8e 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelper.kt @@ -25,10 +25,10 @@ import org.bouncycastle.operator.ContentSigner import org.bouncycastle.util.io.pem.PemObject import java.io.IOException import java.io.StringWriter +import java.lang.IllegalStateException import java.net.ConnectException import java.net.URL import java.nio.file.Path -import java.security.KeyPair import java.security.PublicKey import java.security.cert.X509Certificate import java.time.Duration @@ -63,6 +63,7 @@ open class NetworkRegistrationHelper( private val requestIdStore = certificatesDirectory / "certificate-request-id.txt" protected val rootTrustStore: X509KeyStore protected val rootCert: X509Certificate + private val notaryServiceConfig: NotaryServiceConfig? = config.notaryServiceConfig init { require(networkRootTrustStorePath.exists()) { @@ -95,34 +96,70 @@ open class NetworkRegistrationHelper( return } + notaryServiceConfig?.let { validateNotaryServiceKeyAndCert(certStore, it.notaryServiceKeyAlias, it.notaryServiceLegalName) } + val tlsCrlIssuerCert = getTlsCrlIssuerCert() // We use SELF_SIGNED_PRIVATE_KEY as progress indicator so we just store a dummy key and cert. // When registration succeeds, this entry should be deleted. certStore.query { setPrivateKey(SELF_SIGNED_PRIVATE_KEY, AliasPrivateKey(SELF_SIGNED_PRIVATE_KEY), listOf(NOT_YET_REGISTERED_MARKER_KEYS_AND_CERTS.ECDSAR1_CERT), certificateStore.entryPassword) } - val nodeCaPublicKey = loadOrGenerateKeyPair() + val (entityPublicKey, receivedCertificates) = generateKeyPairAndCertificate(nodeCaKeyAlias, myLegalName, certRole, certStore) - val requestId = submitOrResumeCertificateSigningRequest(nodeCaPublicKey, cryptoService.getSigner(nodeCaKeyAlias)) - - val nodeCaCertificates = pollServerForCertificates(requestId) - validateCertificates(nodeCaPublicKey, nodeCaCertificates) - - certStore.setCertPathOnly(nodeCaKeyAlias, nodeCaCertificates) - certStore.value.internal.deleteEntry(SELF_SIGNED_PRIVATE_KEY) - certStore.value.save() - logProgress("Private key '$nodeCaKeyAlias' and its certificate-chain stored successfully.") - - onSuccess(nodeCaPublicKey, cryptoService.getSigner(nodeCaKeyAlias), nodeCaCertificates, tlsCrlIssuerCert?.subjectX500Principal?.toX500Name()) + onSuccess(entityPublicKey, cryptoService.getSigner(nodeCaKeyAlias), receivedCertificates, tlsCrlIssuerCert?.subjectX500Principal?.toX500Name()) // All done, clean up temp files. requestIdStore.deleteIfExists() } - private fun loadOrGenerateKeyPair(): PublicKey { - return if (cryptoService.containsKey(nodeCaKeyAlias)) { - cryptoService.getPublicKey(nodeCaKeyAlias)!! + private fun generateKeyPairAndCertificate(keyAlias: String, legalName: CordaX500Name, certificateRole: CertRole, certStore: CertificateStore): Pair> { + val entityPublicKey = loadOrGenerateKeyPair(keyAlias) + + val requestId = submitOrResumeCertificateSigningRequest(entityPublicKey, legalName, certificateRole, cryptoService.getSigner(keyAlias)) + + val receivedCertificates = pollServerForCertificates(requestId) + validateCertificates(entityPublicKey, legalName, certificateRole, receivedCertificates) + + certStore.setCertPathOnly(keyAlias, receivedCertificates) + certStore.value.internal.deleteEntry(SELF_SIGNED_PRIVATE_KEY) + certStore.value.save() + logProgress("Private key '$keyAlias' and its certificate-chain stored successfully.") + return Pair(entityPublicKey, receivedCertificates) + } + + /** + * Used when registering a notary to validate that the shared notary service key and certificate can be accessed. + * + * In the case that the notary service certificate and key is not available, a new key key is generated and a separate CSR is + * submitted to the Identity Manager. + * + * If this method successfully completes then the [cryptoService] will contain the notary service key and the [certStore] will contain + * the notary service certificate chain. + * + * @throws IllegalStateException If the notary service certificate already exists but the private key is not available. + */ + private fun validateNotaryServiceKeyAndCert(certStore: CertificateStore, notaryServiceKeyAlias: String, notaryServiceLegalName: CordaX500Name) { + if (certStore.contains(notaryServiceKeyAlias) && !cryptoService.containsKey(notaryServiceKeyAlias)) { + throw IllegalStateException("Notary service identity certificate exists but key pair missing. " + + "Please check no old certificates exist in the certificate store.") + } + + if (certStore.contains(notaryServiceKeyAlias)) { + logProgress("Notary service certificate already exists. Continuing with node registration...") + return + } + + logProgress("Generating notary service identity for $notaryServiceLegalName...") + generateKeyPairAndCertificate(notaryServiceKeyAlias, notaryServiceLegalName, CertRole.SERVICE_IDENTITY, certStore) + // The request id store is reused for the next step - registering the node identity. + // Therefore we can remove this to enable it to be reused. + requestIdStore.deleteIfExists() + } + + private fun loadOrGenerateKeyPair(keyAlias: String): PublicKey { + return if (cryptoService.containsKey(keyAlias)) { + cryptoService.getPublicKey(keyAlias)!! } else { - cryptoService.generateKeyPair(nodeCaKeyAlias, cryptoService.defaultTLSSignatureScheme()) + cryptoService.generateKeyPair(keyAlias, cryptoService.defaultTLSSignatureScheme()) } } @@ -137,26 +174,31 @@ open class NetworkRegistrationHelper( return tlsCrlIssuerCert } - private fun validateCertificates(registeringPublicKey: PublicKey, certificates: List) { - val nodeCACertificate = certificates.first() + private fun validateCertificates( + registeringPublicKey: PublicKey, + registeringLegalName: CordaX500Name, + expectedCertRole: CertRole, + certificates: List + ) { + val receivedCertificate = certificates.first() - val nodeCaSubject = try { - CordaX500Name.build(nodeCACertificate.subjectX500Principal) + val certificateSubject = try { + CordaX500Name.build(receivedCertificate.subjectX500Principal) } catch (e: IllegalArgumentException) { - throw CertificateRequestException("Received node CA cert has invalid subject name: ${e.message}") + throw CertificateRequestException("Received cert has invalid subject name: ${e.message}") } - if (nodeCaSubject != myLegalName) { - throw CertificateRequestException("Subject of received node CA cert doesn't match with node legal name: $nodeCaSubject") + if (certificateSubject != registeringLegalName) { + throw CertificateRequestException("Subject of received cert doesn't match with legal name: $certificateSubject") } - val nodeCaCertRole = try { - CertRole.extract(nodeCACertificate) + val receivedCertRole = try { + CertRole.extract(receivedCertificate) } catch (e: IllegalArgumentException) { - throw CertificateRequestException("Unable to extract cert role from received node CA cert: ${e.message}") + throw CertificateRequestException("Unable to extract cert role from received cert: ${e.message}") } - if (certRole != nodeCaCertRole) { - throw CertificateRequestException("Received certificate contains invalid cert role, expected '$certRole', got '$nodeCaCertRole'.") + if (expectedCertRole != receivedCertRole) { + throw CertificateRequestException("Received certificate contains invalid cert role, expected '$expectedCertRole', got '$receivedCertRole'.") } // Validate returned certificate is for the correct public key. @@ -169,22 +211,6 @@ open class NetworkRegistrationHelper( logProgress("Certificate signing request approved, storing private key with the certificate chain.") } - private fun CertificateStore.loadOrCreateKeyPair(alias: String, entryPassword: String = password): KeyPair { - // Create or load self signed keypair from the key store. - // We use the self sign certificate to store the key temporarily in the keystore while waiting for the request approval. - if (alias !in this) { - // NODE_CA should be TLS compatible due to the cert hierarchy structure. - val keyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) - val selfSignCert = X509Utilities.createSelfSignedCACertificate(myLegalName.x500Principal, keyPair) - // Save to the key store. - with(value) { - setPrivateKey(alias, keyPair.private, listOf(selfSignCert), keyPassword = entryPassword) - save() - } - } - return query { getCertificateAndKeyPair(alias, entryPassword) }.keyPair - } - /** * Poll Certificate Signing Server for approved certificate, * enter a slow polling loop if server return null. @@ -226,20 +252,27 @@ open class NetworkRegistrationHelper( * Submit Certificate Signing Request to Certificate signing service if request ID not found in file system. * New request ID will be stored in requestId.txt * @param publicKey public key for which we need a certificate. + * @param legalName legal name of the entity for which we need a certificate. + * @param certRole desired role of the entities certificate. * @param contentSigner the [ContentSigner] that will sign the CSR. * @return Request ID return from the server. */ - private fun submitOrResumeCertificateSigningRequest(publicKey: PublicKey, contentSigner: ContentSigner): String { + private fun submitOrResumeCertificateSigningRequest( + publicKey: PublicKey, + legalName: CordaX500Name, + certRole: CertRole, + contentSigner: ContentSigner + ): String { try { // Retrieve request id from file if exists, else post a request to server. return if (!requestIdStore.exists()) { - val request = X509Utilities.createCertificateSigningRequest(myLegalName.x500Principal, emailAddress, publicKey, contentSigner, certRole) + val request = X509Utilities.createCertificateSigningRequest(legalName.x500Principal, emailAddress, publicKey, contentSigner, certRole) val writer = StringWriter() JcaPEMWriter(writer).use { it.writeObject(PemObject("CERTIFICATE REQUEST", request.encoded)) } logProgress("Certificate signing request with the following information will be submitted to the Corda certificate signing server.") - logProgress("Legal Name: $myLegalName") + logProgress("Legal Name: $legalName") logProgress("Email: $emailAddress") logProgress("Public Key: $publicKey") logProgress("$writer") @@ -277,7 +310,8 @@ class NodeRegistrationConfiguration( val certificatesDirectory: Path, val emailAddress: String, val cryptoService: CryptoService, - val certificateStore: CertificateStore) { + val certificateStore: CertificateStore, + val notaryServiceConfig: NotaryServiceConfig? = null) { constructor(config: NodeConfiguration) : this( p2pSslOptions = config.p2pSslOptions, @@ -287,10 +321,29 @@ class NodeRegistrationConfiguration( certificatesDirectory = config.certificatesDirectory, emailAddress = config.emailAddress, cryptoService = BCCryptoService(config.myLegalName.x500Principal, config.signingCertificateStore), - certificateStore = config.signingCertificateStore.get(true) + certificateStore = config.signingCertificateStore.get(true), + notaryServiceConfig = config.notary?.let { + // Validation of the presence of the notary service legal name is only done here and not in the top level configuration + // file. This is to maintain backwards compatibility with older notaries using the legacy identity structure. Older + // notaries will be signing requests using the nodes legal identity key and therefore no separate notary service entity + // exists. Just having the validation here prevents any new notaries from being created with the legacy identity scheme + // but still allows drop in JAR replacements for old notaries. + requireNotNull(it.serviceLegalName) { + "The notary service legal name must be provided via the 'notary.serviceLegalName' configuration parameter" + } + require(it.serviceLegalName != config.myLegalName) { + "The notary service legal name must be different from the node legal name" + } + NotaryServiceConfig(X509Utilities.DISTRIBUTED_NOTARY_KEY_ALIAS, it.serviceLegalName!!) + } ) } +data class NotaryServiceConfig( + val notaryServiceKeyAlias: String, + val notaryServiceLegalName: CordaX500Name +) + class NodeRegistrationException( message: String?, cause: Throwable? diff --git a/node/src/main/resources/reference.conf b/node/src/main/resources/corda-reference.conf similarity index 100% rename from node/src/main/resources/reference.conf rename to node/src/main/resources/corda-reference.conf diff --git a/node/src/main/resources/log4j2.component.properties b/node/src/main/resources/log4j2.component.properties index 1b55982139..405c40b154 100644 --- a/node/src/main/resources/log4j2.component.properties +++ b/node/src/main/resources/log4j2.component.properties @@ -1,2 +1,2 @@ Log4jContextSelector=net.corda.node.utilities.logging.AsyncLoggerContextSelectorNoThreadLocal -AsyncLogger.RingBufferSize=262144 \ No newline at end of file +AsyncLogger.RingBufferSize=16384 \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml b/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml index 3f9ed5cab1..6aedc510b4 100644 --- a/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml +++ b/node/src/main/resources/migration/node-core.changelog-v19-postgres.xml @@ -69,13 +69,13 @@ - + - + - + diff --git a/node/src/main/resources/migration/node-core.changelog-v19.xml b/node/src/main/resources/migration/node-core.changelog-v19.xml index cba014503c..6b8c1e9b24 100644 --- a/node/src/main/resources/migration/node-core.changelog-v19.xml +++ b/node/src/main/resources/migration/node-core.changelog-v19.xml @@ -69,13 +69,13 @@ - + - + - + diff --git a/node/src/test/kotlin/net/corda/node/CordaRPCOpsImplTest.kt b/node/src/test/kotlin/net/corda/node/CordaRPCOpsImplTest.kt index dcd4706019..d7b45e42e6 100644 --- a/node/src/test/kotlin/net/corda/node/CordaRPCOpsImplTest.kt +++ b/node/src/test/kotlin/net/corda/node/CordaRPCOpsImplTest.kt @@ -2,11 +2,13 @@ package net.corda.node import co.paralleluniverse.fibers.Suspendable import net.corda.client.rpc.PermissionException +import net.corda.client.rpc.RPCException import net.corda.core.context.AuthServiceId import net.corda.core.context.InvocationContext import net.corda.core.contracts.Amount import net.corda.core.contracts.ContractState import net.corda.core.contracts.Issued +import net.corda.core.crypto.SecureHash import net.corda.core.crypto.isFulfilledBy import net.corda.core.crypto.keys import net.corda.core.flows.FlowLogic @@ -353,6 +355,17 @@ class CordaRPCOpsImplTest { } } + @Test(timeout=300_000) + fun `trying to open attachment which doesnt exist throws error`() { + CURRENT_RPC_CONTEXT.set(RpcAuthContext(InvocationContext.rpc(testActor()), buildSubject("TEST_USER", emptySet()))) + withPermissions(invokeRpc(CordaRPCOps::openAttachment)) { + assertThatThrownBy { + rpc.openAttachment(SecureHash.zeroHash) + }.isInstanceOf(RPCException::class.java) + .withFailMessage("Unable to open attachment with id: ${SecureHash.zeroHash}") + } + } + @Test(timeout=300_000) fun `attachment uploaded with metadata has specified uploader`() { CURRENT_RPC_CONTEXT.set(RpcAuthContext(InvocationContext.rpc(testActor()), buildSubject("TEST_USER", emptySet()))) diff --git a/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt b/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt index e6655efd90..0fb0382234 100644 --- a/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt +++ b/node/src/test/kotlin/net/corda/node/internal/AbstractNodeTests.kt @@ -6,7 +6,6 @@ import net.corda.core.internal.concurrent.transpose import net.corda.core.utilities.contextLogger import net.corda.core.utilities.getOrThrow import net.corda.nodeapi.internal.persistence.DatabaseConfig -import net.corda.testing.common.internal.relaxedThoroughness import net.corda.testing.internal.configureDatabase import net.corda.testing.node.internal.ProcessUtilities.startJavaProcess import org.junit.Rule @@ -43,8 +42,7 @@ class AbstractNodeTests { @Test(timeout=300_000) fun `H2 fix is applied`() { val pool = Executors.newFixedThreadPool(5) - val runs = if (relaxedThoroughness) 1 else 100 - (0 until runs).map { + (0 until 5).map { // Four "nodes" seems to be the magic number to reproduce the problem on CI: val urls = (0 until 4).map { freshURL() } // Haven't been able to reproduce in a warm JVM: diff --git a/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt b/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt index 119729ac76..5433f96758 100644 --- a/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt +++ b/node/src/test/kotlin/net/corda/node/internal/NodeStartupCliTest.kt @@ -22,11 +22,13 @@ class NodeStartupCliTest { companion object { private lateinit var workingDirectory: Path - + private lateinit var rootDirectory: Path + private var customNodeConf = "custom_node.conf" @BeforeClass @JvmStatic fun initDirectories() { workingDirectory = Paths.get(".").normalize().toAbsolutePath() + rootDirectory = Paths.get("/").normalize().toAbsolutePath() } } @@ -56,6 +58,18 @@ class NodeStartupCliTest { Assertions.assertThat(startup.cmdLineOptions.networkRootTrustStorePathParameter).isEqualTo(null) } + @Test(timeout=300_000) + fun `--nodeconf using relative path will be changed to absolute path`() { + CommandLine.populateCommand(startup, CommonCliConstants.CONFIG_FILE, customNodeConf) + Assertions.assertThat(startup.cmdLineOptions.configFile).isEqualTo(workingDirectory / customNodeConf) + } + + @Test(timeout=300_000) + fun `--nodeconf using absolute path will not be changed`() { + CommandLine.populateCommand(startup, CommonCliConstants.CONFIG_FILE, (rootDirectory / customNodeConf).toString()) + Assertions.assertThat(startup.cmdLineOptions.configFile).isEqualTo(rootDirectory / customNodeConf) + } + @Test(timeout=3_000) @Ignore fun `test logs are written to correct location correctly if verbose flag set`() { diff --git a/node/src/test/kotlin/net/corda/node/internal/NodeStartupTest.kt b/node/src/test/kotlin/net/corda/node/internal/NodeStartupTest.kt index ccaa925268..da793ee97c 100644 --- a/node/src/test/kotlin/net/corda/node/internal/NodeStartupTest.kt +++ b/node/src/test/kotlin/net/corda/node/internal/NodeStartupTest.kt @@ -1,9 +1,9 @@ package net.corda.node.internal -import com.google.common.io.Files import org.assertj.core.api.Assertions.assertThat import org.junit.Test import java.nio.channels.OverlappingFileLockException +import java.nio.file.Files import java.util.concurrent.CountDownLatch import kotlin.concurrent.thread import kotlin.test.assertFailsWith @@ -11,8 +11,7 @@ import kotlin.test.assertFailsWith class NodeStartupTest { @Test(timeout=300_000) fun `test that you cant start two nodes in the same directory`() { - val dir = Files.createTempDir().toPath() - + val dir = Files.createTempDirectory("node_startup_test") val latch = CountDownLatch(1) thread(start = true) { diff --git a/node/src/test/kotlin/net/corda/node/internal/artemis/UserValidationPluginTest.kt b/node/src/test/kotlin/net/corda/node/internal/artemis/UserValidationPluginTest.kt new file mode 100644 index 0000000000..3f316258e7 --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/internal/artemis/UserValidationPluginTest.kt @@ -0,0 +1,89 @@ +package net.corda.node.internal.artemis + +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import net.corda.coretesting.internal.rigorousMock +import net.corda.nodeapi.internal.ArtemisMessagingComponent +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import org.apache.activemq.artemis.api.core.ActiveMQSecurityException +import org.apache.activemq.artemis.api.core.SimpleString +import org.apache.activemq.artemis.core.client.impl.ClientMessageImpl +import org.apache.activemq.artemis.core.server.ServerSession +import org.apache.activemq.artemis.protocol.amqp.broker.AMQPMessage +import org.apache.activemq.artemis.protocol.amqp.converter.AMQPConverter +import org.assertj.core.api.Assertions +import org.junit.Test + +class UserValidationPluginTest { + private val plugin = UserValidationPlugin() + private val coreMessage = ClientMessageImpl(0, false, 0, System.currentTimeMillis(), 4.toByte(), 1024) + private val amqpMessage get() = AMQPConverter.getInstance().fromCore(coreMessage) + private val session = rigorousMock().also { + doReturn(ArtemisMessagingComponent.PEER_USER).whenever(it).username + doReturn(ALICE_NAME.toString()).whenever(it).validatedUser + } + + @Test(timeout = 300_000) + fun `accept AMQP message without user`() { + plugin.beforeSend(session, rigorousMock(), amqpMessage, direct = false, noAutoCreateQueue = false) + } + + @Test(timeout = 300_000) + fun `accept AMQP message with user`() { + coreMessage.putStringProperty("_AMQ_VALIDATED_USER", ALICE_NAME.toString()) + plugin.beforeSend(session, rigorousMock(), amqpMessage, direct = false, noAutoCreateQueue = false) + } + + @Test(timeout = 300_000) + fun `reject AMQP message with different user`() { + coreMessage.putStringProperty("_AMQ_VALIDATED_USER", BOB_NAME.toString()) + Assertions.assertThatExceptionOfType(ActiveMQSecurityException::class.java).isThrownBy { + plugin.beforeSend(session, rigorousMock(), amqpMessage, direct = false, noAutoCreateQueue = false) + }.withMessageContaining("_AMQ_VALIDATED_USER") + } + + @Test(timeout = 300_000) + fun `accept AMQP message with different user on internal session`() { + val internalSession = rigorousMock().also { + doReturn(ArtemisMessagingComponent.NODE_P2P_USER).whenever(it).username + doReturn(ALICE_NAME.toString()).whenever(it).validatedUser + } + coreMessage.putStringProperty("_AMQ_VALIDATED_USER", BOB_NAME.toString()) + plugin.beforeSend(internalSession, rigorousMock(), amqpMessage, direct = false, noAutoCreateQueue = false) + } + + @Test(timeout = 300_000) + fun `reject core message`() { + Assertions.assertThatExceptionOfType(ActiveMQSecurityException::class.java).isThrownBy { + plugin.beforeSend(session, rigorousMock(), coreMessage, direct = false, noAutoCreateQueue = false) + }.withMessageContaining("message type") + } + + @Test(timeout = 300_000) + fun `reject message with exception`() { + coreMessage.putStringProperty("_AMQ_VALIDATED_USER", BOB_NAME.toString()) + val messageWithException = object : AMQPMessage(0, amqpMessage.buffer.array(), null) { + override fun getStringProperty(key: SimpleString?): String { + throw IllegalStateException("My exception") + } + } + // Artemis swallows all exceptions except ActiveMQException, so making sure that proper exception is thrown + Assertions.assertThatExceptionOfType(ActiveMQSecurityException::class.java).isThrownBy { + plugin.beforeSend(session, rigorousMock(), messageWithException, direct = false, noAutoCreateQueue = false) + }.withMessageContaining("Message validation failed") + } + + @Test(timeout = 300_000) + fun `reject message with security exception`() { + coreMessage.putStringProperty("_AMQ_VALIDATED_USER", BOB_NAME.toString()) + val messageWithException = object : AMQPMessage(0, amqpMessage.buffer.array(), null) { + override fun getStringProperty(key: SimpleString?): String { + throw ActiveMQSecurityException("My security exception") + } + } + Assertions.assertThatExceptionOfType(ActiveMQSecurityException::class.java).isThrownBy { + plugin.beforeSend(session, rigorousMock(), messageWithException, direct = false, noAutoCreateQueue = false) + }.withMessageContaining("My security exception") + } +} \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt index 594932f5c0..b6bb0817b2 100644 --- a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt @@ -50,13 +50,13 @@ class FinalityHandlerTest { getOrThrow() } - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() bob = mockNet.restartNode(bob) // Since we've not done anything to fix the orignal error, we expect the finality handler to be sent to the hospital // again on restart - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() } @@ -96,7 +96,7 @@ class FinalityHandlerTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -104,7 +104,7 @@ class FinalityHandlerTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } private fun TestStartedNode.getTransaction(id: SecureHash): SignedTransaction? { diff --git a/node/src/test/kotlin/net/corda/node/services/config/ConfigHelperTests.kt b/node/src/test/kotlin/net/corda/node/services/config/ConfigHelperTests.kt new file mode 100644 index 0000000000..cd378de825 --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/services/config/ConfigHelperTests.kt @@ -0,0 +1,83 @@ +package net.corda.node.services.config + +import com.typesafe.config.Config +import com.typesafe.config.ConfigFactory +import net.corda.core.internal.delete +import net.corda.core.internal.div +import org.junit.After +import org.junit.Assert +import org.junit.Before +import org.junit.Test +import java.nio.file.Files +import java.nio.file.Path + +class ConfigHelperTests { + private var baseDir: Path? = null + + @Before + fun setup() { + baseDir = Files.createTempDirectory("corda_config") + } + + @After + fun cleanup() { + baseDir?.delete() + } + + @Test(timeout = 300_000) + fun `config is overridden by underscore variable`() { + val sshPort: Long = 9000 + + // Verify the port isn't set when not provided + var config = loadConfig() + Assert.assertFalse("SSH port should not be configured when not provided", config!!.hasPath("sshd.port")) + + config = loadConfig("corda_sshd_port" to sshPort) + Assert.assertEquals(sshPort, config?.getLong("sshd.port")) + } + + @Test(timeout = 300_000) + fun `config is overridden by case insensitive underscore variable`() { + val sshPort: Long = 10000 + val config = loadConfig("CORDA_sshd_port" to sshPort) + Assert.assertEquals(sshPort, config?.getLong("sshd.port")) + } + + @Test(timeout = 300_000) + fun `config is overridden by case insensitive dot variable`() { + val sshPort: Long = 11000 + val config = loadConfig("CORDA.sshd.port" to sshPort, + "corda.devMode" to true.toString()) + Assert.assertEquals(sshPort, config?.getLong("sshd.port")) + } + + @Test(timeout = 300_000, expected = ShadowingException::class) + fun `shadowing is forbidden`() { + val sshPort: Long = 12000 + loadConfig("CORDA_sshd_port" to sshPort.toString(), + "corda.sshd.port" to sshPort.toString()) + } + + /** + * Load the node configuration with the given environment variable + * overrides. + * + * @param environmentVariables pairs of keys and values for environment variables + * to simulate when loading the configuration. + */ + @Suppress("SpreadOperator") + private fun loadConfig(vararg environmentVariables: Pair): Config? { + return baseDir?.let { + ConfigHelper.loadConfig( + baseDirectory = it, + configFile = it / ConfigHelper.DEFAULT_CONFIG_FILENAME, + allowMissingConfig = true, + configOverrides = ConfigFactory.empty(), + rawSystemOverrides = ConfigFactory.empty(), + rawEnvironmentOverrides = ConfigFactory.empty().plus( + mapOf(*environmentVariables) + ) + ) + } + } +} diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt index a75960523b..3c526e4a9d 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/DBCheckpointStorageTests.kt @@ -872,6 +872,24 @@ class DBCheckpointStorageTests { } } + @Test(timeout = 300_000) + fun `update only the flow status`() { + val (id, checkpoint) = newCheckpoint() + val serializedFlowState = checkpoint.serializeFlowState() + database.transaction { + checkpointStorage.addCheckpoint(id, checkpoint, serializedFlowState, checkpoint.serializeCheckpointState()) + } + database.transaction { + checkpointStorage.updateStatus(id, Checkpoint.FlowStatus.HOSPITALIZED) + } + database.transaction { + assertEquals( + checkpoint.copy(status = Checkpoint.FlowStatus.HOSPITALIZED), + checkpointStorage.checkpoints().single().deserialize() + ) + } + } + data class IdAndCheckpoint(val id: StateMachineRunId, val checkpoint: Checkpoint) private fun changeStatus(oldCheckpoint: Checkpoint, status: Checkpoint.FlowStatus): IdAndCheckpoint { diff --git a/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt b/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt index 320137e8b4..f2f141cac5 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt @@ -416,7 +416,7 @@ class NodeVaultServiceTest { } val softLockId = UUID.randomUUID() - val lockCount = NodeVaultService.MAX_SQL_IN_CLAUSE_SET * 2 + val lockCount = NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE * 2 database.transaction { assertEquals(100, queryStates(SoftLockingType.UNLOCKED_ONLY).size) val unconsumedStates = vaultService.queryBy().states @@ -429,18 +429,18 @@ class NodeVaultServiceTest { assertEquals(lockCount, queryStates(SoftLockingType.LOCKED_ONLY).size) val unlockSet0 = mutableSetOf() - for (i in 0 until NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 1) { + for (i in 0 until NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE + 1) { unlockSet0.add(lockSet[i]) } vaultService.softLockRelease(softLockId, NonEmptySet.copyOf(unlockSet0)) - assertEquals(NodeVaultService.MAX_SQL_IN_CLAUSE_SET - 1, queryStates(SoftLockingType.LOCKED_ONLY).size) + assertEquals(NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE - 1, queryStates(SoftLockingType.LOCKED_ONLY).size) val unlockSet1 = mutableSetOf() - for (i in NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 1 until NodeVaultService.MAX_SQL_IN_CLAUSE_SET + 3) { + for (i in NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE + 1 until NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE + 3) { unlockSet1.add(lockSet[i]) } vaultService.softLockRelease(softLockId, NonEmptySet.copyOf(unlockSet1)) - assertEquals(NodeVaultService.MAX_SQL_IN_CLAUSE_SET - 1 - 2, queryStates(SoftLockingType.LOCKED_ONLY).size) + assertEquals(NodeVaultService.DEFAULT_SOFT_LOCKING_SQL_IN_CLAUSE_SIZE - 1 - 2, queryStates(SoftLockingType.LOCKED_ONLY).size) vaultService.softLockRelease(softLockId) // release the rest assertEquals(100, queryStates(SoftLockingType.UNLOCKED_ONLY).size) diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index f66e903bc4..0eeb7939fe 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -501,7 +501,8 @@ abstract class VaultQueryTestsBase : VaultQueryParties { assertThat(queriedStates).containsExactlyElementsOf(allStates) } } - + + @Ignore @Test(timeout=300_000) fun `query with sort criteria and pagination on large volume of states should complete in time`() { val numberOfStates = 1000 diff --git a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt index dde5082f6b..b17b437fad 100644 --- a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt +++ b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt @@ -28,6 +28,8 @@ import net.corda.testing.core.ALICE_NAME import net.corda.testing.internal.createDevIntermediateCaCertPath import net.corda.coretesting.internal.rigorousMock import net.corda.coretesting.internal.stubs.CertificateStoreStubs +import net.corda.node.services.config.NotaryConfig +import net.corda.testing.core.DUMMY_NOTARY_NAME import org.assertj.core.api.Assertions.* import org.bouncycastle.asn1.x509.GeneralName import org.bouncycastle.asn1.x509.GeneralSubtree @@ -37,6 +39,7 @@ import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest import org.junit.After import org.junit.Before import org.junit.Test +import java.lang.IllegalStateException import java.nio.file.Files import java.security.PublicKey import java.security.cert.CertPathValidatorException @@ -71,6 +74,7 @@ class NetworkRegistrationHelperTest { doReturn(null).whenever(it).tlsCertCrlDistPoint doReturn(null).whenever(it).tlsCertCrlIssuer doReturn(true).whenever(it).crlCheckSoftFail + doReturn(null).whenever(it).notary } } @@ -120,7 +124,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `missing truststore`() { - val nodeCaCertPath = createNodeCaCertPath() + val nodeCaCertPath = createCertPath() assertThatThrownBy { createFixedResponseRegistrationHelper(nodeCaCertPath) }.hasMessageContaining("This file must contain the root CA cert of your compatibility zone. Please contact your CZ operator.") @@ -128,7 +132,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `node CA with incorrect cert role`() { - val nodeCaCertPath = createNodeCaCertPath(type = CertificateType.TLS) + val nodeCaCertPath = createCertPath(type = CertificateType.TLS) saveNetworkTrustStore(CORDA_ROOT_CA to nodeCaCertPath.last()) val registrationHelper = createFixedResponseRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) @@ -139,7 +143,7 @@ class NetworkRegistrationHelperTest { @Test(timeout=300_000) fun `node CA with incorrect subject`() { val invalidName = CordaX500Name("Foo", "MU", "GB") - val nodeCaCertPath = createNodeCaCertPath(legalName = invalidName) + val nodeCaCertPath = createCertPath(legalName = invalidName) saveNetworkTrustStore(CORDA_ROOT_CA to nodeCaCertPath.last()) val registrationHelper = createFixedResponseRegistrationHelper(nodeCaCertPath) assertThatExceptionOfType(CertificateRequestException::class.java) @@ -220,36 +224,118 @@ class NetworkRegistrationHelperTest { createRegistrationHelper(rootAndIntermediateCA = rootAndIntermediateCA).generateKeysAndRegister() } - private fun createNodeCaCertPath(type: CertificateType = CertificateType.NODE_CA, - legalName: CordaX500Name = nodeLegalName, - publicKey: PublicKey = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME).public, - rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()): List { + @Test(timeout=300_000) + fun `successful registration for notary node`() { + val notaryServiceLegalName = DUMMY_NOTARY_NAME + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = notaryServiceLegalName) + assertThat(notaryNodeConfig.notary).isNotNull + + val rootAndIntermediateCA = createDevIntermediateCaCertPath().also { + saveNetworkTrustStore(CORDA_ROOT_CA to it.first.certificate) + } + + // Mock out the registration service to ensure notary service registration is handled correctly + createRegistrationHelper(CertRole.NODE_CA, notaryNodeConfig) { + when { + it.subject == nodeLegalName.toX500Name() -> { + val certType = CertificateType.values().first { it.role == CertRole.NODE_CA } + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) + } + it.subject == notaryServiceLegalName.toX500Name() -> { + val certType = CertificateType.values().first { it.role == CertRole.SERVICE_IDENTITY } + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType, legalName = notaryServiceLegalName) + } + else -> throw IllegalStateException("Unknown CSR") + } + }.generateKeysAndRegister() + + val nodeKeystore = config.signingCertificateStore.get() + + nodeKeystore.run { + assertFalse(contains(X509Utilities.CORDA_INTERMEDIATE_CA)) + assertFalse(contains(CORDA_ROOT_CA)) + assertFalse(contains(X509Utilities.CORDA_CLIENT_TLS)) + assertThat(CertRole.extract(this[X509Utilities.CORDA_CLIENT_CA])).isEqualTo(CertRole.NODE_CA) + assertThat(CertRole.extract(this[DISTRIBUTED_NOTARY_KEY_ALIAS])).isEqualTo(CertRole.SERVICE_IDENTITY) + } + } + + @Test(timeout=300_000) + fun `notary registration fails when no separate notary service identity configured`() { + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = null) + assertThat(notaryNodeConfig.notary).isNotNull + + assertThatThrownBy { + createRegistrationHelper(nodeConfig = notaryNodeConfig) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("notary service legal name must be provided") + } + + @Test(timeout=300_000) + fun `notary registration fails when notary service identity configured with same legal name as node`() { + val notaryNodeConfig = createNotaryNodeConfiguration(notaryServiceLegalName = config.myLegalName) + assertThat(notaryNodeConfig.notary).isNotNull + + assertThatThrownBy { + createRegistrationHelper(nodeConfig = notaryNodeConfig) + }.isInstanceOf(IllegalArgumentException::class.java) + .hasMessageContaining("notary service legal name must be different from the node") + } + + private fun createNotaryNodeConfiguration(notaryServiceLegalName: CordaX500Name?): NodeConfiguration { + return rigorousMock().also { + doReturn(config.baseDirectory).whenever(it).baseDirectory + doReturn(config.certificatesDirectory).whenever(it).certificatesDirectory + doReturn(CertificateStoreStubs.P2P.withCertificatesDirectory(config.certificatesDirectory)).whenever(it).p2pSslOptions + doReturn(CertificateStoreStubs.Signing.withCertificatesDirectory(config.certificatesDirectory)).whenever(it) + .signingCertificateStore + doReturn(nodeLegalName).whenever(it).myLegalName + doReturn("").whenever(it).emailAddress + doReturn(null).whenever(it).tlsCertCrlDistPoint + doReturn(null).whenever(it).tlsCertCrlIssuer + doReturn(true).whenever(it).crlCheckSoftFail + doReturn(NotaryConfig(validating = false, serviceLegalName = notaryServiceLegalName)).whenever(it).notary + } + } + + private fun createCertPath(type: CertificateType = CertificateType.NODE_CA, + legalName: CordaX500Name = nodeLegalName, + publicKey: PublicKey = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME).public, + rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()): List { val (rootCa, intermediateCa) = rootAndIntermediateCA val nameConstraints = if (type == CertificateType.NODE_CA) { NameConstraints(arrayOf(GeneralSubtree(GeneralName(GeneralName.directoryName, legalName.toX500Name()))), arrayOf()) } else { null } - val nodeCaCert = X509Utilities.createCertificate( + val cert = X509Utilities.createCertificate( type, intermediateCa.certificate, intermediateCa.keyPair, legalName.x500Principal, publicKey, nameConstraints = nameConstraints) - return listOf(nodeCaCert, intermediateCa.certificate, rootCa.certificate) + return listOf(cert, intermediateCa.certificate, rootCa.certificate) } private fun createFixedResponseRegistrationHelper(response: List, certRole: CertRole = CertRole.NODE_CA): NetworkRegistrationHelper { return createRegistrationHelper(certRole) { response } } - private fun createRegistrationHelper(certRole: CertRole = CertRole.NODE_CA, rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath()) = createRegistrationHelper(certRole) { + private fun createRegistrationHelper( + certRole: CertRole = CertRole.NODE_CA, + rootAndIntermediateCA: Pair = createDevIntermediateCaCertPath(), + nodeConfig: NodeConfiguration = config + ) = createRegistrationHelper(certRole, nodeConfig) { val certType = CertificateType.values().first { it.role == certRole } - createNodeCaCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) + createCertPath(rootAndIntermediateCA = rootAndIntermediateCA, publicKey = it.publicKey, type = certType) } - private fun createRegistrationHelper(certRole: CertRole = CertRole.NODE_CA, dynamicResponse: (JcaPKCS10CertificationRequest) -> List): NetworkRegistrationHelper { + private fun createRegistrationHelper( + certRole: CertRole = CertRole.NODE_CA, + nodeConfig: NodeConfiguration = config, + dynamicResponse: (JcaPKCS10CertificationRequest) -> List + ): NetworkRegistrationHelper { val certService = rigorousMock().also { val requests = mutableMapOf() doAnswer { @@ -265,11 +351,11 @@ class NetworkRegistrationHelperTest { } return when (certRole) { - CertRole.NODE_CA -> NodeRegistrationHelper(NodeRegistrationConfiguration(config), certService, NodeRegistrationOption(config.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword)) + CertRole.NODE_CA -> NodeRegistrationHelper(NodeRegistrationConfiguration(nodeConfig), certService, NodeRegistrationOption(nodeConfig.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword)) CertRole.SERVICE_IDENTITY -> NetworkRegistrationHelper( - NodeRegistrationConfiguration(config), + NodeRegistrationConfiguration(nodeConfig), certService, - config.certificatesDirectory / networkRootTrustStoreFileName, + nodeConfig.certificatesDirectory / networkRootTrustStoreFileName, networkRootTrustStorePassword, DISTRIBUTED_NOTARY_KEY_ALIAS, CertRole.SERVICE_IDENTITY) diff --git a/samples/attachment-demo/build.gradle b/samples/attachment-demo/build.gradle index 70dfe3c5d8..c3aeafa1b9 100644 --- a/samples/attachment-demo/build.gradle +++ b/samples/attachment-demo/build.gradle @@ -93,8 +93,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating: true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 cordapps = [] rpcUsers = ext.rpcUsers diff --git a/samples/bank-of-corda-demo/build.gradle b/samples/bank-of-corda-demo/build.gradle index 18ac7b21f6..aa377a6cee 100644 --- a/samples/bank-of-corda-demo/build.gradle +++ b/samples/bank-of-corda-demo/build.gradle @@ -51,8 +51,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating: true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10003" diff --git a/samples/cordapp-configuration/build.gradle b/samples/cordapp-configuration/build.gradle index e0cbe8afb2..4c57c1b941 100644 --- a/samples/cordapp-configuration/build.gradle +++ b/samples/cordapp-configuration/build.gradle @@ -28,8 +28,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { port 10003 diff --git a/samples/irs-demo/cordapp/build.gradle b/samples/irs-demo/cordapp/build.gradle index 8a0df29aef..e46aea2330 100644 --- a/samples/irs-demo/cordapp/build.gradle +++ b/samples/irs-demo/cordapp/build.gradle @@ -63,8 +63,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address("localhost:10003") @@ -122,7 +124,9 @@ task prepareDockerNodes(type: net.corda.plugins.Dockerform, dependsOn: ['jar', n } node { name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] cordapps = ["${project(":finance").group}:contracts:$corda_release_version", "${project(":finance").group}:workflows:$corda_release_version"] rpcUsers = rpcUsersList useTestClock true @@ -154,7 +158,7 @@ task integrationTest(type: Test, dependsOn: []) { // This fixes the "line too long" error when running this demo with windows CLI // TODO: Automatically apply to all projects via a plugin -tasks.withType(CreateStartScripts).each { task -> +tasks.withType(CreateStartScripts).configureEach { task -> task.doLast { String text = task.windowsScript.text // Replaces the per file classpath (which are all jars in "lib") with a wildcard on lib diff --git a/samples/network-verifier/build.gradle b/samples/network-verifier/build.gradle index 83ff64cf24..ab6d4d37a5 100644 --- a/samples/network-verifier/build.gradle +++ b/samples/network-verifier/build.gradle @@ -39,8 +39,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : false] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : false, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { port 10003 diff --git a/samples/notary-demo/build.gradle b/samples/notary-demo/build.gradle index 3c1280de1e..1cd9710a65 100644 --- a/samples/notary-demo/build.gradle +++ b/samples/notary-demo/build.gradle @@ -56,13 +56,15 @@ task deployNodesSingle(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { rpcUsers = [[user: "demou", password: "demop", permissions: ["ALL"]]] } node { - name "O=Notary Service,L=Zurich,C=CH" + name "O=Notary Node,L=Zurich,C=CH" p2pPort 10009 rpcSettings { address "localhost:10010" adminAddress "localhost:10110" } - notary = [validating: true] + notary = [validating: true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] } } @@ -86,7 +88,7 @@ task deployNodesCustom(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { rpcUsers = [[user: "demou", password: "demop", permissions: ["ALL"]]] } node { - name "O=Notary Service,L=Zurich,C=CH" + name "O=Notary Node,L=Zurich,C=CH" p2pPort 10009 rpcSettings { address "localhost:10010" @@ -94,7 +96,8 @@ task deployNodesCustom(type: Cordform, dependsOn: ['jar', nodeTask, webTask]) { } notary = [ validating: true, - className: "net.corda.notarydemo.MyCustomValidatingNotaryService" + className: "net.corda.notarydemo.MyCustomValidatingNotaryService", + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" ] } } diff --git a/samples/simm-valuation-demo/build.gradle b/samples/simm-valuation-demo/build.gradle index 9736c3a998..7f2f3e17bc 100644 --- a/samples/simm-valuation-demo/build.gradle +++ b/samples/simm-valuation-demo/build.gradle @@ -94,8 +94,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask, runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10014" diff --git a/samples/trader-demo/build.gradle b/samples/trader-demo/build.gradle index fa498ddcfe..dba40ec2c6 100644 --- a/samples/trader-demo/build.gradle +++ b/samples/trader-demo/build.gradle @@ -84,8 +84,10 @@ task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar', nodeTask]) runSchemaMigration = true } node { - name "O=Notary Service,L=Zurich,C=CH" - notary = [validating : true] + name "O=Notary Node,L=Zurich,C=CH" + notary = [validating : true, + serviceLegalName: "O=Notary Service,L=Zurich,C=CH" + ] p2pPort 10002 rpcSettings { address "localhost:10003" diff --git a/serialization-deterministic/README.md b/serialization-deterministic/README.md new file mode 100644 index 0000000000..abd4a19f0c --- /dev/null +++ b/serialization-deterministic/README.md @@ -0,0 +1,2 @@ +## corda-serialization-deterministic. +This artifact is a deterministic subset of the binary contents of `corda-serialization`. diff --git a/serialization-deterministic/build.gradle b/serialization-deterministic/build.gradle index 773522460d..6ad42b0208 100644 --- a/serialization-deterministic/build.gradle +++ b/serialization-deterministic/build.gradle @@ -50,8 +50,8 @@ tasks.named('jar', Jar) { enabled = false } -def serializationJarTask = tasks.getByPath(':serialization:jar') -def originalJar = serializationJarTask.outputs.files.singleFile +def serializationJarTask = project(':serialization').tasks.named('jar', Jar) +def originalJar = serializationJarTask.map { it.outputs.files.singleFile } def patchSerialization = tasks.register('patchSerialization', Zip) { dependsOn serializationJarTask @@ -77,7 +77,7 @@ def patchSerialization = tasks.register('patchSerialization', Zip) { } def predeterminise = tasks.register('predeterminise', ProGuardTask) { - dependsOn project(':core-deterministic').assemble + dependsOn project(':core-deterministic').tasks.named('assemble') injars patchSerialization outjars file("$buildDir/proguard/pre-deterministic-${project.version}.jar") @@ -125,7 +125,7 @@ def jarFilter = tasks.register('jarFilter', JarFilterTask) { } } -task determinise(type: ProGuardTask) { +def determinise = tasks.register('determinise', ProGuardTask) { injars jarFilter outjars file("$buildDir/proguard/$jarBaseName-${project.version}.jar") @@ -154,16 +154,19 @@ task determinise(type: ProGuardTask) { keepclassmembers 'class net.corda.serialization.** { public synthetic ; }' } -task metafix(type: MetaFixerTask) { +def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) + +def metafix = tasks.register('metafix', MetaFixerTask) { outputDir file("$buildDir/libs") jars determinise suffix "" // Strip timestamps from the JAR to make it reproducible. preserveTimestamps = false + finalizedBy checkDeterminism } -def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { +checkDeterminism.configure { dependsOn jdkTask injars metafix @@ -183,22 +186,33 @@ def checkDeterminism = tasks.register('checkDeterminism', ProGuardTask) { } defaultTasks "determinise" -determinise.finalizedBy metafix -metafix.finalizedBy checkDeterminism -assemble.dependsOn checkDeterminism +determinise.configure { + finalizedBy metafix +} +tasks.named('assemble') { + dependsOn checkDeterminism +} -def deterministicJar = metafix.outputs.files.singleFile +def deterministicJar = metafix.map { it.outputs.files.singleFile } artifacts { - deterministicArtifacts file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix - publish file: deterministicJar, name: jarBaseName, type: 'jar', extension: 'jar', builtBy: metafix + deterministicArtifacts deterministicJar + publish deterministicJar +} + +tasks.named('sourceJar', Jar) { + from 'README.md' + include 'README.md' +} + +tasks.named('javadocJar', Jar) { + from 'README.md' + include 'README.md' } publish { dependenciesFrom(configurations.deterministicArtifacts) { defaultScope = 'compile' } - publishSources = false - publishJavadoc = false name jarBaseName } diff --git a/serialization-djvm/build.gradle b/serialization-djvm/build.gradle index 7ec41f9d32..f51557e2a3 100644 --- a/serialization-djvm/build.gradle +++ b/serialization-djvm/build.gradle @@ -56,7 +56,7 @@ jar { } } -tasks.withType(Test) { +tasks.withType(Test).configureEach { useJUnitPlatform() systemProperty 'deterministic-rt.path', configurations.jdkRt.asPath systemProperty 'sandbox-libraries.path', configurations.sandboxTesting.asPath diff --git a/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/SandboxSerializerFactoryFactory.kt b/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/SandboxSerializerFactoryFactory.kt index bc614f2f95..96f8c44a03 100644 --- a/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/SandboxSerializerFactoryFactory.kt +++ b/serialization-djvm/src/main/kotlin/net/corda/serialization/djvm/SandboxSerializerFactoryFactory.kt @@ -75,7 +75,7 @@ class SandboxSerializerFactoryFactory( ) ) - val fingerPrinter = TypeModellingFingerPrinter(customSerializerRegistry) + val fingerPrinter = TypeModellingFingerPrinter(customSerializerRegistry, classLoader) val localSerializerFactory = DefaultLocalSerializerFactory( whitelist = context.whitelist, diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt index 9cf064a70b..9b0ce7b9ae 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt @@ -7,7 +7,6 @@ import net.corda.core.utilities.debug import net.corda.core.utilities.trace import net.corda.serialization.internal.model.* import net.corda.serialization.internal.model.TypeIdentifier.* -import net.corda.serialization.internal.model.TypeIdentifier.Companion.classLoaderFor import org.apache.qpid.proton.amqp.Symbol import java.lang.reflect.ParameterizedType import java.lang.reflect.Type @@ -161,7 +160,7 @@ class DefaultLocalSerializerFactory( val declaredGenericType = if (declaredType !is ParameterizedType && localTypeInformation.typeIdentifier is Parameterised && declaredClass != Class::class.java) { - localTypeInformation.typeIdentifier.getLocalType(classLoaderFor(declaredClass)) + localTypeInformation.typeIdentifier.getLocalType(classloader) } else { declaredType } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index 27650621d7..e1d0aaee77 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -101,7 +101,7 @@ object SerializerFactoryBuilder { val localTypeModel = ConfigurableLocalTypeModel(typeModelConfiguration) val fingerPrinter = overrideFingerPrinter ?: - TypeModellingFingerPrinter(customSerializerRegistry) + TypeModellingFingerPrinter(customSerializerRegistry, classCarpenter.classloader) val localSerializerFactory = DefaultLocalSerializerFactory( whitelist, diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt index 2697b107a8..3477c02a48 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt @@ -45,12 +45,12 @@ sealed class TypeIdentifier { * Obtain a nicely-formatted representation of the identified type, for help with debugging. */ fun prettyPrint(simplifyClassNames: Boolean = true): String = when(this) { - is TypeIdentifier.UnknownType -> "?" - is TypeIdentifier.TopType -> "*" - is TypeIdentifier.Unparameterised -> name.simplifyClassNameIfRequired(simplifyClassNames) - is TypeIdentifier.Erased -> "${name.simplifyClassNameIfRequired(simplifyClassNames)} (erased)" - is TypeIdentifier.ArrayOf -> "${componentType.prettyPrint(simplifyClassNames)}[]" - is TypeIdentifier.Parameterised -> + is UnknownType -> "?" + is TopType -> "*" + is Unparameterised -> name.simplifyClassNameIfRequired(simplifyClassNames) + is Erased -> "${name.simplifyClassNameIfRequired(simplifyClassNames)} (erased)" + is ArrayOf -> "${componentType.prettyPrint(simplifyClassNames)}[]" + is Parameterised -> name.simplifyClassNameIfRequired(simplifyClassNames) + parameters.joinToString(", ", "<", ">") { it.prettyPrint(simplifyClassNames) } @@ -63,8 +63,6 @@ sealed class TypeIdentifier { // This method has locking. So we memo the value here. private val systemClassLoader: ClassLoader = ClassLoader.getSystemClassLoader() - fun classLoaderFor(clazz: Class<*>): ClassLoader = clazz.classLoader ?: systemClassLoader - /** * Obtain the [TypeIdentifier] for an erased Java class. * @@ -81,7 +79,7 @@ sealed class TypeIdentifier { * Obtain the [TypeIdentifier] for a Java [Type] (typically obtained by calling one of * [java.lang.reflect.Parameter.getAnnotatedType], * [java.lang.reflect.Field.getGenericType] or - * [java.lang.reflect.Method.getGenericReturnType]). Wildcard types and type variables are converted to [Unknown]. + * [java.lang.reflect.Method.getGenericReturnType]). Wildcard types and type variables are converted to [UnknownType]. * * @param type The [Type] to obtain a [TypeIdentifier] for. * @param resolutionContext Optionally, a [Type] which can be used to resolve type variables, for example a @@ -273,5 +271,5 @@ private class ReconstitutedParameterizedType( other.ownerType == ownerType && Arrays.equals(other.actualTypeArguments, actualTypeArguments) override fun hashCode(): Int = - Arrays.hashCode(actualTypeArguments) xor Objects.hashCode(ownerType) xor Objects.hashCode(rawType) + actualTypeArguments.contentHashCode() xor Objects.hashCode(ownerType) xor Objects.hashCode(rawType) } \ No newline at end of file diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt index c5d79ed41f..8965a5c8e1 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt @@ -5,7 +5,6 @@ import net.corda.core.utilities.contextLogger import net.corda.core.utilities.toBase64 import net.corda.serialization.internal.amqp.* import net.corda.serialization.internal.model.TypeIdentifier.* -import net.corda.serialization.internal.model.TypeIdentifier.Companion.classLoaderFor import java.lang.reflect.ParameterizedType /** @@ -31,6 +30,7 @@ interface FingerPrinter { */ class TypeModellingFingerPrinter( private val customTypeDescriptorLookup: CustomSerializerRegistry, + private val classLoader: ClassLoader, private val debugEnabled: Boolean = false) : FingerPrinter { private val cache: MutableMap = DefaultCacheProvider.createCache() @@ -42,7 +42,7 @@ class TypeModellingFingerPrinter( * the Fingerprinter cannot guarantee that. */ cache.getOrPut(typeInformation.typeIdentifier) { - FingerPrintingState(customTypeDescriptorLookup, FingerprintWriter(debugEnabled)) + FingerPrintingState(customTypeDescriptorLookup, classLoader, FingerprintWriter(debugEnabled)) .fingerprint(typeInformation) } } @@ -95,6 +95,7 @@ internal class FingerprintWriter(debugEnabled: Boolean = false) { */ private class FingerPrintingState( private val customSerializerRegistry: CustomSerializerRegistry, + private val classLoader: ClassLoader, private val writer: FingerprintWriter) { companion object { @@ -200,7 +201,7 @@ private class FingerPrintingState( private fun fingerprintName(type: LocalTypeInformation) { val identifier = type.typeIdentifier when (identifier) { - is TypeIdentifier.ArrayOf -> writer.write(identifier.componentType.name).writeArray() + is ArrayOf -> writer.write(identifier.componentType.name).writeArray() else -> writer.write(identifier.name) } } @@ -239,7 +240,7 @@ private class FingerPrintingState( val observedGenericType = if (observedType !is ParameterizedType && type.typeIdentifier is Parameterised && observedClass != Class::class.java) { - type.typeIdentifier.getLocalType(classLoaderFor(observedClass)) + type.typeIdentifier.getLocalType(classLoader) } else { observedType } @@ -259,6 +260,5 @@ private class FingerPrintingState( // and deserializing (assuming deserialization is occurring in a factory that didn't // serialise the object in the first place (and thus the cache lookup fails). This is also // true of Any, where we need Example and Example to have the same fingerprint - private fun hasSeen(type: TypeIdentifier) = (type in typesSeen) - && (type != TypeIdentifier.UnknownType) + private fun hasSeen(type: TypeIdentifier) = (type in typesSeen) && (type != UnknownType) } diff --git a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/TypeModellingFingerPrinterTests.kt b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/TypeModellingFingerPrinterTests.kt index 362972afc7..84c3a27e63 100644 --- a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/TypeModellingFingerPrinterTests.kt +++ b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/TypeModellingFingerPrinterTests.kt @@ -12,7 +12,7 @@ class TypeModellingFingerPrinterTests { val descriptorBasedSerializerRegistry = DefaultDescriptorBasedSerializerRegistry() val customRegistry = CachingCustomSerializerRegistry(descriptorBasedSerializerRegistry) - val fingerprinter = TypeModellingFingerPrinter(customRegistry, true) + val fingerprinter = TypeModellingFingerPrinter(customRegistry, ClassLoader.getSystemClassLoader(), true) // See https://r3-cev.atlassian.net/browse/CORDA-2266 @Test(timeout=300_000) diff --git a/settings.gradle b/settings.gradle index ae6dad0838..abcdd7c676 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,9 +2,27 @@ pluginManagement { ext.artifactory_contextUrl = 'https://software.r3.com/artifactory' repositories { - mavenLocal() - gradlePluginPortal() - maven { url "$artifactory_contextUrl/corda-dependencies" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenLocal() + gradlePluginPortal() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + } } } // The project is named 'corda-project' and not 'corda' because if this is named the same as the @@ -104,7 +122,6 @@ include 'core-deterministic:testing:data' include 'core-deterministic:testing:verifier' include 'serialization-deterministic' -include 'tools:checkpoint-agent' include 'detekt-plugins' include 'tools:error-tool' diff --git a/testing/DockerfileJDK11Azul b/testing/DockerfileJDK11Azul index 9c8042346f..655e49406d 100644 --- a/testing/DockerfileJDK11Azul +++ b/testing/DockerfileJDK11Azul @@ -1,4 +1,3 @@ FROM stefanotestingcr.azurecr.io/buildbase:11latest COPY . /tmp/source CMD cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info - diff --git a/testing/core-test-utils/src/main/kotlin/net/corda/testing/core/Expect.kt b/testing/core-test-utils/src/main/kotlin/net/corda/testing/core/Expect.kt index af961ba1a2..8a4d7d282a 100644 --- a/testing/core-test-utils/src/main/kotlin/net/corda/testing/core/Expect.kt +++ b/testing/core-test-utils/src/main/kotlin/net/corda/testing/core/Expect.kt @@ -164,14 +164,14 @@ fun S.genericExpectEvents( } val next = state.nextState(event) val expectedStates = state.getExpectedEvents() - log.info("$event :: ${expectedStates.map { it.simpleName }} -> ${next?.second?.getExpectedEvents()?.map { it.simpleName }}") + log.debug("$event :: ${expectedStates.map { it.simpleName }} -> ${next?.second?.getExpectedEvents()?.map { it.simpleName }}") if (next == null) { val message = "Got $event, did not match any expectations of type ${expectedStates.map { it.simpleName }}" if (isStrict) { finishFuture.setException(Exception(message)) state = ExpectComposeState.Finished() } else { - log.info("$message, discarding event as isStrict=false") + log.debug("$message, discarding event as isStrict=false") } } else { state = next.second diff --git a/testing/node-driver/README.md b/testing/node-driver/README.md new file mode 100644 index 0000000000..0ae850af61 --- /dev/null +++ b/testing/node-driver/README.md @@ -0,0 +1,2 @@ +## corda-node-driver. +This artifact is the node-driver used for testing Corda. diff --git a/testing/node-driver/build.gradle b/testing/node-driver/build.gradle index c632949b1a..7f3b3be7ee 100644 --- a/testing/node-driver/build.gradle +++ b/testing/node-driver/build.gradle @@ -73,9 +73,17 @@ jar { } } + +tasks.named('javadocJar', Jar) { + from 'README.md' + include 'README.md' +} + +tasks.named('javadoc', Javadoc) { + enabled = false +} + publish { - publishSources = true - publishJavadoc = false name jar.baseName } diff --git a/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java b/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java index a3ca4e8aef..7d97f33140 100644 --- a/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java +++ b/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java @@ -17,7 +17,7 @@ import java.nio.channels.FileChannel; * import sun.misc.Unsafe; * import sun.nio.ch.DirectBuffer; */ -class SharedMemoryIncremental extends PortAllocation { +public class SharedMemoryIncremental extends PortAllocation { static private final int DEFAULT_START_PORT = 10_000; static private final int FIRST_EPHEMERAL_PORT = 30_000; diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index 536d5f9c6f..339583a8a1 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -9,6 +9,7 @@ import com.typesafe.config.ConfigRenderOptions import com.typesafe.config.ConfigValue import com.typesafe.config.ConfigValueFactory import net.corda.client.rpc.CordaRPCClient +import net.corda.client.rpc.RPCException import net.corda.cliutils.CommonCliConstants.BASE_DIR import net.corda.core.concurrent.CordaFuture import net.corda.core.concurrent.firstOf @@ -226,16 +227,24 @@ class DriverDSLImpl( } } - private fun establishRpc(config: NodeConfig, processDeathFuture: CordaFuture): CordaFuture { + /** + * @param pollInterval the interval to wait between attempting to connect, if + * a connection attempt fails. + */ + private fun establishRpc(config: NodeConfig, + processDeathFuture: CordaFuture): CordaFuture { val rpcAddress = config.corda.rpcOptions.address val clientRpcSslOptions = clientSslOptionsCompatibleWith(config.corda.rpcOptions) val client = CordaRPCClient(rpcAddress, sslConfiguration = clientRpcSslOptions) - val connectionFuture = poll(executorService, "RPC connection") { + val connectionFuture = poll( + executorService = executorService, + pollName = "RPC connection", + pollInterval = RPC_CONNECT_POLL_INTERVAL) { try { config.corda.rpcUsers[0].run { client.start(username, password) } - } catch (e: Exception) { + } catch (e: RPCException) { if (processDeathFuture.isDone) throw e - log.info("Exception while connecting to RPC, retrying to connect at $rpcAddress", e) + log.info("Failed to connect to RPC at $rpcAddress") null } } @@ -698,6 +707,7 @@ class DriverDSLImpl( } ) val nodeFuture: CordaFuture = nodeAndThreadFuture.flatMap { (node, thread) -> + node.node.nodeReadyFuture.get() // Wait for the node to be ready before we connect to the node establishRpc(config, openFuture()).flatMap { rpc -> visibilityHandle.listen(rpc).map { InProcessImpl(rpc.nodeInfo(), rpc, config.corda, webAddress, useHTTPS, thread, onNodeExit, node) @@ -750,7 +760,7 @@ class DriverDSLImpl( val effectiveP2PAddress = config.corda.messagingServerAddress ?: config.corda.p2pAddress val p2pReadyFuture = nodeMustBeStartedFuture( executorService, - effectiveP2PAddress, + config.corda.baseDirectory / "net.corda.node.Corda.${identifier}.stdout.log", process ) { NodeListenProcessDeathException( @@ -810,6 +820,7 @@ class DriverDSLImpl( } companion object { + private val RPC_CONNECT_POLL_INTERVAL: Duration = 100.millis internal val log = contextLogger() // While starting with inProcess mode, we need to have different names to avoid clashes diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt index 1f4b3ab632..3c6de690bf 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalTestUtils.kt @@ -11,6 +11,7 @@ import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.concurrent.openFuture import net.corda.core.internal.div +import net.corda.core.internal.readText import net.corda.core.internal.times import net.corda.core.messaging.CordaRPCOps import net.corda.core.node.services.AttachmentFixup @@ -41,8 +42,10 @@ import rx.subjects.AsyncSubject import java.io.InputStream import java.net.Socket import java.net.SocketException +import java.nio.file.Path import java.sql.DriverManager import java.time.Duration +import java.time.Instant import java.util.concurrent.ScheduledExecutorService import java.util.concurrent.TimeUnit import java.util.jar.JarOutputStream @@ -79,6 +82,8 @@ val FINANCE_CORDAPPS: Set = setOf(FINANCE_CONTRACTS_CORDAPP, FI @JvmField val DUMMY_CONTRACTS_CORDAPP: CustomCordapp = cordappWithPackages("net.corda.testing.contracts") +private const val SECONDS_TO_WAIT_FOR_P2P: Long = 20 + fun cordappsForPackages(vararg packageNames: String): Set = cordappsForPackages(packageNames.asList()) fun cordappsForPackages(packageNames: Iterable): Set { @@ -172,20 +177,28 @@ fun addressMustBeBoundFuture(executorService: ScheduledExecutorService, hostAndP } fun nodeMustBeStartedFuture( - executorService: ScheduledExecutorService, - hostAndPort: NetworkHostAndPort, - listenProcess: Process? = null, - exception: () -> NodeListenProcessDeathException + executorService: ScheduledExecutorService, + logFile: Path, + listenProcess: Process, + exception: () -> NodeListenProcessDeathException ): CordaFuture { - return poll(executorService, "address $hostAndPort to bind") { - if (listenProcess != null && !listenProcess.isAlive) { + val stopPolling = Instant.now().plusSeconds(SECONDS_TO_WAIT_FOR_P2P) + return poll(executorService, "process $listenProcess is running") { + if (!listenProcess.isAlive) { throw exception() } - try { - Socket(hostAndPort.host, hostAndPort.port).close() - Unit - } catch (_exception: SocketException) { - null + when { + logFile.readText().contains("Running P2PMessaging loop") -> { + Unit + } + Instant.now().isAfter(stopPolling) -> { + // Waited for 20 seconds and the log file did not indicate that the PWP loop is running. + // This could be because the log is disabled, so lets try to create a client anyway. + Unit + } + else -> { + null + } } } } @@ -210,6 +223,10 @@ fun addressMustNotBeBoundFuture(executorService: ScheduledExecutorService, hostA } } +/** + * @param pollInterval the interval running the background task. + * @param warnCount number of iterations to poll before printing a warning message. + */ fun poll( executorService: ScheduledExecutorService, pollName: String, diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/NodeBasedTest.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/NodeBasedTest.kt index 8690310e4f..97a354c8fd 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/NodeBasedTest.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/NodeBasedTest.kt @@ -39,9 +39,10 @@ import kotlin.test.assertFalse // TODO Some of the logic here duplicates what's in the driver - the reason why it's not straightforward to replace it by // using DriverDSLImpl in `init()` and `stopAllNodes()` is because of the platform version passed to nodes (driver doesn't // support this, and it's a property of the Corda JAR) -abstract class NodeBasedTest -@JvmOverloads -constructor(private val cordappPackages: List = emptyList(), private val notaries: List = emptyList()) { +abstract class NodeBasedTest @JvmOverloads constructor( + private val cordappPackages: Set = emptySet(), + private val notaries: List = emptyList() +) { companion object { private val WHITESPACE = "\\s++".toRegex() } @@ -120,7 +121,11 @@ constructor(private val cordappPackages: List = emptyList(), private val ) + configOverrides ) - val customCordapps = cordappsForPackages(getCallerPackage(NodeBasedTest::class)?.let { cordappPackages + it } ?: cordappPackages) + val customCordapps = if (cordappPackages.isNotEmpty()) { + cordappPackages + } else { + cordappsForPackages(getCallerPackage(NodeBasedTest::class)?.let { listOf(it) } ?: emptyList()) + } TestCordappInternal.installCordapps(baseDirectory, emptySet(), customCordapps) val parsedConfig = config.parseAsNodeConfiguration().value() diff --git a/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt b/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt deleted file mode 100644 index 298eaa5660..0000000000 --- a/testing/test-common/src/main/kotlin/net/corda/testing/common/internal/Thoroughness.kt +++ /dev/null @@ -1,3 +0,0 @@ -package net.corda.testing.common.internal - -val relaxedThoroughness = System.getenv("TEAMCITY_PROJECT_NAME") == "Pull Requests" diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt index fa3ec67c93..9a25595d63 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt @@ -14,6 +14,8 @@ import net.corda.core.node.ServiceHub import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.node.services.TransactionStorage +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction @@ -130,6 +132,8 @@ data class TestTransactionDSLInterpreter private constructor( ledgerInterpreter.services.cordappProvider override val notaryService: NotaryService? = null + + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) } private fun copy(): TestTransactionDSLInterpreter = diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt index 6802fd042e..402b3757c0 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt @@ -26,6 +26,7 @@ class TestingNamedCacheFactory private constructor(private val sizeOverride: Lon val configuredCaffeine = when (name) { "DBTransactionStorage_transactions" -> caffeine.maximumWeight(1.MB) "NodeAttachmentService_attachmentContent" -> caffeine.maximumWeight(1.MB) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(sizeOverride) else -> caffeine.maximumSize(sizeOverride) } return configuredCaffeine.build(loader) diff --git a/testing/testserver/src/main/java/CordaWebserverCaplet.java b/testing/testserver/src/main/java/CordaWebserverCaplet.java index 61ada57e9b..ba0fbb7054 100644 --- a/testing/testserver/src/main/java/CordaWebserverCaplet.java +++ b/testing/testserver/src/main/java/CordaWebserverCaplet.java @@ -27,7 +27,7 @@ public class CordaWebserverCaplet extends Capsule { File configFile = (config == null) ? new File(baseDir, "node.conf") : new File(config); try { ConfigParseOptions parseOptions = ConfigParseOptions.defaults().setAllowMissing(false); - Config defaultConfig = ConfigFactory.parseResources("reference.conf", parseOptions); + Config defaultConfig = ConfigFactory.parseResources("corda-reference.conf", parseOptions); Config baseDirectoryConfig = ConfigFactory.parseMap(Collections.singletonMap("baseDirectory", baseDir)); Config nodeConfig = ConfigFactory.parseFile(configFile, parseOptions); return baseDirectoryConfig.withFallback(nodeConfig).withFallback(defaultConfig).resolve(); diff --git a/testing/testserver/testcapsule/build.gradle b/testing/testserver/testcapsule/build.gradle index b93547a028..f231f12c68 100644 --- a/testing/testserver/testcapsule/build.gradle +++ b/testing/testserver/testcapsule/build.gradle @@ -35,7 +35,7 @@ task buildWebserverJar(type: FatCapsule, dependsOn: project(':node').tasks.jar) project(':testing:testserver').tasks.jar, project(':testing:testserver').sourceSets.main.java.outputDir.toString() + '/CordaWebserverCaplet.class', project(':testing:testserver').sourceSets.main.java.outputDir.toString() + '/CordaWebserverCaplet$1.class', - project(':node').buildDir.toString() + '/resources/main/reference.conf', + project(':node').buildDir.toString() + '/resources/main/corda-reference.conf', "$rootDir/config/dev/log4j2.xml", project(':node:capsule').projectDir.toString() + '/NOTICE' // Copy CDDL notice ) @@ -64,9 +64,8 @@ task buildWebserverJar(type: FatCapsule, dependsOn: project(':node').tasks.jar) } } -assemble.dependsOn buildWebserverJar - artifacts { + archives buildWebserverJar runtimeArtifacts buildWebserverJar publish buildWebserverJar { classifier '' diff --git a/tools/cliutils/src/main/kotlin/net/corda/cliutils/CordaCliWrapper.kt b/tools/cliutils/src/main/kotlin/net/corda/cliutils/CordaCliWrapper.kt index ce4d01ce3e..8bb9cb6432 100644 --- a/tools/cliutils/src/main/kotlin/net/corda/cliutils/CordaCliWrapper.kt +++ b/tools/cliutils/src/main/kotlin/net/corda/cliutils/CordaCliWrapper.kt @@ -211,6 +211,7 @@ fun printError(message: String) = System.err.println("${ShellConstants.RED}$mess */ object CommonCliConstants { const val BASE_DIR = "--base-directory" + const val CONFIG_FILE = "--config-file" } /** diff --git a/tools/demobench/build.gradle b/tools/demobench/build.gradle index 1522d4b82d..be75806d58 100644 --- a/tools/demobench/build.gradle +++ b/tools/demobench/build.gradle @@ -90,7 +90,7 @@ dependencies { testCompile "junit:junit:$junit_version" } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { // Resolves a Gradle warning about not scanning for pre-processors. options.compilerArgs << '-proc:none' } diff --git a/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeConfig.kt b/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeConfig.kt index 12be389088..86787676f5 100644 --- a/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeConfig.kt +++ b/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeConfig.kt @@ -19,7 +19,7 @@ import java.nio.file.StandardCopyOption import java.util.Properties /** - * This is a subset of FullNodeConfiguration, containing only those configs which we need. The node uses reference.conf + * This is a subset of FullNodeConfiguration, containing only those configs which we need. The node uses corda-reference.conf * to fill in the defaults so we're not required to specify them here. */ data class NodeConfig( diff --git a/tools/demobench/src/test/kotlin/net/corda/demobench/model/NodeConfigTest.kt b/tools/demobench/src/test/kotlin/net/corda/demobench/model/NodeConfigTest.kt index e63c9a6ad6..05171410cf 100644 --- a/tools/demobench/src/test/kotlin/net/corda/demobench/model/NodeConfigTest.kt +++ b/tools/demobench/src/test/kotlin/net/corda/demobench/model/NodeConfigTest.kt @@ -37,7 +37,7 @@ class NodeConfigTest { val nodeConfig = config.nodeConf() .withValue("baseDirectory", valueFor(baseDir.toString())) - .withFallback(ConfigFactory.parseResources("reference.conf")) + .withFallback(ConfigFactory.parseResources("corda-reference.conf")) .withFallback(ConfigFactory.parseMap(mapOf("devMode" to true))) .resolve() val fullConfig = nodeConfig.parseAsNodeConfiguration().value() @@ -70,7 +70,7 @@ class NodeConfigTest { .withValue("systemProperties", valueFor(mapOf("property.name" to "value"))) .withValue("custom.jvmArgs", valueFor("-Xmx1000G")) .withValue("baseDirectory", valueFor(baseDir.toString())) - .withFallback(ConfigFactory.parseResources("reference.conf")) + .withFallback(ConfigFactory.parseResources("corda-reference.conf")) .withFallback(ConfigFactory.parseMap(mapOf("devMode" to true))) .resolve() val fullConfig = nodeConfig.parseAsNodeConfiguration().value() diff --git a/tools/error-tool/build.gradle b/tools/error-tool/build.gradle index d1e11ec376..908775f7c2 100644 --- a/tools/error-tool/build.gradle +++ b/tools/error-tool/build.gradle @@ -1,17 +1,13 @@ apply plugin: 'kotlin' apply plugin: 'com.github.johnrengelman.shadow' -repositories { - mavenCentral() -} - dependencies { implementation project(":common-logging") implementation project(":tools:cliutils") implementation "info.picocli:picocli:$picocli_version" implementation "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version" - testCompile "junit:junit:4.12" + testImplementation "junit:junit:$junit_version" } jar { @@ -28,4 +24,6 @@ shadowJar { } } -assemble.dependsOn shadowJar \ No newline at end of file +artifacts { + archives shadowJar +} diff --git a/tools/explorer/build.gradle b/tools/explorer/build.gradle index f4bc37cee9..82838e5c80 100644 --- a/tools/explorer/build.gradle +++ b/tools/explorer/build.gradle @@ -71,7 +71,7 @@ dependencies { compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version" } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { // Resolves a Gradle warning about not scanning for pre-processors. options.compilerArgs << '-proc:none' } @@ -82,4 +82,4 @@ jar { 'Automatic-Module-Name': 'net.corda.tools.explorer' ) } -} \ No newline at end of file +} diff --git a/tools/explorer/capsule/build.gradle b/tools/explorer/capsule/build.gradle index 05750ee7d6..56b5f3a5de 100644 --- a/tools/explorer/capsule/build.gradle +++ b/tools/explorer/capsule/build.gradle @@ -41,9 +41,8 @@ task buildExplorerJAR(type: FatCapsule, dependsOn: project(':tools:explorer').ta } } -assemble.dependsOn buildExplorerJAR - artifacts { + archives buildExplorerJAR runtimeArtifacts buildExplorerJAR publish buildExplorerJAR { classifier "" diff --git a/tools/network-builder/build.gradle b/tools/network-builder/build.gradle index 3306ac7c15..01c65cdcd4 100644 --- a/tools/network-builder/build.gradle +++ b/tools/network-builder/build.gradle @@ -62,7 +62,7 @@ dependencies { compile "org.controlsfx:controlsfx:$controlsfx_version" } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { // Resolves a Gradle warning about not scanning for pre-processors. options.compilerArgs << '-proc:none' } @@ -78,13 +78,13 @@ shadowJar { zip64 true } -task buildNetworkBuilder(dependsOn: shadowJar) -assemble.dependsOn buildNetworkBuilder +tasks.register('buildNetworkBuilder') { + dependsOn shadowJar +} artifacts { - publish shadowJar { - archiveClassifier = jdkClassifier - } + archives shadowJar + publish shadowJar } jar { diff --git a/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/nodes/NodeBuilder.kt b/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/nodes/NodeBuilder.kt index 6dd3d23359..6c732f8ca9 100644 --- a/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/nodes/NodeBuilder.kt +++ b/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/nodes/NodeBuilder.kt @@ -63,7 +63,7 @@ open class NodeBuilder { fun Config.parseAsNodeConfigWithFallback(preCopyConfig: Config): Validated { val nodeConfig = this .withValue("baseDirectory", ConfigValueFactory.fromAnyRef("")) - .withFallback(ConfigFactory.parseResources("reference.conf")) + .withFallback(ConfigFactory.parseResources("corda-reference.conf")) .withFallback(preCopyConfig) .resolve() return nodeConfig.parseAsNodeConfiguration() diff --git a/tools/shell-cli/build.gradle b/tools/shell-cli/build.gradle index 31214428cb..80ebd521dd 100644 --- a/tools/shell-cli/build.gradle +++ b/tools/shell-cli/build.gradle @@ -27,16 +27,17 @@ processResources { } shadowJar { + archiveClassifier = jdkClassifier mergeServiceFiles() } -task buildShellCli(dependsOn: shadowJar) -assemble.dependsOn buildShellCli +tasks.register('buildShellCli') { + dependsOn shadowJar +} artifacts { - publish shadowJar { - archiveClassifier = jdkClassifier - } + archives shadowJar + publish shadowJar } jar { diff --git a/tools/shell/build.gradle b/tools/shell/build.gradle index f76070853b..4e6f55fda7 100644 --- a/tools/shell/build.gradle +++ b/tools/shell/build.gradle @@ -71,7 +71,7 @@ dependencies { integrationTestCompile project(':node-driver') } -tasks.withType(JavaCompile) { +tasks.withType(JavaCompile).configureEach { // Resolves a Gradle warning about not scanning for pre-processors. options.compilerArgs << '-proc:none' }