diff --git a/.ci/api-current.txt b/.ci/api-current.txt index 2b58759903..23a4b402be 100644 --- a/.ci/api-current.txt +++ b/.ci/api-current.txt @@ -4034,7 +4034,7 @@ public interface net.corda.core.node.services.VaultService public abstract net.corda.core.concurrent.CordaFuture> whenConsumed(net.corda.core.contracts.StateRef) ## public final class net.corda.core.node.services.VaultServiceKt extends java.lang.Object - public static final int MAX_CONSTRAINT_DATA_SIZE = 563 + public static final int MAX_CONSTRAINT_DATA_SIZE = 20000 ## @CordaSerializable public final class net.corda.core.node.services.vault.AggregateFunctionType extends java.lang.Enum diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 603bc520dc..0c93fd2b79 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -4,64 +4,75 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { + agent { label 'k8s' } + options { timestamps() } + triggers { issueCommentTrigger('.*smoke tests.*') } - agent { label 'k8s' } - options { timestamps() } - environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}st" EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" } stages { - stage('Smoke Tests') { + stage('Corda Smoke Tests') { steps { script { - pullRequest.createStatus(status: 'pending', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Building', - targetUrl: "${env.JOB_URL}") - } + if (currentBuildTriggeredByComment()) { + stage('Run Smoke Tests') { + script { + pullRequest.createStatus(status: 'pending', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Running', + targetUrl: "${env.JOB_URL}") + } - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-DpreAllocatePods=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelSmokeTest" + withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { + sh "./gradlew " + + "-Dkubenetize=true " + + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + " clean allParallelSmokeTest --stacktrace" + } + } + + } } } } } post { - always { - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: false + script { + if (currentBuildTriggeredByComment()) { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' + } + } } + success { script { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Passed', - targetUrl: "${env.JOB_URL}testResults") + if (currentBuildTriggeredByComment()) { + pullRequest.createStatus(status: 'success', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Passed', + targetUrl: "${env.JOB_URL}testResults") + } } } failure { script { - pullRequest.createStatus(status: 'failure', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Failed', - targetUrl: "${env.JOB_URL}testResults") + if (currentBuildTriggeredByComment()) { + pullRequest.createStatus(status: 'failure', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Failed', + targetUrl: "${env.JOB_URL}testResults") + } } } @@ -69,4 +80,18 @@ pipeline { deleteDir() /* clean up our workspace */ } } +} + +@NonCPS +def currentBuildTriggeredByComment() { + def triggerCause = currentBuild.rawBuild.getCause(org.jenkinsci.plugins.pipeline.github.trigger.IssueCommentCause) + if (triggerCause) { + echo("Build was started by ${triggerCause.userLogin}, who wrote: " + + "\"${triggerCause.comment}\", which matches the " + + "\"${triggerCause.triggerPattern}\" trigger pattern.") + } else { + echo('Build was not started by a trigger') + } + + return triggerCause != null } \ No newline at end of file diff --git a/build.gradle b/build.gradle index 6de950b67a..8a3f88de7e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ -import net.corda.testing.DistributedTesting import net.corda.testing.DistributeTestsBy +import net.corda.testing.DistributedTesting import net.corda.testing.ImageBuilding import net.corda.testing.ParallelTestGroup import net.corda.testing.PodLogLevel @@ -608,7 +608,7 @@ task allParallelIntegrationTest(type: ParallelTestGroup) { streamOutput false coresPerFork 5 memoryInGbPerFork 10 - distribute DistributeTestsBy.CLASS + distribute DistributeTestsBy.METHOD } task allParallelUnitTest(type: ParallelTestGroup) { podLogLevel PodLogLevel.INFO @@ -645,5 +645,3 @@ task allParallelSmokeTest(type: ParallelTestGroup) { } apply plugin: ImageBuilding apply plugin: DistributedTesting - - diff --git a/buildSrc/src/main/groovy/net/corda/testing/DistributeTestsBy.java b/buildSrc/src/main/groovy/net/corda/testing/DistributeTestsBy.java new file mode 100644 index 0000000000..4c53b07aad --- /dev/null +++ b/buildSrc/src/main/groovy/net/corda/testing/DistributeTestsBy.java @@ -0,0 +1,5 @@ +package net.corda.testing; + +public enum DistributeTestsBy { + CLASS, METHOD +} diff --git a/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy b/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy index 91fe8dd521..0a36baed9d 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy +++ b/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy @@ -50,7 +50,7 @@ class DistributedTesting implements Plugin { project.logger.info("Evaluating ${task.getPath()}") if (task in requestedTasks && !task.hasProperty("ignoreForDistribution")) { project.logger.info "Modifying ${task.getPath()}" - ListTests testListerTask = createTestListingTasks(task, subProject) + Task testListerTask = createTestListingTasks(task, subProject) globalAllocator.addSource(testListerTask, task) Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, globalAllocator) } else { @@ -79,7 +79,7 @@ class DistributedTesting implements Plugin { userGroups.forEach { testGrouping -> //for each "group" (ie: test, integrationTest) within the grouping find all the Test tasks which have the same name. - List testTasksToRunInGroup = ((ParallelTestGroup) testGrouping).groups.collect { + List testTasksToRunInGroup = ((ParallelTestGroup) testGrouping).getGroups().collect { allTestTasksGroupedByType.get(it) }.flatten() @@ -95,7 +95,7 @@ class DistributedTesting implements Plugin { imageBuildTask.dependsOn preAllocateTask } - def userDefinedParallelTask = project.rootProject.tasks.create("userDefined" + testGrouping.name.capitalize(), KubesTest) { + def userDefinedParallelTask = project.rootProject.tasks.create("userDefined" + testGrouping.getName().capitalize(), KubesTest) { group = GRADLE_GROUP if (!tagToUseForRunningTests) { @@ -106,24 +106,24 @@ class DistributedTesting implements Plugin { dependsOn deAllocateTask } numberOfPods = testGrouping.getShardCount() - printOutput = testGrouping.printToStdOut + printOutput = testGrouping.getPrintToStdOut() fullTaskToExecutePath = superListOfTasks - taskToExecuteName = testGrouping.groups.join("And") - memoryGbPerFork = testGrouping.gbOfMemory - numberOfCoresPerFork = testGrouping.coresToUse - distribution = testGrouping.distribution - podLogLevel = testGrouping.logLevel + taskToExecuteName = testGrouping.getGroups().join("And") + memoryGbPerFork = testGrouping.getGbOfMemory() + numberOfCoresPerFork = testGrouping.getCoresToUse() + distribution = testGrouping.getDistribution() + podLogLevel = testGrouping.getLogLevel() doFirst { dockerTag = tagToUseForRunningTests ? (ImageBuilding.registryName + ":" + tagToUseForRunningTests) : (imagePushTask.imageName.get() + ":" + imagePushTask.tag.get()) } } - def reportOnAllTask = project.rootProject.tasks.create("userDefinedReports${testGrouping.name.capitalize()}", KubesReporting) { + def reportOnAllTask = project.rootProject.tasks.create("userDefinedReports${testGrouping.getName().capitalize()}", KubesReporting) { group = GRADLE_GROUP dependsOn userDefinedParallelTask - destinationDir new File(project.rootProject.getBuildDir(), "userDefinedReports${testGrouping.name.capitalize()}") + destinationDir new File(project.rootProject.getBuildDir(), "userDefinedReports${testGrouping.getName().capitalize()}") doFirst { destinationDir.deleteDir() - shouldPrintOutput = !testGrouping.printToStdOut + shouldPrintOutput = !testGrouping.getPrintToStdOut() podResults = userDefinedParallelTask.containerResults reportOn(userDefinedParallelTask.testOutput) } @@ -145,14 +145,14 @@ class DistributedTesting implements Plugin { private List generatePreAllocateAndDeAllocateTasksForGrouping(Project project, ParallelTestGroup testGrouping) { PodAllocator allocator = new PodAllocator(project.getLogger()) - Task preAllocateTask = project.rootProject.tasks.create("preAllocateFor" + testGrouping.name.capitalize()) { + Task preAllocateTask = project.rootProject.tasks.create("preAllocateFor" + testGrouping.getName().capitalize()) { group = GRADLE_GROUP doFirst { String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_BUILDING_PROPERTY) if (dockerTag == null) { throw new GradleException("pre allocation cannot be used without a stable docker tag - please provide one using -D" + ImageBuilding.PROVIDE_TAG_FOR_BUILDING_PROPERTY) } - int seed = (dockerTag.hashCode() + testGrouping.name.hashCode()) + int seed = (dockerTag.hashCode() + testGrouping.getName().hashCode()) String podPrefix = new BigInteger(64, new Random(seed)).toString(36) //here we will pre-request the correct number of pods for this testGroup int numberOfPodsToRequest = testGrouping.getShardCount() @@ -162,14 +162,14 @@ class DistributedTesting implements Plugin { } } - Task deAllocateTask = project.rootProject.tasks.create("deAllocateFor" + testGrouping.name.capitalize()) { + Task deAllocateTask = project.rootProject.tasks.create("deAllocateFor" + testGrouping.getName().capitalize()) { group = GRADLE_GROUP doFirst { String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY) if (dockerTag == null) { throw new GradleException("pre allocation cannot be used without a stable docker tag - please provide one using -D" + ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY) } - int seed = (dockerTag.hashCode() + testGrouping.name.hashCode()) + int seed = (dockerTag.hashCode() + testGrouping.getName().hashCode()) String podPrefix = new BigInteger(64, new Random(seed)).toString(36); allocator.tearDownPods(podPrefix) } @@ -249,12 +249,12 @@ class DistributedTesting implements Plugin { project.plugins.apply(ImageBuilding) } - private ListTests createTestListingTasks(Test task, Project subProject) { + private Task createTestListingTasks(Test task, Project subProject) { def taskName = task.getName() def capitalizedTaskName = task.getName().capitalize() //determine all the tests which are present in this test task. //this list will then be shared between the various worker forks - def createdListTask = subProject.tasks.create("listTestsFor" + capitalizedTaskName, ListTests) { + ListTests createdListTask = subProject.tasks.create("listTestsFor" + capitalizedTaskName, ListTests) { group = GRADLE_GROUP //the convention is that a testing task is backed by a sourceSet with the same name dependsOn subProject.getTasks().getByName("${taskName}Classes") @@ -281,7 +281,7 @@ class DistributedTesting implements Plugin { subProject.logger.info("created task: " + createdListTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdListTask.dependsOn) subProject.logger.info("created task: " + createdPrintTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdPrintTask.dependsOn) - return createdListTask as ListTests + return createdListTask } } diff --git a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java index 09e713772f..736e3a80c1 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java +++ b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java @@ -257,7 +257,7 @@ public class KubesTest extends DefaultTask { client.pods().delete(createdPod); client.persistentVolumeClaims().delete(pvc); } - return new KubePodResult(resCode, podOutput, binaryResults); + return new KubePodResult(podIdx, resCode, podOutput, binaryResults); }); } catch (Retry.RetryException e) { throw new RuntimeException("Failed to build in pod " + podName + " (" + podIdx + "/" + numberOfPods + ") in " + numberOfRetries + " attempts", e); diff --git a/buildSrc/src/main/groovy/net/corda/testing/ListShufflerAndAllocator.java b/buildSrc/src/main/groovy/net/corda/testing/ListShufflerAndAllocator.java new file mode 100644 index 0000000000..6b25e3242c --- /dev/null +++ b/buildSrc/src/main/groovy/net/corda/testing/ListShufflerAndAllocator.java @@ -0,0 +1,37 @@ +package net.corda.testing; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Random; +import java.util.stream.Collectors; + +class ListShufflerAndAllocator { + + private final List tests; + + public ListShufflerAndAllocator(List tests) { + this.tests = new ArrayList<>(tests); + } + + public List getTestsForFork(int fork, int forks, Integer seed) { + final Random shuffler = new Random(seed); + final List copy = new ArrayList<>(tests); + while (copy.size() < forks) { + //pad the list + copy.add(null); + } + Collections.shuffle(copy, shuffler); + final int numberOfTestsPerFork = Math.max((copy.size() / forks), 1); + final int consumedTests = numberOfTestsPerFork * forks; + final int ourStartIdx = numberOfTestsPerFork * fork; + final int ourEndIdx = ourStartIdx + numberOfTestsPerFork; + final int ourSupplementaryIdx = consumedTests + fork; + final ArrayList toReturn = new ArrayList<>(copy.subList(ourStartIdx, ourEndIdx)); + if (ourSupplementaryIdx < copy.size()) { + toReturn.add(copy.get(ourSupplementaryIdx)); + } + return toReturn.stream().filter(Objects::nonNull).collect(Collectors.toList()); + } +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy b/buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy deleted file mode 100644 index 789bab4392..0000000000 --- a/buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy +++ /dev/null @@ -1,110 +0,0 @@ -package net.corda.testing - -import io.github.classgraph.ClassGraph -import io.github.classgraph.ClassInfo -import org.gradle.api.DefaultTask -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.TaskAction - -import java.util.stream.Collectors - -class ListShufflerAndAllocator { - - private final List tests - - public ListShufflerAndAllocator(List tests) { - this.tests = new ArrayList<>(tests) - } - - List getTestsForFork(int fork, int forks, Integer seed) { - Random shuffler = new Random(seed); - List copy = new ArrayList<>(tests); - while (copy.size() < forks) { - //pad the list - copy.add(null); - } - Collections.shuffle(copy, shuffler); - int numberOfTestsPerFork = Math.max((copy.size() / forks).intValue(), 1); - int consumedTests = numberOfTestsPerFork * forks; - int ourStartIdx = numberOfTestsPerFork * fork; - int ourEndIdx = ourStartIdx + numberOfTestsPerFork; - int ourSupplementaryIdx = consumedTests + fork; - ArrayList toReturn = new ArrayList<>(copy.subList(ourStartIdx, ourEndIdx)); - if (ourSupplementaryIdx < copy.size()) { - toReturn.add(copy.get(ourSupplementaryIdx)); - } - return toReturn.stream().filter { it -> it != null }.collect(Collectors.toList()); - } -} - -interface TestLister { - List getAllTestsDiscovered() -} - -class ListTests extends DefaultTask implements TestLister { - - public static final String DISTRIBUTION_PROPERTY = "distributeBy" - - FileCollection scanClassPath - List allTests - DistributeTestsBy distribution = System.getProperty(DISTRIBUTION_PROPERTY) ? DistributeTestsBy.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : DistributeTestsBy.METHOD - - def getTestsForFork(int fork, int forks, Integer seed) { - def gitSha = new BigInteger(project.hasProperty("corda_revision") ? project.property("corda_revision").toString() : "0", 36) - if (fork >= forks) { - throw new IllegalArgumentException("requested shard ${fork + 1} for total shards ${forks}") - } - def seedToUse = seed ? (seed + ((String) this.getPath()).hashCode() + gitSha.intValue()) : 0 - return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse) - } - - @Override - public List getAllTestsDiscovered() { - return new ArrayList<>(allTests) - } - - @TaskAction - def discoverTests() { - switch (distribution) { - case DistributeTestsBy.METHOD: - Collection results = new ClassGraph() - .enableClassInfo() - .enableMethodInfo() - .ignoreClassVisibility() - .ignoreMethodVisibility() - .enableAnnotationInfo() - .overrideClasspath(scanClassPath) - .scan() - .getClassesWithMethodAnnotation("org.junit.Test") - .collect { c -> (c.getSubclasses() + Collections.singletonList(c)) } - .flatten() - .collect { ClassInfo c -> - c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name } - }.flatten() - .toSet() - - this.allTests = results.stream().sorted().collect(Collectors.toList()) - break - case DistributeTestsBy.CLASS: - Collection results = new ClassGraph() - .enableClassInfo() - .enableMethodInfo() - .ignoreClassVisibility() - .ignoreMethodVisibility() - .enableAnnotationInfo() - .overrideClasspath(scanClassPath) - .scan() - .getClassesWithMethodAnnotation("org.junit.Test") - .collect { c -> (c.getSubclasses() + Collections.singletonList(c)) } - .flatten() - .collect { ClassInfo c -> c.name }.flatten() - .toSet() - this.allTests = results.stream().sorted().collect(Collectors.toList()) - break - } - } -} - -public enum DistributeTestsBy { - CLASS, METHOD -} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/net/corda/testing/ListTests.java b/buildSrc/src/main/groovy/net/corda/testing/ListTests.java new file mode 100644 index 0000000000..a2ee34eda0 --- /dev/null +++ b/buildSrc/src/main/groovy/net/corda/testing/ListTests.java @@ -0,0 +1,99 @@ +package net.corda.testing; + +import io.github.classgraph.ClassGraph; +import io.github.classgraph.ClassInfo; +import io.github.classgraph.ClassInfoList; +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.TaskAction; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +interface TestLister { + List getAllTestsDiscovered(); +} + +public class ListTests extends DefaultTask implements TestLister { + + public static final String DISTRIBUTION_PROPERTY = "distributeBy"; + + public FileCollection scanClassPath; + private List allTests; + private DistributeTestsBy distribution = System.getProperty(DISTRIBUTION_PROPERTY) != null && !System.getProperty(DISTRIBUTION_PROPERTY).isEmpty() ? + DistributeTestsBy.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : DistributeTestsBy.METHOD; + + public List getTestsForFork(int fork, int forks, Integer seed) { + BigInteger gitSha = new BigInteger(getProject().hasProperty("corda_revision") ? + getProject().property("corda_revision").toString() : "0", 36); + if (fork >= forks) { + throw new IllegalArgumentException("requested shard ${fork + 1} for total shards ${forks}"); + } + int seedToUse = seed != null ? (seed + (this.getPath()).hashCode() + gitSha.intValue()) : 0; + return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse); + } + + @Override + public List getAllTestsDiscovered() { + return new ArrayList<>(allTests); + } + + @TaskAction + void discoverTests() { + Collection results; + switch (distribution) { + case METHOD: + results = new ClassGraph() + .enableClassInfo() + .enableMethodInfo() + .ignoreClassVisibility() + .ignoreMethodVisibility() + .enableAnnotationInfo() + .overrideClasspath(scanClassPath) + .scan() + .getClassesWithMethodAnnotation("org.junit.Test") + .stream() + .map(classInfo -> { + ClassInfoList returnList = new ClassInfoList(); + returnList.add(classInfo); + returnList.addAll(classInfo.getSubclasses()); + return returnList; + }) + .flatMap(ClassInfoList::stream) + .map(classInfo -> classInfo.getMethodInfo().filter(methodInfo -> methodInfo.hasAnnotation("org.junit.Test")) + .stream().map(methodInfo -> classInfo.getName() + "." + methodInfo.getName())) + .flatMap(Function.identity()) + .collect(Collectors.toSet()); + + this.allTests = results.stream().sorted().collect(Collectors.toList()); + break; + case CLASS: + results = new ClassGraph() + .enableClassInfo() + .enableMethodInfo() + .ignoreClassVisibility() + .ignoreMethodVisibility() + .enableAnnotationInfo() + .overrideClasspath(scanClassPath) + .scan() + .getClassesWithMethodAnnotation("org.junit.Test") + .stream() + .map(classInfo -> { + ClassInfoList returnList = new ClassInfoList(); + returnList.add(classInfo); + returnList.addAll(classInfo.getSubclasses()); + return returnList; + }) + .flatMap(ClassInfoList::stream) + .map(ClassInfo::getName) + .collect(Collectors.toSet()); + this.allTests = results.stream().sorted().collect(Collectors.toList()); + break; + } + getProject().getLogger().lifecycle("THESE ARE ALL THE TESTSSS!!!!!!!!: " + allTests.toString()); + } +} \ No newline at end of file diff --git a/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy b/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy deleted file mode 100644 index 19e589fb5f..0000000000 --- a/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy +++ /dev/null @@ -1,51 +0,0 @@ -package net.corda.testing - -import org.gradle.api.DefaultTask - -class ParallelTestGroup extends DefaultTask { - - DistributeTestsBy distribution = DistributeTestsBy.METHOD - - List groups = new ArrayList<>() - int shardCount = 20 - int coresToUse = 4 - int gbOfMemory = 4 - boolean printToStdOut = true - PodLogLevel logLevel = PodLogLevel.INFO - - void numberOfShards(int shards) { - this.shardCount = shards - } - - void podLogLevel(PodLogLevel level) { - this.logLevel = level - } - - void distribute(DistributeTestsBy dist) { - this.distribution = dist - } - - void coresPerFork(int cores) { - this.coresToUse = cores - } - - void memoryInGbPerFork(int gb) { - this.gbOfMemory = gb - } - - //when this is false, only containers will "failed" exit codes will be printed to stdout - void streamOutput(boolean print) { - this.printToStdOut = print - } - - void testGroups(String... group) { - testGroups(group.toList()) - } - - void testGroups(List group) { - group.forEach { - groups.add(it) - } - } - -} diff --git a/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.java b/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.java new file mode 100644 index 0000000000..d8e4d5bf44 --- /dev/null +++ b/buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.java @@ -0,0 +1,80 @@ +package net.corda.testing; + +import org.gradle.api.DefaultTask; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ParallelTestGroup extends DefaultTask { + + private DistributeTestsBy distribution = DistributeTestsBy.METHOD; + private List groups = new ArrayList<>(); + private int shardCount = 20; + private int coresToUse = 4; + private int gbOfMemory = 4; + private boolean printToStdOut = true; + private PodLogLevel logLevel = PodLogLevel.INFO; + + public DistributeTestsBy getDistribution() { + return distribution; + } + + public List getGroups() { + return groups; + } + + public int getShardCount() { + return shardCount; + } + + public int getCoresToUse() { + return coresToUse; + } + + public int getGbOfMemory() { + return gbOfMemory; + } + + public boolean getPrintToStdOut() { + return printToStdOut; + } + + public PodLogLevel getLogLevel() { + return logLevel; + } + + public void numberOfShards(int shards) { + this.shardCount = shards; + } + + public void podLogLevel(PodLogLevel level) { + this.logLevel = level; + } + + public void distribute(DistributeTestsBy dist) { + this.distribution = dist; + } + + public void coresPerFork(int cores) { + this.coresToUse = cores; + } + + public void memoryInGbPerFork(int gb) { + this.gbOfMemory = gb; + } + + //when this is false, only containers will "failed" exit codes will be printed to stdout + public void streamOutput(boolean print) { + this.printToStdOut = print; + } + + public void testGroups(String... group) { + testGroups(Arrays.asList(group)); + } + + private void testGroups(List group) { + groups.addAll(group); + } + +} diff --git a/buildSrc/src/main/groovy/net/corda/testing/RunInParallel.groovy b/buildSrc/src/main/groovy/net/corda/testing/RunInParallel.groovy deleted file mode 100644 index c4ad60b59a..0000000000 --- a/buildSrc/src/main/groovy/net/corda/testing/RunInParallel.groovy +++ /dev/null @@ -1,32 +0,0 @@ -package net.corda.testing - -import org.gradle.api.Action -import org.gradle.api.DefaultTask -import org.gradle.api.Task -import org.gradle.api.tasks.TaskAction - -import java.util.concurrent.CompletableFuture - -class RunInParallel extends DefaultTask { - - private List tasksToRunInParallel = new ArrayList<>() - - public RunInParallel runInParallel(Task... tasks) { - for (Task task : tasks) { - tasksToRunInParallel.add(task) - } - return this; - } - - @TaskAction - def void run() { - tasksToRunInParallel.collect { t -> - CompletableFuture.runAsync { - def actions = t.getActions() - for (Action action : actions) { - action.execute(t) - } - } - }.join() - } -} diff --git a/buildSrc/src/main/java/net/corda/testing/KubePodResult.java b/buildSrc/src/main/java/net/corda/testing/KubePodResult.java index 2f9a987f11..76ba668c39 100644 --- a/buildSrc/src/main/java/net/corda/testing/KubePodResult.java +++ b/buildSrc/src/main/java/net/corda/testing/KubePodResult.java @@ -5,11 +5,13 @@ import java.util.Collection; public class KubePodResult { + private final int podIndex; private final int resultCode; private final File output; private final Collection binaryResults; - public KubePodResult(int resultCode, File output, Collection binaryResults) { + public KubePodResult(int podIndex, int resultCode, File output, Collection binaryResults) { + this.podIndex = podIndex; this.resultCode = resultCode; this.output = output; this.binaryResults = binaryResults; @@ -26,4 +28,8 @@ public class KubePodResult { public Collection getBinaryResults() { return binaryResults; } + + public int getPodIndex() { + return podIndex; + } } diff --git a/buildSrc/src/main/java/net/corda/testing/KubesReporting.java b/buildSrc/src/main/java/net/corda/testing/KubesReporting.java index ac2306541e..4f8f8aee0c 100644 --- a/buildSrc/src/main/java/net/corda/testing/KubesReporting.java +++ b/buildSrc/src/main/java/net/corda/testing/KubesReporting.java @@ -151,12 +151,12 @@ public class KubesReporting extends DefaultTask { if (!containersWithNonZeroReturnCodes.isEmpty()) { String reportUrl = new ConsoleRenderer().asClickableFileUrl(new File(destinationDir, "index.html")); - if (shouldPrintOutput){ + if (shouldPrintOutput) { containersWithNonZeroReturnCodes.forEach(podResult -> { try { - System.out.println("\n##### CONTAINER OUTPUT START #####"); + System.out.println("\n##### CONTAINER " + podResult.getPodIndex() + " OUTPUT START #####"); IOUtils.copy(new FileInputStream(podResult.getOutput()), System.out); - System.out.println("##### CONTAINER OUTPUT END #####\n"); + System.out.println("##### CONTAINER " + podResult.getPodIndex() + " OUTPUT END #####\n"); } catch (IOException ignored) { } }); diff --git a/buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.groovy b/buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.java similarity index 62% rename from buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.groovy rename to buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.java index f88b6b819c..c45745d5dd 100644 --- a/buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.groovy +++ b/buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.java @@ -1,25 +1,30 @@ -package net.corda.testing +package net.corda.testing; -import org.hamcrest.CoreMatchers -import org.junit.Assert -import org.junit.Test +import org.hamcrest.CoreMatchers; +import org.junit.Assert; +import org.junit.Test; -import java.util.stream.Collectors -import java.util.stream.IntStream +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; -import static org.hamcrest.core.Is.is -import static org.hamcrest.core.IsEqual.equalTo +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; -class ListTestsTest { +public class ListTestsTest { @Test - void shouldAllocateTests() { + public void shouldAllocateTests() { for (int numberOfTests = 0; numberOfTests < 100; numberOfTests++) { for (int numberOfForks = 1; numberOfForks < 100; numberOfForks++) { - List tests = IntStream.range(0, numberOfTests).collect { z -> "Test.method" + z } + List tests = IntStream.range(0, numberOfTests).boxed() + .map(integer -> "Test.method" + integer.toString()) + .collect(Collectors.toList()); ListShufflerAndAllocator testLister = new ListShufflerAndAllocator(tests); List listOfLists = new ArrayList<>(); diff --git a/core-tests/src/test/kotlin/net/corda/coretests/contracts/TransactionVerificationExceptionSerialisationTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/contracts/TransactionVerificationExceptionSerialisationTests.kt index e8bf2cf807..0167d85b2a 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/contracts/TransactionVerificationExceptionSerialisationTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/contracts/TransactionVerificationExceptionSerialisationTests.kt @@ -96,6 +96,20 @@ class TransactionVerificationExceptionSerialisationTests { assertEquals(exception.txId, exception2.txId) } + @Test + fun invalidConstraintRejectionError() { + val exception = TransactionVerificationException.InvalidConstraintRejection(txid, "Some contract class", "for being too funny") + val exceptionAfterSerialisation = DeserializationInput(factory).deserialize( + SerializationOutput(factory).serialize(exception, context), + context + ) + + assertEquals(exception.message, exceptionAfterSerialisation.message) + assertEquals(exception.cause?.message, exceptionAfterSerialisation.cause?.message) + assertEquals(exception.contractClass, exceptionAfterSerialisation.contractClass) + assertEquals(exception.reason, exceptionAfterSerialisation.reason) + } + @Test fun contractCreationErrorTest() { val cause = Throwable("wibble") diff --git a/core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt b/core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt index 826a7d9eb4..e941f8adce 100644 --- a/core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt +++ b/core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt @@ -92,6 +92,16 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S class ContractConstraintRejection(txId: SecureHash, val contractClass: String) : TransactionVerificationException(txId, "Contract constraints failed for $contractClass", null) + /** + * A constraint attached to a state was invalid, e.g. due to size limitations. + * + * @property contractClass The fully qualified class name of the failing contract. + * @property reason a message containing the reason the constraint is invalid included in thrown the exception. + */ + @KeepForDJVM + class InvalidConstraintRejection(txId: SecureHash, val contractClass: String, val reason: String) + : TransactionVerificationException(txId, "Contract constraints failed for $contractClass. $reason", null) + /** * A state requested a contract class via its [TransactionState.contract] field that didn't appear in any attached * JAR at all. This usually implies the attachments were forgotten or a version mismatch. diff --git a/core/src/main/kotlin/net/corda/core/flows/NotaryError.kt b/core/src/main/kotlin/net/corda/core/flows/NotaryError.kt index 835f0291ab..b389e0c150 100644 --- a/core/src/main/kotlin/net/corda/core/flows/NotaryError.kt +++ b/core/src/main/kotlin/net/corda/core/flows/NotaryError.kt @@ -21,6 +21,10 @@ class NotaryException( /** Specifies the cause for notarisation request failure. */ @CordaSerializable sealed class NotaryError { + companion object { + const val NUM_STATES = 5 + } + /** Occurs when one or more input states have already been consumed by another transaction. */ data class Conflict( /** Id of the transaction that was attempted to be notarised. */ @@ -28,8 +32,9 @@ sealed class NotaryError { /** Specifies which states have already been consumed in another transaction. */ val consumedStates: Map ) : NotaryError() { - override fun toString() = "One or more input states or referenced states have already been used as input states in other transactions. Conflicting state count: ${consumedStates.size}, consumption details:\n" + - "${consumedStates.asSequence().joinToString(",\n", limit = 5) { it.key.toString() + " -> " + it.value }}.\n" + + override fun toString() = "One or more input states or referenced states have already been used as input states in other transactions. " + + "Conflicting state count: ${consumedStates.size}, consumption details:\n" + + "${consumedStates.asSequence().joinToString(",\n", limit = NUM_STATES) { it.key.toString() + " -> " + it.value }}.\n" + "To find out if any of the conflicting transactions have been generated by this node you can use the hashLookup Corda shell command." } diff --git a/core/src/main/kotlin/net/corda/core/internal/ConstraintsUtils.kt b/core/src/main/kotlin/net/corda/core/internal/ConstraintsUtils.kt index f12b2619e0..c05ae94680 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ConstraintsUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ConstraintsUtils.kt @@ -10,6 +10,13 @@ import net.corda.core.utilities.loggerFor */ typealias Version = Int +/** + * The maximum number of keys in a signature constraint that the platform supports. + * + * Attention: this value affects consensus, so it requires a minimum platform version bump in order to be changed. + */ +const val MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT = 20 + private val log = loggerFor() val Attachment.contractVersion: Version get() = if (this is ContractAttachment) version else CordappImpl.DEFAULT_CORDAPP_VERSION diff --git a/core/src/main/kotlin/net/corda/core/internal/TransactionVerifierServiceInternal.kt b/core/src/main/kotlin/net/corda/core/internal/TransactionVerifierServiceInternal.kt index ff44f5140f..7925fabfc4 100644 --- a/core/src/main/kotlin/net/corda/core/internal/TransactionVerifierServiceInternal.kt +++ b/core/src/main/kotlin/net/corda/core/internal/TransactionVerifierServiceInternal.kt @@ -5,6 +5,7 @@ import net.corda.core.KeepForDJVM import net.corda.core.concurrent.CordaFuture import net.corda.core.contracts.* import net.corda.core.contracts.TransactionVerificationException.TransactionContractConflictException +import net.corda.core.crypto.CompositeKey import net.corda.core.crypto.SecureHash import net.corda.core.internal.rules.StateContractValidationEnforcementRule import net.corda.core.transactions.LedgerTransaction @@ -329,8 +330,23 @@ abstract class Verifier(val ltx: LedgerTransaction, protected val transactionCla private fun verifyConstraints(contractAttachmentsByContract: Map) { // For each contract/constraint pair check that the relevant attachment is valid. allStates.map { it.contract to it.constraint }.toSet().forEach { (contract, constraint) -> - if (constraint is SignatureAttachmentConstraint) + if (constraint is SignatureAttachmentConstraint) { + /** + * Support for signature constraints has been added on min. platform version >= 4. + * On minimum platform version >= 5, an explicit check has been introduced on the supported number of leaf keys + * in composite keys of signature constraints in order to harden consensus. + */ checkMinimumPlatformVersion(ltx.networkParameters?.minimumPlatformVersion ?: 1, 4, "Signature constraints") + val constraintKey = constraint.key + if (ltx.networkParameters?.minimumPlatformVersion ?: 1 >= 5) { + if (constraintKey is CompositeKey && constraintKey.leafKeys.size > MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT) { + throw TransactionVerificationException.InvalidConstraintRejection(ltx.id, contract, + "Signature constraint contains composite key with ${constraintKey.leafKeys.size} leaf keys, " + + "which is more than the maximum allowed number of keys " + + "($MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT).") + } + } + } // We already checked that there is one and only one attachment. val contractAttachment = contractAttachmentsByContract[contract]!! diff --git a/core/src/main/kotlin/net/corda/core/node/services/VaultService.kt b/core/src/main/kotlin/net/corda/core/node/services/VaultService.kt index 7993b58317..4e4a501069 100644 --- a/core/src/main/kotlin/net/corda/core/node/services/VaultService.kt +++ b/core/src/main/kotlin/net/corda/core/node/services/VaultService.kt @@ -10,6 +10,7 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.FlowException import net.corda.core.flows.FlowLogic import net.corda.core.identity.AbstractParty +import net.corda.core.internal.MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT import net.corda.core.internal.concurrent.doneFuture import net.corda.core.messaging.DataFeed import net.corda.core.node.services.Vault.RelevancyStatus.* @@ -256,9 +257,15 @@ class Vault(val states: Iterable>) { /** * The maximum permissible size of contract constraint type data (for storage in vault states database table). - * Maximum value equates to a CompositeKey with 10 EDDSA_ED25519_SHA512 keys stored in. + * + * This value establishes an upper limit of a CompositeKey with up to [MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT] keys stored in. + * However, note this assumes a rather conservative upper bound per key. + * For reference, measurements have shown the following numbers for each algorithm: + * - 2048-bit RSA keys: 1 key -> 294 bytes, 2 keys -> 655 bytes, 3 keys -> 961 bytes + * - 256-bit ECDSA (k1) keys: 1 key -> 88 bytes, 2 keys -> 231 bytes, 3 keys -> 331 bytes + * - 256-bit EDDSA keys: 1 key -> 44 bytes, 2 keys -> 140 bytes, 3 keys -> 195 bytes */ -const val MAX_CONSTRAINT_DATA_SIZE = 563 +const val MAX_CONSTRAINT_DATA_SIZE = 1_000 * MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT /** * A [VaultService] is responsible for securely and safely persisting the current state of a vault to storage. The diff --git a/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt b/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt index 95377233b4..91bfe9bc0d 100644 --- a/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt +++ b/core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt @@ -92,6 +92,7 @@ sealed class QueryCriteria : GenericQueryCriteria>? = null, @@ -264,6 +265,7 @@ sealed class QueryCriteria : GenericQueryCriteria? = null, val uuid: List? = null, @@ -545,6 +547,7 @@ sealed class AttachmentQueryCriteria : GenericQueryCriteria? = null, val filenameCondition: ColumnPredicate? = null, val uploadDateCondition: ColumnPredicate? = null, diff --git a/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt b/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt index b77aabb736..105046e1c8 100644 --- a/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt +++ b/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt @@ -90,7 +90,9 @@ class PersistentState(@EmbeddedId override var stateRef: PersistentStateRef? = n @KeepForDJVM @Embeddable @Immutable + data class PersistentStateRef( + @Suppress("MagicNumber") // column width @Column(name = "transaction_id", length = 64, nullable = false) var txId: String, diff --git a/detekt-baseline.xml b/detekt-baseline.xml index e5317b50bd..65ceba089a 100644 --- a/detekt-baseline.xml +++ b/detekt-baseline.xml @@ -143,8 +143,6 @@ ComplexMethod:CustomSerializerRegistry.kt$CachingCustomSerializerRegistry$private fun doFindCustomSerializer(clazz: Class<*>, declaredType: Type): AMQPSerializer<Any>? ComplexMethod:DeserializationInput.kt$DeserializationInput$fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, context: SerializationContext): Any ComplexMethod:DriverDSLImpl.kt$DriverDSLImpl$override fun start() - ComplexMethod:DriverDSLImpl.kt$DriverDSLImpl$private fun startNodeInternal(config: NodeConfig, webAddress: NetworkHostAndPort, localNetworkMap: LocalNetworkMap?, parameters: NodeParameters): CordaFuture<NodeHandle> - ComplexMethod:DriverDSLImpl.kt$DriverDSLImpl$private fun startRegisteredNode(name: CordaX500Name, localNetworkMap: LocalNetworkMap?, parameters: NodeParameters, p2pAddress: NetworkHostAndPort = portAllocation.nextHostAndPort()): CordaFuture<NodeHandle> ComplexMethod:Expect.kt$ fun <S, E : Any> S.genericExpectEvents( isStrict: Boolean = true, stream: S.((E) -> Unit) -> Unit, expectCompose: () -> ExpectCompose<E> ) ComplexMethod:FinalityFlow.kt$FinalityFlow$@Suspendable @Throws(NotaryException::class) override fun call(): SignedTransaction ComplexMethod:FlowMonitor.kt$FlowMonitor$private fun warningMessageForFlowWaitingOnIo(request: FlowIORequest<*>, flow: FlowStateMachineImpl<*>, now: Instant): String @@ -688,7 +686,7 @@ LongParameterList:DriverDSL.kt$DriverDSL$( defaultParameters: NodeParameters = NodeParameters(), providedName: CordaX500Name? = defaultParameters.providedName, rpcUsers: List<User> = defaultParameters.rpcUsers, verifierType: VerifierType = defaultParameters.verifierType, customOverrides: Map<String, Any?> = defaultParameters.customOverrides, startInSameProcess: Boolean? = defaultParameters.startInSameProcess, maximumHeapSize: String = defaultParameters.maximumHeapSize ) LongParameterList:DriverDSL.kt$DriverDSL$( defaultParameters: NodeParameters = NodeParameters(), providedName: CordaX500Name? = defaultParameters.providedName, rpcUsers: List<User> = defaultParameters.rpcUsers, verifierType: VerifierType = defaultParameters.verifierType, customOverrides: Map<String, Any?> = defaultParameters.customOverrides, startInSameProcess: Boolean? = defaultParameters.startInSameProcess, maximumHeapSize: String = defaultParameters.maximumHeapSize, logLevelOverride: String? = defaultParameters.logLevelOverride ) LongParameterList:DriverDSLImpl.kt$( isDebug: Boolean = DriverParameters().isDebug, driverDirectory: Path = DriverParameters().driverDirectory, portAllocation: PortAllocation = DriverParameters().portAllocation, debugPortAllocation: PortAllocation = DriverParameters().debugPortAllocation, systemProperties: Map<String, String> = DriverParameters().systemProperties, useTestClock: Boolean = DriverParameters().useTestClock, startNodesInProcess: Boolean = DriverParameters().startNodesInProcess, extraCordappPackagesToScan: List<String> = @Suppress("DEPRECATION") DriverParameters().extraCordappPackagesToScan, waitForAllNodesToFinish: Boolean = DriverParameters().waitForAllNodesToFinish, notarySpecs: List<NotarySpec> = DriverParameters().notarySpecs, jmxPolicy: JmxPolicy = DriverParameters().jmxPolicy, networkParameters: NetworkParameters = DriverParameters().networkParameters, compatibilityZone: CompatibilityZoneParams? = null, notaryCustomOverrides: Map<String, Any?> = DriverParameters().notaryCustomOverrides, inMemoryDB: Boolean = DriverParameters().inMemoryDB, cordappsForAllNodes: Collection<TestCordappInternal>? = null, dsl: DriverDSLImpl.() -> A ) - LongParameterList:DriverDSLImpl.kt$DriverDSLImpl.Companion$( config: NodeConfig, quasarJarPath: String, debugPort: Int?, overriddenSystemProperties: Map<String, String>, maximumHeapSize: String, logLevelOverride: String?, vararg extraCmdLineFlag: String ) + LongParameterList:DriverDSLImpl.kt$DriverDSLImpl.Companion$( config: NodeConfig, quasarJarPath: String, debugPort: Int?, bytemanJarPath: String?, bytemanPort: Int?, overriddenSystemProperties: Map<String, String>, maximumHeapSize: String, logLevelOverride: String?, vararg extraCmdLineFlag: String ) LongParameterList:DummyFungibleContract.kt$DummyFungibleContract$(inputs: List<State>, outputs: List<State>, tx: LedgerTransaction, issueCommand: CommandWithParties<Commands.Issue>, currency: Currency, issuer: PartyAndReference) LongParameterList:IRS.kt$FloatingRatePaymentEvent$(date: LocalDate = this.date, accrualStartDate: LocalDate = this.accrualStartDate, accrualEndDate: LocalDate = this.accrualEndDate, dayCountBasisDay: DayCountBasisDay = this.dayCountBasisDay, dayCountBasisYear: DayCountBasisYear = this.dayCountBasisYear, fixingDate: LocalDate = this.fixingDate, notional: Amount<Currency> = this.notional, rate: Rate = this.rate) LongParameterList:IRS.kt$InterestRateSwap$(floatingLeg: FloatingLeg, fixedLeg: FixedLeg, calculation: Calculation, common: Common, oracle: Party, notary: Party) @@ -1209,6 +1207,7 @@ MagicNumber:TransactionUtils.kt$4 MagicNumber:TransactionVerificationException.kt$TransactionVerificationException.ConstraintPropagationRejection$3 MagicNumber:TransactionVerifierServiceInternal.kt$Verifier$4 + MagicNumber:TransactionVerifierServiceInternal.kt$Verifier$5 MagicNumber:TransactionViewer.kt$TransactionViewer$15.0 MagicNumber:TransactionViewer.kt$TransactionViewer$20.0 MagicNumber:TransactionViewer.kt$TransactionViewer$200.0 @@ -2062,9 +2061,9 @@ MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$private MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val flowOverrideConfig = FlowOverrideConfig(parameters.flowOverrides.map { FlowOverride(it.key.canonicalName, it.value.canonicalName) }) MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val jdbcUrl = "jdbc:h2:mem:persistence${inMemoryCounter.getAndIncrement()};DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=100" - MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val process = startOutOfProcessNode(config, quasarJarPath, debugPort, systemProperties, parameters.maximumHeapSize, parameters.logLevelOverride) + MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion$if (bytemanAgent != null && debugPort != null) listOf("-Dorg.jboss.byteman.verbose=true", "-Dorg.jboss.byteman.debug=true") else emptyList() MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion$private operator fun Config.plus(property: Pair<String, Any>) - MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion${ log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) val systemProperties = mutableMapOf( "name" to config.corda.myLegalName, "visualvm.display.name" to "corda-${config.corda.myLegalName}" ) debugPort?.let { systemProperties += "log4j2.level" to "debug" systemProperties += "log4j2.debug" to "true" } systemProperties += inheritFromParentProcess() systemProperties += overriddenSystemProperties // See experimental/quasar-hook/README.md for how to generate. val excludePattern = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;" + "com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;" + "com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;" + "io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;" + "org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;" + "org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;" + "org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**;org.jolokia**;" + "com.lmax**;picocli**;liquibase**;com.github.benmanes**;org.json**;org.postgresql**;nonapi.io.github.classgraph**;)" val extraJvmArguments = systemProperties.removeResolvedClasspath().map { "-D${it.key}=${it.value}" } + "-javaagent:$quasarJarPath=$excludePattern" val loggingLevel = when { logLevelOverride != null -> logLevelOverride debugPort == null -> "INFO" else -> "DEBUG" } val arguments = mutableListOf( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { it += extraCmdLineFlag }.toList() // The following dependencies are excluded from the classpath of the created JVM, so that the environment resembles a real one as close as possible. // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.) or irrelevant testing libraries (test, corda-mock etc.). // TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164. val exclude = listOf("samples", "finance", "integrationTest", "test", "corda-mock", "com.opengamma.strata") val cp = ProcessUtilities.defaultClassPath.filterNot { cpEntry -> exclude.any { token -> cpEntry.contains("${File.separatorChar}$token") } || cpEntry.endsWith("-tests.jar") } return ProcessUtilities.startJavaProcess( className = "net.corda.node.Corda", // cannot directly get class for this, so just use string arguments = arguments, jdwpPort = debugPort, extraJvmArguments = extraJvmArguments, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, classPath = cp ) } + MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion${ log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, " + "debug port is " + (debugPort ?: "not enabled") + ", " + "byteMan: " + if (bytemanJarPath == null) "not in classpath" else "port is " + (bytemanPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) val systemProperties = mutableMapOf( "name" to config.corda.myLegalName, "visualvm.display.name" to "corda-${config.corda.myLegalName}" ) debugPort?.let { systemProperties += "log4j2.level" to "debug" systemProperties += "log4j2.debug" to "true" } systemProperties += inheritFromParentProcess() systemProperties += overriddenSystemProperties // See experimental/quasar-hook/README.md for how to generate. val excludePattern = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;" + "com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;" + "com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;" + "io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;" + "org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;" + "org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;" + "org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**;org.jolokia**;" + "com.lmax**;picocli**;liquibase**;com.github.benmanes**;org.json**;org.postgresql**;nonapi.io.github.classgraph**;)" val extraJvmArguments = systemProperties.removeResolvedClasspath().map { "-D${it.key}=${it.value}" } + "-javaagent:$quasarJarPath=$excludePattern" val loggingLevel = when { logLevelOverride != null -> logLevelOverride debugPort == null -> "INFO" else -> "DEBUG" } val arguments = mutableListOf( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { it += extraCmdLineFlag }.toList() val bytemanJvmArgs = { val bytemanAgent = bytemanJarPath?.let { bytemanPort?.let { "-javaagent:$bytemanJarPath=port:$bytemanPort,listener:true" } } listOfNotNull(bytemanAgent) + if (bytemanAgent != null && debugPort != null) listOf("-Dorg.jboss.byteman.verbose=true", "-Dorg.jboss.byteman.debug=true") else emptyList() }.invoke() // The following dependencies are excluded from the classpath of the created JVM, so that the environment resembles a real one as close as possible. // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.) or irrelevant testing libraries (test, corda-mock etc.). // TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164. val exclude = listOf("samples", "finance", "integrationTest", "test", "corda-mock", "com.opengamma.strata") val cp = ProcessUtilities.defaultClassPath.filterNot { cpEntry -> exclude.any { token -> cpEntry.contains("${File.separatorChar}$token") } || cpEntry.endsWith("-tests.jar") } return ProcessUtilities.startJavaProcess( className = "net.corda.node.Corda", // cannot directly get class for this, so just use string arguments = arguments, jdwpPort = debugPort, extraJvmArguments = extraJvmArguments + bytemanJvmArgs, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, classPath = cp ) } MaxLineLength:DriverDSLImpl.kt$InternalDriverDSL$ fun <A> pollUntilNonNull(pollName: String, pollInterval: Duration = DEFAULT_POLL_INTERVAL, warnCount: Int = DEFAULT_WARN_COUNT, check: () -> A?): CordaFuture<A> MaxLineLength:DriverDSLImpl.kt$InternalDriverDSL$ fun pollUntilTrue(pollName: String, pollInterval: Duration = DEFAULT_POLL_INTERVAL, warnCount: Int = DEFAULT_WARN_COUNT, check: () -> Boolean): CordaFuture<Unit> MaxLineLength:DriverDSLImpl.kt$fun DriverDSL.startNode(providedName: CordaX500Name, devMode: Boolean, parameters: NodeParameters = NodeParameters()): CordaFuture<NodeHandle> @@ -3530,6 +3529,7 @@ MaxLineLength:TransactionVerifierServiceInternal.kt$Verifier$if (ltx.attachments.size != ltx.attachments.toSet().size) throw TransactionVerificationException.DuplicateAttachmentsRejection(ltx.id, ltx.attachments.groupBy { it }.filterValues { it.size > 1 }.keys.first()) MaxLineLength:TransactionVerifierServiceInternal.kt$Verifier$if (result.keys != contractClasses) throw TransactionVerificationException.MissingAttachmentRejection(ltx.id, contractClasses.minus(result.keys).first()) MaxLineLength:TransactionVerifierServiceInternal.kt$Verifier$val constraintAttachment = AttachmentWithContext(contractAttachment, contract, ltx.networkParameters!!.whitelistedContractImplementations) + MaxLineLength:TransactionVerifierServiceInternal.kt$Verifier${ /** * Signature constraints are supported on min. platform version >= 4, but this only includes support for a single key per constraint. * Signature contstraints with composite keys containing more than 1 leaf key are supported on min. platform version >= 5. */ checkMinimumPlatformVersion(ltx.networkParameters?.minimumPlatformVersion ?: 1, 4, "Signature constraints") val constraintKey = constraint.key if (constraintKey is CompositeKey && constraintKey.leafKeys.size > 1) { checkMinimumPlatformVersion(ltx.networkParameters?.minimumPlatformVersion ?: 1, 5, "Composite keys for signature constraints") val leafKeysNumber = constraintKey.leafKeys.size if (leafKeysNumber > MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT) throw TransactionVerificationException.InvalidConstraintRejection(ltx.id, contract, "Signature constraint contains composite key with $leafKeysNumber leaf keys, " + "which is more than the maximum allowed number of keys " + "($MAX_NUMBER_OF_KEYS_IN_SIGNATURE_CONSTRAINT).") } } MaxLineLength:TransactionVerifierServiceInternal.kt$Verifier${ // checkNoNotaryChange and checkEncumbrancesValid are called here, and not in the c'tor, as they need access to the "outputs" // list, the contents of which need to be deserialized under the correct classloader. checkNoNotaryChange() checkEncumbrancesValid() // The following checks ensure the integrity of the current transaction and also of the future chain. // See: https://docs.corda.net/head/api-contract-constraints.html // A transaction contains both the data and the code that must be executed to validate the transition of the data. // Transactions can be created by malicious adversaries, who can try to use code that allows them to create transactions that appear valid but are not. // 1. Check that there is one and only one attachment for each relevant contract. val contractAttachmentsByContract = getUniqueContractAttachmentsByContract() // 2. Check that the attachments satisfy the constraints of the states. (The contract verification code is correct.) verifyConstraints(contractAttachmentsByContract) // 3. Check that the actual state constraints are correct. This is necessary because transactions can be built by potentially malicious nodes // who can create output states with a weaker constraint which can be exploited in a future transaction. verifyConstraintsValidity(contractAttachmentsByContract) // 4. Check that the [TransactionState] objects are correctly formed. validateStatesAgainstContract() // 5. Final step is to run the contract code. After the first 4 steps we are now sure that we are running the correct code. verifyContracts() } MaxLineLength:TransactionViewer.kt$TransactionViewer$private MaxLineLength:TransactionViewer.kt$TransactionViewer$private fun ObservableList<StateAndRef<ContractState>>.getParties() @@ -3804,6 +3804,7 @@ NestedBlockDepth:StartedFlowTransition.kt$StartedFlowTransition$private fun TransitionBuilder.sendToSessionsTransition(sourceSessionIdToMessage: Map<SessionId, SerializedBytes<Any>>) NestedBlockDepth:StatusTransitions.kt$StatusTransitions$ fun verify(tx: LedgerTransaction) NestedBlockDepth:ThrowableSerializer.kt$ThrowableSerializer$override fun fromProxy(proxy: ThrowableProxy): Throwable + NestedBlockDepth:TransactionVerifierServiceInternal.kt$Verifier$ private fun verifyConstraints(contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) NestedBlockDepth:TransactionVerifierServiceInternal.kt$Verifier$ private fun verifyConstraintsValidity(contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) SpreadOperator:AMQPSerializationScheme.kt$AbstractAMQPSerializationScheme$(*it.whitelist.toTypedArray()) SpreadOperator:AbstractNode.kt$FlowStarterImpl$(logicType, *args) @@ -3841,7 +3842,7 @@ SpreadOperator:DemoBench.kt$DemoBench.Companion$(DemoBench::class.java, *args) SpreadOperator:DevCertificatesTest.kt$DevCertificatesTest$(*oldX509Certificates) SpreadOperator:DockerInstantiator.kt$DockerInstantiator$(*it.toTypedArray()) - SpreadOperator:DriverDSLImpl.kt$DriverDSLImpl$( config, quasarJarPath, debugPort, systemProperties, "512m", null, *extraCmdLineFlag ) + SpreadOperator:DriverDSLImpl.kt$DriverDSLImpl$( config, quasarJarPath, debugPort, bytemanJarPath, null, systemProperties, "512m", null, *extraCmdLineFlag ) SpreadOperator:DummyContract.kt$DummyContract.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owner.owningKey), /* OUTPUT */ StateAndContract(state, PROGRAM_ID) ) SpreadOperator:DummyContract.kt$DummyContract.Companion$(*items) SpreadOperator:DummyContractV2.kt$DummyContractV2.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owners.map { it.owningKey }), /* OUTPUT */ StateAndContract(state, DummyContractV2.PROGRAM_ID) ) diff --git a/docs/source/api-contract-constraints.rst b/docs/source/api-contract-constraints.rst index 12e79b6401..9ce088c679 100644 --- a/docs/source/api-contract-constraints.rst +++ b/docs/source/api-contract-constraints.rst @@ -100,6 +100,9 @@ Expanding on the previous section, for an app to use Signature Constraints, it m The signers of the app can consist of a single organisation or multiple organisations. Once the app has been signed, it can be distributed across the nodes that intend to use it. +.. note:: The platform currently supports ``CompositeKey``\s with up to 20 keys maximum. + This maximum limit is assuming keys that are either 2048-bit ``RSA`` keys or 256-bit elliptic curve (``EC``) keys. + Each transaction received by a node will then verify that the apps attached to it have the correct signers as specified by its Signature Constraints. This ensures that the version of each app is acceptable to the transaction's input states. diff --git a/docs/source/example-code/src/main/kotlin/net/corda/docs/kotlin/FlowCookbook.kt b/docs/source/example-code/src/main/kotlin/net/corda/docs/kotlin/FlowCookbook.kt index 2ea4f89c49..1d821caf7c 100644 --- a/docs/source/example-code/src/main/kotlin/net/corda/docs/kotlin/FlowCookbook.kt +++ b/docs/source/example-code/src/main/kotlin/net/corda/docs/kotlin/FlowCookbook.kt @@ -266,6 +266,7 @@ class InitiatorFlow(val arg1: Boolean, val arg2: Int, private val counterparty: val ourOutputState: DummyState = DummyState() // DOCEND 22 // Or as copies of other states with some properties changed. + @Suppress("MagicNumber") // literally a magic number // DOCSTART 23 val ourOtherOutputState: DummyState = ourOutputState.copy(magicNumber = 77) // DOCEND 23 diff --git a/docs/source/node-flow-hospital.rst b/docs/source/node-flow-hospital.rst index b78dbc2825..28908e6d92 100644 --- a/docs/source/node-flow-hospital.rst +++ b/docs/source/node-flow-hospital.rst @@ -51,7 +51,13 @@ Specifically, there are two main ways a flow is hospitalized: * **Database constraint violation** (``ConstraintViolationException``): This scenario may occur due to natural contention between racing flows as Corda delegates handling using the database's optimistic concurrency control. - As the likelihood of re-occurrence should be low, the flow will actually error and fail if it experiences this at the same point more than 3 times. No intervention required. + If this exception occurs, the flow will retry. After retrying a number of times, the errored flow is kept in for observation. + + * ``SQLTransientConnectionException``: + Database connection pooling errors are dealt with. If this exception occurs, the flow will retry. After retrying a number of times, the errored flow is kept in for observation. + + * All other instances of ``SQLException``: + Any ``SQLException`` that is thrown and not handled by any of the scenarios detailed above, will be kept in for observation after their first failure. * **Finality Flow handling** - Corda 3.x (old style) ``FinalityFlow`` and Corda 4.x ``ReceiveFinalityFlow`` handling: If on the receive side of the finality flow, any error will result in the flow being kept in for observation to allow the cause of the @@ -64,7 +70,8 @@ Specifically, there are two main ways a flow is hospitalized: The time is hard to document as the notary members, if actually alive, will inform the requester of the ETA of a response. This can occur an infinite number of times. i.e. we never give up notarising. No intervention required. - * ``SQLTransientConnectionException``: - Database connection pooling errors are dealt with. If this exception occurs, the flow will retry. After retrying a number of times, the errored flow is kept in for observation. + * **Internal Corda errors**: + Flows that experience errors from inside the Corda statemachine, that are not handled by any of the scenarios details above, will be retried a number of times + and then kept in for observation if the error continues. .. note:: Flows that are kept in for observation are retried upon node restart. diff --git a/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt b/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt index 7486ddb590..fe8469baf2 100644 --- a/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt +++ b/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt @@ -18,6 +18,7 @@ object CashSchema * First version of a cash contract ORM schema that maps all fields of the [Cash] contract state as it stood * at the time of writing. */ +@Suppress("MagicNumber") // SQL column length @CordaSerializable object CashSchemaV1 : MappedSchema(schemaFamily = CashSchema.javaClass, version = 1, mappedTypes = listOf(PersistentCashState::class.java)) { diff --git a/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt b/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt index 87c31cfc4a..ba7324ec0e 100644 --- a/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt +++ b/finance/contracts/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt @@ -22,6 +22,7 @@ object CommercialPaperSchema * as it stood at the time of writing. */ @CordaSerializable +@Suppress("MagicNumber") // SQL column length object CommercialPaperSchemaV1 : MappedSchema(schemaFamily = CommercialPaperSchema.javaClass, version = 1, mappedTypes = listOf(PersistentCommercialPaperState::class.java)) { override val migrationResource = "commercial-paper.changelog-master" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index c0e6ff1077..371177f26b 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -20,6 +20,7 @@ import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.atomic.AtomicInteger import javax.persistence.AttributeConverter +import javax.persistence.PersistenceException import javax.sql.DataSource /** @@ -98,7 +99,8 @@ class CordaPersistence( cacheFactory: NamedCacheFactory, attributeConverters: Collection> = emptySet(), customClassLoader: ClassLoader? = null, - val closeConnection: Boolean = true + val closeConnection: Boolean = true, + val errorHandler: (t: Throwable) -> Unit = {} ) : Closeable { companion object { private val log = contextLogger() @@ -189,10 +191,18 @@ class CordaPersistence( } fun createSession(): Connection { - // We need to set the database for the current [Thread] or [Fiber] here as some tests share threads across databases. - _contextDatabase.set(this) - currentDBSession().flush() - return contextTransaction.connection + try { + // We need to set the database for the current [Thread] or [Fiber] here as some tests share threads across databases. + _contextDatabase.set(this) + currentDBSession().flush() + return contextTransaction.connection + } catch (sqlException: SQLException) { + errorHandler(sqlException) + throw sqlException + } catch (persistenceException: PersistenceException) { + errorHandler(persistenceException) + throw persistenceException + } } /** @@ -220,10 +230,18 @@ class CordaPersistence( recoverAnyNestedSQLException: Boolean, statement: DatabaseTransaction.() -> T): T { _contextDatabase.set(this) val outer = contextTransactionOrNull - return if (outer != null) { - outer.statement() - } else { - inTopLevelTransaction(isolationLevel, recoverableFailureTolerance, recoverAnyNestedSQLException, statement) + try { + return if (outer != null) { + outer.statement() + } else { + inTopLevelTransaction(isolationLevel, recoverableFailureTolerance, recoverAnyNestedSQLException, statement) + } + } catch (sqlException: SQLException) { + errorHandler(sqlException) + throw sqlException + } catch (persistenceException: PersistenceException) { + errorHandler(persistenceException) + throw persistenceException } } diff --git a/node/build.gradle b/node/build.gradle index 63db4fade4..24eb9c0daf 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -253,6 +253,12 @@ dependencies { // Required by JVMAgentUtil (x-compatible java 8 & 11 agent lookup mechanism) compile files("${System.properties['java.home']}/../lib/tools.jar") + // Byteman for runtime (termination) rules injection on the running node + // Submission tool allowing to install rules on running nodes + integrationTestCompile "org.jboss.byteman:byteman-submit:4.0.3" + // The actual Byteman agent which should only be in the classpath of the out of process nodes + integrationTestCompile "org.jboss.byteman:byteman:4.0.3" + testCompile(project(':test-cli')) testCompile(project(':test-utils')) @@ -262,6 +268,8 @@ dependencies { slowIntegrationTestCompile configurations.testCompile slowIntegrationTestRuntime configurations.runtime slowIntegrationTestRuntime configurations.testRuntime + + testCompile project(':testing:cordapps:dbfailure:dbfworkflows') } tasks.withType(JavaCompile) { diff --git a/node/src/integration-test/kotlin/net/corda/contracts/SignatureConstraintGatingTests.kt b/node/src/integration-test/kotlin/net/corda/contracts/SignatureConstraintGatingTests.kt new file mode 100644 index 0000000000..ae1ca0ec72 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/contracts/SignatureConstraintGatingTests.kt @@ -0,0 +1,89 @@ +package net.corda.contracts + +import net.corda.core.contracts.TransactionVerificationException +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.getOrThrow +import net.corda.finance.DOLLARS +import net.corda.finance.flows.CashIssueFlow +import net.corda.testing.common.internal.testNetworkParameters +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.internal.FINANCE_WORKFLOWS_CORDAPP +import net.corda.testing.node.internal.cordappWithPackages +import org.assertj.core.api.Assertions +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class SignatureConstraintGatingTests { + + @Rule + @JvmField + val tempFolder = TemporaryFolder() + + @Test + fun `signature constraints can be used with up to the maximum allowed number of (RSA) keys`() { + tempFolder.root.toPath().let {path -> + val financeCordapp = cordappWithPackages("net.corda.finance.contracts", "net.corda.finance.schemas") + .signed(keyStorePath = path, numberOfSignatures = 20, keyAlgorithm = "RSA") + + driver(DriverParameters( + networkParameters = testNetworkParameters().copy(minimumPlatformVersion = 5), + cordappsForAllNodes = setOf(financeCordapp, FINANCE_WORKFLOWS_CORDAPP), + startNodesInProcess = true, + inMemoryDB = true + )) { + val node = startNode().getOrThrow() + + node.rpc.startFlowDynamic(CashIssueFlow::class.java, 10.DOLLARS, OpaqueBytes.of(0), defaultNotaryIdentity) + .returnValue.getOrThrow() + } + } + } + + @Test + fun `signature constraints can be used with up to the maximum allowed number of (EC) keys`() { + tempFolder.root.toPath().let {path -> + val financeCordapp = cordappWithPackages("net.corda.finance.contracts", "net.corda.finance.schemas") + .signed(keyStorePath = path, numberOfSignatures = 20, keyAlgorithm = "EC") + + driver(DriverParameters( + networkParameters = testNetworkParameters().copy(minimumPlatformVersion = 5), + cordappsForAllNodes = setOf(financeCordapp, FINANCE_WORKFLOWS_CORDAPP), + startNodesInProcess = true, + inMemoryDB = true + )) { + val node = startNode().getOrThrow() + + node.rpc.startFlowDynamic(CashIssueFlow::class.java, 10.DOLLARS, OpaqueBytes.of(0), defaultNotaryIdentity) + .returnValue.getOrThrow() + } + } + } + + @Test + fun `signature constraints cannot be used with more than the maximum allowed number of keys`() { + tempFolder.root.toPath().let {path -> + val financeCordapp = cordappWithPackages("net.corda.finance.contracts", "net.corda.finance.schemas") + .signed(keyStorePath = path, numberOfSignatures = 21) + + driver(DriverParameters( + networkParameters = testNetworkParameters().copy(minimumPlatformVersion = 5), + cordappsForAllNodes = setOf(financeCordapp, FINANCE_WORKFLOWS_CORDAPP), + startNodesInProcess = true, + inMemoryDB = true + )) { + val node = startNode().getOrThrow() + + Assertions.assertThatThrownBy { + node.rpc.startFlowDynamic(CashIssueFlow::class.java, 10.DOLLARS, OpaqueBytes.of(0), defaultNotaryIdentity) + .returnValue.getOrThrow() + } + .isInstanceOf(TransactionVerificationException.InvalidConstraintRejection::class.java) + .hasMessageContaining("Signature constraint contains composite key with 21 leaf keys, " + + "which is more than the maximum allowed number of keys (20).") + } + } + } + +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt index 25c0b78326..cfe19709c9 100644 --- a/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/flows/FlowRetryTest.kt @@ -17,6 +17,7 @@ import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap import net.corda.node.services.Permissions import net.corda.node.services.statemachine.FlowTimeoutException +import net.corda.node.services.statemachine.StaffedFlowHospital import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.singleIdentity @@ -25,6 +26,7 @@ import net.corda.testing.driver.driver import net.corda.testing.node.User import org.assertj.core.api.Assertions.assertThatExceptionOfType import org.hibernate.exception.ConstraintViolationException +import org.junit.After import org.junit.Before import org.junit.Test import java.lang.management.ManagementFactory @@ -46,6 +48,12 @@ class FlowRetryTest { TransientConnectionFailureFlow.retryCount = -1 WrappedTransientConnectionFailureFlow.retryCount = -1 GeneralExternalFailureFlow.retryCount = -1 + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { true } + } + + @After + fun cleanUp() { + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.clear() } @Test @@ -390,7 +398,9 @@ class WrappedTransientConnectionFailureFlow(private val party: Party) : FlowLogi initiateFlow(party).send("hello there") // checkpoint will restart the flow after the send retryCount += 1 - throw IllegalStateException("wrapped error message", IllegalStateException("another layer deep", SQLTransientConnectionException("Connection is not available")/*.fillInStackTrace()*/)) + throw IllegalStateException( + "wrapped error message", + IllegalStateException("another layer deep", SQLTransientConnectionException("Connection is not available"))) } } diff --git a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt new file mode 100644 index 0000000000..7ce883c37b --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/StatemachineErrorHandlingTest.kt @@ -0,0 +1,2067 @@ +package net.corda.node.services.statemachine + +import co.paralleluniverse.fibers.Suspendable +import net.corda.client.rpc.CordaRPCClient +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.ReceiveFinalityFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party +import net.corda.core.internal.ResolveTransactionsFlow +import net.corda.core.internal.list +import net.corda.core.internal.readAllLines +import net.corda.core.messaging.startFlow +import net.corda.core.messaging.startTrackedFlow +import net.corda.core.node.AppServiceHub +import net.corda.core.node.services.CordaService +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.ProgressTracker +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.core.utilities.unwrap +import net.corda.finance.DOLLARS +import net.corda.finance.flows.CashIssueAndPaymentFlow +import net.corda.node.services.Permissions +import net.corda.node.services.api.ServiceHubInternal +import net.corda.node.services.messaging.DeduplicationHandler +import net.corda.node.services.statemachine.transitions.TopLevelTransition +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.CHARLIE_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverDSL +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.NodeHandle +import net.corda.testing.driver.NodeParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.NotarySpec +import net.corda.testing.node.TestCordapp +import net.corda.testing.node.User +import net.corda.testing.node.internal.FINANCE_CORDAPPS +import net.corda.testing.node.internal.InternalDriverDSL +import org.jboss.byteman.agent.submit.ScriptText +import org.jboss.byteman.agent.submit.Submit +import org.junit.Before +import org.junit.Ignore +import org.junit.Test +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeoutException +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +@Suppress("MaxLineLength") // Byteman rules cannot be easily wrapped +class StatemachineErrorHandlingTest { + + companion object { + val rpcUser = User("user1", "test", permissions = setOf(Permissions.all())) + var counter = 0 + } + + @Before + fun setup() { + counter = 0 + } + + /** + * Throws an exception when performing an [Action.SendInitial] action. + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and is then kept in + * the hospital for observation. + */ + @Test + fun `error during transition with SendInitial action is retried 3 times and kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendInitial action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.SendInitial] event. + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test + fun `error during transition with SendInitial action that does not persist will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendInitial action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF readCounter("counter") < 3 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when executing [DeduplicationHandler.afterDatabaseTransaction] from + * inside an [Action.AcknowledgeMessages] action. + * The exception is thrown every time [DeduplicationHandler.afterDatabaseTransaction] is executed + * inside of [ActionExecutorImpl.executeAcknowledgeMessages] + * + * The exceptions should be swallowed. Therefore there should be no trips to the hospital and no retries. + * The flow should complete successfully as the error is swallowed. + */ + @Test + fun `error during transition with AcknowledgeMessages action is swallowed and flow completes successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Set flag when inside executeAcknowledgeMessages + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeAcknowledgeMessages + AT INVOKE ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction() + IF !flagged("exception_flag") + DO flag("exception_flag"); traceln("Setting flag to true") + ENDRULE + + RULE Throw exception when executing ${DeduplicationHandler::class.java.name}.afterDatabaseTransaction when inside executeAcknowledgeMessages + INTERFACE ${DeduplicationHandler::class.java.name} + METHOD afterDatabaseTransaction + AT ENTRY + IF flagged("exception_flag") + DO traceln("Throwing exception"); clear("exception_flag"); traceln("SETTING FLAG TO FALSE"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(0, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state). + * The exception is thrown 5 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + * + * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition + * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to + * verify that 3 retries are attempted before recovering. + */ + @Test + fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 5 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event before the flow has suspended (remains in an unstarted + * state). + * The exception is thrown 7 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + * + * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition + * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to + * verify that 3 retries are attempted before recovering. + * + * CORDA-3352 - it is currently hanging after putting the flow in for observation + */ + @Test + @Ignore + fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and be kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 7 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event after the flow has suspended (has moved to a started state). + * The exception is thrown 5 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + * + * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition + * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to + * verify that 3 retries are attempted before recovering. + */ + @Test + fun `error during transition with CommitTransaction action that occurs after the first suspend will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + // seems to be restarting the flow from the beginning every time + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when executing first suspend + CLASS ${TopLevelTransition::class.java.name} + METHOD suspendTransition + AT ENTRY + IF !flagged("suspend_flag") + DO flag("suspend_flag"); traceln("Setting suspend flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && readCounter("counter") < 5 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Set flag when executing first commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && !flagged("commit_flag") + DO flag("commit_flag"); traceln("Setting commit flag to true") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing. + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + */ + @Test + fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + // seems to be restarting the flow from the beginning every time + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 + DO incrementCounter("counter"); clear("remove_checkpoint_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when replaying a flow that has already successfully created its initial checkpoint. + * + * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another + * exception is then thrown during the retry itself. + * + * The flow is discharged and replayed from the hospital once. After failing during the replay, the flow is forced into overnight + * observation. It is not ran again after this point + */ + @Test + fun `error during retry of a flow will force the flow into overnight observation`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Set flag when executing first suspend + CLASS ${TopLevelTransition::class.java.name} + METHOD suspendTransition + AT ENTRY + IF !flagged("suspend_flag") + DO flag("suspend_flag"); traceln("Setting suspend flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag") + DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Set flag when executing first commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && !flagged("commit_flag") + DO flag("commit_flag"); traceln("Setting commit flag to true") + ENDRULE + + RULE Throw exception on retry + CLASS ${SingleThreadedStateMachineManager::class.java.name} + METHOD addAndStartFlow + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("retry_exception_flag") + DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(1, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when replaying a flow that has already successfully created its initial checkpoint. + * + * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another + * exception is then thrown during the database commit that comes as part of retrying a flow. + * + * The flow is discharged and replayed from the hospital once. When the database commit failure occurs as part of retrying the + * flow, the starting and completion of the retried flow is affected. In other words, the error occurs as part of the replay, but the + * flow will still finish successfully. This is due to the even being scheduled as part of the retry and the failure in the database + * commit occurs after this point. As the flow is already scheduled, the failure has not affect on it. + */ + @Test + fun `error during commit transaction action when retrying a flow will retry the flow again and complete successfully`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Set flag when executing first suspend + CLASS ${TopLevelTransition::class.java.name} + METHOD suspendTransition + AT ENTRY + IF !flagged("suspend_flag") + DO flag("suspend_flag"); traceln("Setting suspend flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_flag") && !flagged("commit_exception_flag") + DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Set flag when executing first commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && !flagged("commit_flag") + DO flag("commit_flag"); traceln("Setting commit flag to true") + ENDRULE + + RULE Throw exception on retry + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("suspend_flag") && flagged("commit_exception_flag") && !flagged("retry_exception_flag") + DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(1, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when replaying a flow that has not made its initial checkpoint. + * + * An exception is thrown when committing a database transaction during a transition to trigger the retry of the flow. Another + * exception is then thrown during the retry itself. + * + * The flow is discharged and replayed from the hospital once. After failing during the replay, the flow is forced into overnight + * observation. It is not ran again after this point + * + * CORDA-3352 - it is currently hanging after putting the flow in for observation + * + */ + @Test + @Ignore + fun `error during retrying a flow that failed when committing its original checkpoint will force the flow into overnight observation`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Throw exception on executeCommitTransaction action after first suspend + commit + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF !flagged("commit_exception_flag") + DO flag("commit_exception_flag"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Throw exception on retry + CLASS ${SingleThreadedStateMachineManager::class.java.name} + METHOD onExternalStartFlow + AT ENTRY + IF flagged("commit_exception_flag") && !flagged("retry_exception_flag") + DO flag("retry_exception_flag"); traceln("Throwing retry exception"); throw new java.lang.RuntimeException("Here we go again") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(1, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(1, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + // 1 for the errored flow kept for observation and another for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws a [ConstraintViolationException] when performing an [Action.CommitTransaction] event when the flow is finishing. + * The exception is thrown 4 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it begins from the previous checkpoint where it suspended before failing. + */ + @Test + fun `error during transition with CommitTransaction action and ConstraintViolationException that occurs when completing a flow will retry and be kept for observation if error persists`() { + startDriver { + val charlie = createNode(CHARLIE_NAME) + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 4 + DO incrementCounter("counter"); + clear("remove_checkpoint_flag"); + traceln("Throwing exception"); + throw new org.hibernate.exception.ConstraintViolationException("This flow has a terminal condition", new java.sql.SQLException(), "made up constraint") + ENDRULE + + RULE Entering duplicate insert staff member + CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached duplicate insert staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment not my speciality counter + CLASS ${StaffedFlowHospital.DuplicateInsertSpecialist::class.java.name} + METHOD consult + AT READ NOT_MY_SPECIALTY + IF true + DO traceln("Byteman test - not my speciality") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - not my speciality") }.size) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + // 1 for errored flow and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event on a responding flow. The failure prevents the node from saving + * its original checkpoint. + * + * The exception is thrown 5 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + * + * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition + * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify + * that 3 retries are attempted before recovering. + */ + @Test + fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and complete successfully - responding flow`() { + startDriver { + val charlie = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 5 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(charlie) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + assertEquals(0, charlieClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event on a responding flow. The failure prevents the node from saving + * its original checkpoint. + * + * The exception is thrown 5 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times) and then be kept in for observation. + * + * Each time the flow retries, it starts from the beginning of the flow (due to being in an unstarted state). + * + * 2 of the thrown exceptions are absorbed by the if statement in [TransitionExecutorImpl.executeTransition] that aborts the transition + * if an error transition moves into another error transition. The flow still recovers from this state. 5 exceptions were thrown to verify + * that 3 retries are attempted before recovering. + * + * The final asserts for checking the checkpoints on the nodes are correct since the responding node can replay the flow starting events + * from artemis. Therefore, the checkpoint is missing due the failures from saving the original checkpoint. But, the node will still be + * able to recover when the node is restarted (by using the events). The initiating flow maintains the checkpoint as it is waiting for + * the responding flow to recover and finish its flow. + */ + @Test + fun `error during transition with CommitTransaction action that occurs during the beginning of execution will retry and be kept for observation if error persists - responding flow`() { + startDriver { + val charlie = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF readCounter("counter") < 7 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + } + + val output = getBytemanOutput(charlie) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + assertEquals(1, charlieClient.stateMachinesSnapshot().size) + // 1 for the flow that is waiting for the errored counterparty flow to finish and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for GetNumberOfCheckpointsFlow + // the checkpoint is not persisted since it kept failing the original checkpoint commit + // the flow will recover since artemis will keep the events and replay them on node restart + assertEquals(1, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when performing an [Action.CommitTransaction] event when the flow is finishing on a responding node. + * + * The exception is thrown 3 times. + * + * This causes the transition to be discharged from the hospital 3 times (retries 3 times). On the final retry the transition + * succeeds and the flow finishes. + */ + @Test + fun `error during transition with CommitTransaction action that occurs when completing a flow and deleting its checkpoint will retry and complete successfully - responding flow`() { + startDriver { + val charlie = createBytemanNode(CHARLIE_NAME) + val alice = createNode(ALICE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when adding action to remove checkpoint + CLASS ${TopLevelTransition::class.java.name} + METHOD flowFinishTransition + AT ENTRY + IF !flagged("remove_checkpoint_flag") + DO flag("remove_checkpoint_flag"); traceln("Setting remove checkpoint flag to true") + ENDRULE + + RULE Throw exception on executeCommitTransaction when removing checkpoint + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("remove_checkpoint_flag") && readCounter("counter") < 3 + DO incrementCounter("counter"); + clear("remove_checkpoint_flag"); + traceln("Throwing exception"); + throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()).returnValue.getOrThrow( + 30.seconds + ) + + val output = getBytemanOutput(charlie) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + assertEquals(0, charlieClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when recoding a transaction inside of [ReceiveFinalityFlow] on the responding + * flow's node. + * + * The flow is kept in for observation. + * + * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its + * send to the responding node and the responding node successfully received it. + */ + @Test + fun `error recording a transaction inside of ReceiveFinalityFlow will keep the flow in for observation`() { + startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { + val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + + // could not get rule for FinalityDoctor + observation counter to work + val rules = """ + RULE Set flag when entering receive finality flow + CLASS ${ReceiveFinalityFlow::class.java.name} + METHOD call + AT ENTRY + IF !flagged("finality_flag") + DO flag("finality_flag"); traceln("Setting finality flag") + ENDRULE + + RULE Set flag when leaving resolve transactions flow + CLASS ${ResolveTransactionsFlow::class.java.name} + METHOD call + AT EXIT + IF !flagged("resolve_tx_flag") + DO flag("resolve_tx_flag"); traceln("Setting resolve tx flag") + ENDRULE + + RULE Throw exception when recording transaction + INTERFACE ${ServiceHubInternal::class.java.name} + METHOD recordTransactions + AT ENTRY + IF flagged("finality_flag") && flagged("resolve_tx_flag") + DO traceln("Throwing exception"); + throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow( + ::CashIssueAndPaymentFlow, + 500.DOLLARS, + OpaqueBytes.of(0x01), + charlie.nodeInfo.singleIdentity(), + false, + defaultNotaryIdentity + ).returnValue.getOrThrow(30.seconds) + + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(0, discharge) + assertEquals(1, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + assertEquals(1, charlieClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when resolving a transaction's dependencies inside of [ReceiveFinalityFlow] on the responding + * flow's node. + * + * The flow is kept in for observation. + * + * Only the responding node keeps a checkpoint. The initiating flow has completed successfully as it has complete its + * send to the responding node and the responding node successfully received it. + */ + @Test + fun `error resolving a transaction's dependencies inside of ReceiveFinalityFlow will keep the flow in for observation`() { + startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { + val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + + // could not get rule for FinalityDoctor + observation counter to work + val rules = """ + RULE Set flag when entering receive finality flow + CLASS ${ReceiveFinalityFlow::class.java.name} + METHOD call + AT ENTRY + IF !flagged("finality_flag") + DO flag("finality_flag"); traceln("Setting finality flag") + ENDRULE + + RULE Set flag when entering resolve transactions flow + CLASS ${ResolveTransactionsFlow::class.java.name} + METHOD call + AT ENTRY + IF !flagged("resolve_tx_flag") + DO flag("resolve_tx_flag"); traceln("Setting resolve tx flag") + ENDRULE + + RULE Throw exception when recording transaction + INTERFACE ${ServiceHubInternal::class.java.name} + METHOD recordTransactions + AT ENTRY + IF flagged("finality_flag") && flagged("resolve_tx_flag") + DO traceln("Throwing exception"); + throw new java.lang.RuntimeException("die dammit die") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow( + ::CashIssueAndPaymentFlow, + 500.DOLLARS, + OpaqueBytes.of(0x01), + charlie.nodeInfo.singleIdentity(), + false, + defaultNotaryIdentity + ).returnValue.getOrThrow(30.seconds) + + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(0, discharge) + assertEquals(1, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + assertEquals(1, charlieClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding + * flow's node. + * + * The exception is thrown 5 times. + * + * The responding flow is retried 3 times and then completes successfully. + * + * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the + * flow is retried instead of moving straight to observation. + */ + @Test + fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and complete successfully`() { + startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { + val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when entering receive finality flow + CLASS ${ReceiveFinalityFlow::class.java.name} + METHOD call + AT ENTRY + IF !flagged("finality_flag") + DO flag("finality_flag"); traceln("Setting finality flag") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("finality_flag") && readCounter("counter") < 5 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + aliceClient.startFlow( + ::CashIssueAndPaymentFlow, + 500.DOLLARS, + OpaqueBytes.of(0x01), + charlie.nodeInfo.singleIdentity(), + false, + defaultNotaryIdentity + ).returnValue.getOrThrow(30.seconds) + + val output = getBytemanOutput(charlie) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + assertEquals(0, charlieClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Throws an exception when executing [Action.CommitTransaction] as part of receiving a transaction to record inside of [ReceiveFinalityFlow] on the responding + * flow's node. + * + * The exception is thrown 7 times. + * + * The responding flow is retried 3 times and is then kept in for observation. + * + * Both the initiating node and the responding node keep checkpoints for their flows. The initiating node keeps a checkpoint for the original flow that is + * waiting for the responding flow's receive to complete. The responding flow's checkpoint is kept due to it failing the commit as part of receive. + * + * The [StaffedFlowHospital.TransitionErrorGeneralPractitioner] catches these errors instead of the [StaffedFlowHospital.FinalityDoctor]. Due to this, the + * flow is retried instead of moving straight to observation. + */ + @Test + fun `error during transition with CommitTransaction action while receiving a transaction inside of ReceiveFinalityFlow will be retried and be kept for observation is error persists`() { + startDriver(notarySpec = NotarySpec(DUMMY_NOTARY_NAME, validating = false)) { + val charlie = createBytemanNode(CHARLIE_NAME, FINANCE_CORDAPPS) + val alice = createNode(ALICE_NAME, FINANCE_CORDAPPS) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Set flag when entering receive finality flow + CLASS ${ReceiveFinalityFlow::class.java.name} + METHOD call + AT ENTRY + IF !flagged("finality_flag") + DO flag("finality_flag"); traceln("Setting finality flag") + ENDRULE + + RULE Throw exception on executeCommitTransaction action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeCommitTransaction + AT ENTRY + IF flagged("finality_flag") && readCounter("counter") < 7 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + val charlieClient = + CordaRPCClient(charlie.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + assertFailsWith { + aliceClient.startFlow( + ::CashIssueAndPaymentFlow, + 500.DOLLARS, + OpaqueBytes.of(0x01), + charlie.nodeInfo.singleIdentity(), + false, + defaultNotaryIdentity + ).returnValue.getOrThrow(30.seconds) + } + + val output = getBytemanOutput(charlie) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) + val (discharge, observation) = charlieClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(1, aliceClient.stateMachinesSnapshot().size) + assertEquals(1, charlieClient.stateMachinesSnapshot().size) + // 1 for CashIssueAndPaymentFlow and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + // 1 for ReceiveFinalityFlow and 1 for GetNumberOfCheckpointsFlow + assertEquals(2, charlieClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Triggers `killFlow` while the flow is suspended causing a [InterruptedException] to be thrown and passed through the hospital. + * + * The flow terminates and is not retried. + * + * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. + */ + @Test + fun `error during transition due to an InterruptedException (killFlow) will terminate the flow`() { + startDriver { + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + + RULE Increment terminal counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ TERMINAL + IF true + DO traceln("Byteman test - terminal") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + val flow = aliceClient.startTrackedFlow(::SleepFlow) + + var flowKilled = false + flow.progress.subscribe { + if (it == SleepFlow.STARTED.label) { + Thread.sleep(5000) + flowKilled = aliceClient.killFlow(flow.id) + } + } + + assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } + + val output = getBytemanOutput(alice) + + assertTrue(flowKilled) + // Check the stdout for the lines generated by byteman + assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size + assertEquals(1, numberOfTerminalDiagnoses) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(0, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Triggers `killFlow` during user application code. + * + * The user application code is mimicked by a [Thread.sleep] which is importantly not placed inside the [Suspendable] + * call function. Placing it inside a [Suspendable] function causes quasar to behave unexpectedly. + * + * Although the call to kill the flow is made during user application code. It will not be removed / stop processing + * until the next suspension point is reached within the flow. + * + * The flow terminates and is not retried. + * + * No pass through the hospital is recorded. As the flow is marked as `isRemoved`. + */ + @Test + fun `flow killed during user code execution stops and removes the flow correctly`() { + startDriver { + val alice = createBytemanNode(ALICE_NAME) + + val rules = """ + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + + RULE Increment terminal counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ TERMINAL + IF true + DO traceln("Byteman test - terminal") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + val flow = aliceClient.startTrackedFlow(::ThreadSleepFlow) + + var flowKilled = false + flow.progress.subscribe { + if (it == ThreadSleepFlow.STARTED.label) { + Thread.sleep(5000) + flowKilled = aliceClient.killFlow(flow.id) + } + } + + assertFailsWith { flow.returnValue.getOrThrow(30.seconds) } + + val output = getBytemanOutput(alice) + + assertTrue(flowKilled) + // Check the stdout for the lines generated by byteman + assertEquals(0, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(0, output.filter { it.contains("Byteman test - overnight observation") }.size) + val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size + println(numberOfTerminalDiagnoses) + assertEquals(0, numberOfTerminalDiagnoses) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(0, discharge) + assertEquals(0, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + /** + * Triggers `killFlow` after the flow has already been sent to observation. The flow is not running at this point and + * all that remains is its checkpoint in the database. + * + * The flow terminates and is not retried. + * + * Killing the flow does not lead to any passes through the hospital. All the recorded passes through the hospital are + * from the original flow that was put in for observation. + */ + @Test + fun `flow killed when it is in the flow hospital for observation is removed correctly`() { + startDriver { + val alice = createBytemanNode(ALICE_NAME) + val charlie = createNode(CHARLIE_NAME) + + val rules = """ + RULE Create Counter + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF createCounter("counter", $counter) + DO traceln("Counter created") + ENDRULE + + RULE Throw exception on executeSendInitial action + CLASS ${ActionExecutorImpl::class.java.name} + METHOD executeSendInitial + AT ENTRY + IF readCounter("counter") < 4 + DO incrementCounter("counter"); traceln("Throwing exception"); throw new java.lang.RuntimeException("die dammit die") + ENDRULE + + RULE Entering internal error staff member + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT ENTRY + IF true + DO traceln("Reached internal transition error staff member") + ENDRULE + + RULE Increment discharge counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ DISCHARGE + IF true + DO traceln("Byteman test - discharging") + ENDRULE + + RULE Increment observation counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ OVERNIGHT_OBSERVATION + IF true + DO traceln("Byteman test - overnight observation") + ENDRULE + + RULE Increment terminal counter + CLASS ${StaffedFlowHospital.TransitionErrorGeneralPractitioner::class.java.name} + METHOD consult + AT READ TERMINAL + IF true + DO traceln("Byteman test - terminal") + ENDRULE + """.trimIndent() + + submitBytemanRules(rules) + + val aliceClient = + CordaRPCClient(alice.rpcAddress).start(rpcUser.username, rpcUser.password).proxy + + val flow = aliceClient.startFlow(::SendAMessageFlow, charlie.nodeInfo.singleIdentity()) + + assertFailsWith { flow.returnValue.getOrThrow(20.seconds) } + + aliceClient.killFlow(flow.id) + + val output = getBytemanOutput(alice) + + // Check the stdout for the lines generated by byteman + assertEquals(3, output.filter { it.contains("Byteman test - discharging") }.size) + assertEquals(1, output.filter { it.contains("Byteman test - overnight observation") }.size) + val numberOfTerminalDiagnoses = output.filter { it.contains("Byteman test - terminal") }.size + assertEquals(0, numberOfTerminalDiagnoses) + val (discharge, observation) = aliceClient.startFlow(::GetHospitalCountersFlow).returnValue.get() + assertEquals(3, discharge) + assertEquals(1, observation) + assertEquals(0, aliceClient.stateMachinesSnapshot().size) + // 1 for GetNumberOfCheckpointsFlow + assertEquals(1, aliceClient.startFlow(::GetNumberOfCheckpointsFlow).returnValue.get()) + } + } + + private fun startDriver(notarySpec: NotarySpec = NotarySpec(DUMMY_NOTARY_NAME), dsl: DriverDSL.() -> Unit) { + driver( + DriverParameters( + notarySpecs = listOf(notarySpec), + startNodesInProcess = false, + inMemoryDB = false, + systemProperties = mapOf("co.paralleluniverse.fibers.verifyInstrumentation" to "true") + ) + ) { + dsl() + } + } + + private fun DriverDSL.createBytemanNode( + providedName: CordaX500Name, + additionalCordapps: Collection = emptyList() + ): NodeHandle { + return (this as InternalDriverDSL).startNode( + NodeParameters( + providedName = providedName, + rpcUsers = listOf(rpcUser), + additionalCordapps = additionalCordapps + ), + bytemanPort = 12000 + ).getOrThrow() + } + + private fun DriverDSL.createNode(providedName: CordaX500Name, additionalCordapps: Collection = emptyList()): NodeHandle { + return startNode( + NodeParameters( + providedName = providedName, + rpcUsers = listOf(rpcUser), + additionalCordapps = additionalCordapps + ) + ).getOrThrow() + } + + private fun submitBytemanRules(rules: String) { + val submit = Submit("localhost", 12000) + submit.addScripts(listOf(ScriptText("Test script", rules))) + } + + private fun getBytemanOutput(nodeHandle: NodeHandle): List { + return nodeHandle.baseDirectory + .list() + .first { it.toString().contains("net.corda.node.Corda") && it.toString().contains("stdout.log") } + .readAllLines() + } +} + +@StartableByRPC +@InitiatingFlow +class SendAMessageFlow(private val party: Party) : FlowLogic() { + @Suspendable + override fun call(): String { + val session = initiateFlow(party) + session.send("hello there") + return "Finished executing test flow - ${this.runId}" + } +} + +@InitiatedBy(SendAMessageFlow::class) +class SendAMessageResponder(private val session: FlowSession) : FlowLogic() { + @Suspendable + override fun call() { + session.receive().unwrap { it } + } +} + +@StartableByRPC +class SleepFlow : FlowLogic() { + + object STARTED : ProgressTracker.Step("I am ready to die") + + override val progressTracker = ProgressTracker(STARTED) + + @Suspendable + override fun call() { + sleep(Duration.of(1, ChronoUnit.SECONDS)) + progressTracker.currentStep = STARTED + sleep(Duration.of(2, ChronoUnit.MINUTES)) + } +} + +@StartableByRPC +class ThreadSleepFlow : FlowLogic() { + + object STARTED : ProgressTracker.Step("I am ready to die") + + override val progressTracker = ProgressTracker(STARTED) + + @Suspendable + override fun call() { + sleep(Duration.of(1, ChronoUnit.SECONDS)) + progressTracker.currentStep = STARTED + logger.info("Starting ${ThreadSleepFlow::class.qualifiedName} application sleep") + sleep() + logger.info("Finished ${ThreadSleepFlow::class.qualifiedName} application sleep") + sleep(Duration.of(2, ChronoUnit.MINUTES)) + } + + // Sleep is moved outside of `@Suspendable` function to prevent issues with Quasar + private fun sleep() { + Thread.sleep(20000) + } +} + +@StartableByRPC +class GetNumberOfCheckpointsFlow : FlowLogic() { + override fun call(): Long { + return serviceHub.jdbcSession().prepareStatement("select count(*) from node_checkpoints").use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + } + } +} + +@StartableByRPC +class GetHospitalCountersFlow : FlowLogic() { + override fun call(): HospitalCounts = HospitalCounts( + serviceHub.cordaService(HospitalCounter::class.java).dischargeCounter, + serviceHub.cordaService(HospitalCounter::class.java).observationCounter + ) +} + +@CordaSerializable +data class HospitalCounts(val discharge: Int, val observation: Int) + +@Suppress("UNUSED_PARAMETER") +@CordaService +class HospitalCounter(services: AppServiceHub) : SingletonSerializeAsToken() { + var observationCounter: Int = 0 + var dischargeCounter: Int = 0 + + init { + StaffedFlowHospital.onFlowDischarged.add { _, _ -> + ++dischargeCounter + } + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + ++observationCounter + } + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt new file mode 100644 index 0000000000..06c3bc1acf --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/services/vault/VaultObserverExceptionTest.kt @@ -0,0 +1,313 @@ +package net.corda.node.services.vault + +import com.r3.dbfailure.workflows.CreateStateFlow +import com.r3.dbfailure.workflows.CreateStateFlow.Initiator +import com.r3.dbfailure.workflows.CreateStateFlow.errorTargetsToNum +import net.corda.core.CordaRuntimeException +import net.corda.core.internal.concurrent.openFuture +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.getOrThrow +import net.corda.core.utilities.seconds +import net.corda.node.services.Permissions +import net.corda.node.services.statemachine.StaffedFlowHospital +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.User +import net.corda.testing.node.internal.findCordapp +import org.junit.After +import org.junit.Assert +import org.junit.Test +import rx.exceptions.OnErrorNotImplementedException +import java.sql.SQLException +import java.time.Duration +import java.time.temporal.ChronoUnit +import java.util.concurrent.TimeoutException +import javax.persistence.PersistenceException +import kotlin.test.assertFailsWith + +class VaultObserverExceptionTest { + companion object { + + val log = contextLogger() + + private fun testCordapps() = listOf( + findCordapp("com.r3.dbfailure.contracts"), + findCordapp("com.r3.dbfailure.workflows"), + findCordapp("com.r3.dbfailure.schemas")) + } + + @After + fun tearDown() { + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.clear() + StaffedFlowHospital.onFlowKeptForOvernightObservation.clear() + StaffedFlowHospital.onFlowAdmitted.clear() + } + + /** + * Causing an SqlException via a syntax error in a vault observer causes the flow to hit the + * DatabsaseEndocrinologist in the FlowHospital and being kept for overnight observation + */ + @Test + fun unhandledSqlExceptionFromVaultObserverGetsHospitatlised() { + val testControlFuture = openFuture().toCompletableFuture() + + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { + when (it) { + is OnErrorNotImplementedException -> Assert.fail("OnErrorNotImplementedException should be unwrapped") + is SQLException -> { + testControlFuture.complete(true) + } + } + false + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + aliceNode.rpc.startFlow( + ::Initiator, + "Syntax Error in Custom SQL", + CreateStateFlow.errorTargetsToNum(CreateStateFlow.ErrorTarget.ServiceSqlSyntaxError) + ).returnValue.then { testControlFuture.complete(false) } + val foundExpectedException = testControlFuture.getOrThrow(30.seconds) + + Assert.assertTrue(foundExpectedException) + } + } + + /** + * Throwing a random (non-SQL releated) exception from a vault observer causes the flow to be + * aborted when unhandled in user code + */ + @Test + fun otherExceptionsFromVaultObserverBringFlowDown() { + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + assertFailsWith(CordaRuntimeException::class, "Toys out of pram") { + aliceNode.rpc.startFlow( + ::Initiator, + "InvalidParameterException", + CreateStateFlow.errorTargetsToNum(CreateStateFlow.ErrorTarget.ServiceThrowInvalidParameter) + ).returnValue.getOrThrow(30.seconds) + } + } + } + + /** + * A random exception from a VaultObserver will bring the Rx Observer down, but can be handled in the flow + * triggering the observer, and the flow will continue successfully (for some values of success) + */ + @Test + fun otherExceptionsFromVaultObserverCanBeSuppressedInFlow() { + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + aliceNode.rpc.startFlow(::Initiator, "InvalidParameterException", CreateStateFlow.errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceThrowInvalidParameter, + CreateStateFlow.ErrorTarget.FlowSwallowErrors)) + .returnValue.getOrThrow(30.seconds) + + } + } + + /** + * If the state we are trying to persist triggers a persistence exception, the flow hospital will retry the flow + * and keep it in for observation if errors persist. + */ + @Test + fun persistenceExceptionOnCommitGetsRetriedAndThenGetsKeptForObservation() { + var admitted = 0 + var observation = 0 + StaffedFlowHospital.onFlowAdmitted.add { + ++admitted + } + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + ++observation + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + assertFailsWith { + aliceNode.rpc.startFlow(::Initiator, "EntityManager", errorTargetsToNum(CreateStateFlow.ErrorTarget.TxInvalidState)) + .returnValue.getOrThrow(Duration.of(30, ChronoUnit.SECONDS)) + } + } + Assert.assertTrue("Exception from service has not been to Hospital", admitted > 0) + Assert.assertEquals(1, observation) + } + + /** + * If we have a state causing a database error lined up for persistence, calling jdbConnection() in + * the vault observer will trigger a flush that throws. This will be kept in for observation. + */ + @Test + fun persistenceExceptionOnFlushGetsRetriedAndThenGetsKeptForObservation() { + var counter = 0 + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { + when (it) { + is OnErrorNotImplementedException -> Assert.fail("OnErrorNotImplementedException should be unwrapped") + is PersistenceException -> { + ++counter + log.info("Got a PersistentException in the flow hospital count = $counter") + } + } + false + } + var observation = 0 + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + ++observation + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + assertFailsWith("PersistenceException") { + aliceNode.rpc.startFlow(::Initiator, "EntityManager", errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceValidUpdate, + CreateStateFlow.ErrorTarget.TxInvalidState)) + .returnValue.getOrThrow(30.seconds) + } + } + Assert.assertTrue("Flow has not been to hospital", counter > 0) + Assert.assertEquals(1, observation) + } + + /** + * If we have a state causing a database error lined up for persistence, calling jdbConnection() in + * the vault observer will trigger a flush that throws. + * Trying to catch and suppress that exception in the flow around the code triggering the vault observer + * does not change the outcome - the first exception in the service will bring the service down and will + * be caught by the flow, but the state machine will error the flow anyway as Corda code threw. + */ + @Test + fun persistenceExceptionOnFlushInVaultObserverCannotBeSuppressedInFlow() { + var counter = 0 + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { + when (it) { + is OnErrorNotImplementedException -> Assert.fail("OnErrorNotImplementedException should be unwrapped") + is PersistenceException -> { + ++counter + log.info("Got a PersistentException in the flow hospital count = $counter") + } + } + false + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + val flowHandle = aliceNode.rpc.startFlow( + ::Initiator, + "EntityManager", + CreateStateFlow.errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceValidUpdate, + CreateStateFlow.ErrorTarget.TxInvalidState, + CreateStateFlow.ErrorTarget.FlowSwallowErrors)) + val flowResult = flowHandle.returnValue + assertFailsWith("PersistenceException") { flowResult.getOrThrow(30.seconds) } + Assert.assertTrue("Flow has not been to hospital", counter > 0) + } + } + + /** + * If we have a state causing a persistence exception lined up for persistence, calling jdbConnection() in + * the vault observer will trigger a flush that throws. + * Trying to catch and suppress that exception inside the service does protect the service, but the new + * interceptor will fail the flow anyway. The flow will be kept in for observation if errors persist. + */ + @Test + fun persistenceExceptionOnFlushInVaultObserverCannotBeSuppressedInService() { + var counter = 0 + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { + when (it) { + is OnErrorNotImplementedException -> Assert.fail("OnErrorNotImplementedException should be unwrapped") + is PersistenceException -> { + ++counter + log.info("Got a PersistentException in the flow hospital count = $counter") + } + } + false + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + val flowHandle = aliceNode.rpc.startFlow( + ::Initiator, "EntityManager", + CreateStateFlow.errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceValidUpdate, + CreateStateFlow.ErrorTarget.TxInvalidState, + CreateStateFlow.ErrorTarget.ServiceSwallowErrors)) + val flowResult = flowHandle.returnValue + assertFailsWith("PersistenceException") { flowResult.getOrThrow(30.seconds) } + Assert.assertTrue("Flow has not been to hospital", counter > 0) + } + } + + /** + * User code throwing a syntax error in a raw vault observer will break the recordTransaction call, + * therefore handling it in flow code is no good, and the error will be passed to the flow hospital via the + * interceptor. + */ + @Test + fun syntaxErrorInUserCodeInServiceCannotBeSuppressedInFlow() { + val testControlFuture = openFuture() + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> + log.info("Flow has been kept for overnight observation") + testControlFuture.set(true) + } + + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + val flowHandle = aliceNode.rpc.startFlow(::Initiator, "EntityManager", CreateStateFlow.errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceSqlSyntaxError, + CreateStateFlow.ErrorTarget.FlowSwallowErrors)) + val flowResult = flowHandle.returnValue + flowResult.then { + log.info("Flow has finished") + testControlFuture.set(false) + } + Assert.assertTrue("Flow has not been kept in hospital", testControlFuture.getOrThrow(30.seconds)) + } + } + + /** + * User code throwing a syntax error and catching suppressing that within the observer code is fine + * and should not have any impact on the rest of the flow + */ + @Test + fun syntaxErrorInUserCodeInServiceCanBeSuppressedInService() { + driver(DriverParameters( + startNodesInProcess = true, + cordappsForAllNodes = testCordapps())) { + val aliceUser = User("user", "foo", setOf(Permissions.all())) + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(aliceUser)).getOrThrow() + val flowHandle = aliceNode.rpc.startFlow(::Initiator, "EntityManager", CreateStateFlow.errorTargetsToNum( + CreateStateFlow.ErrorTarget.ServiceSqlSyntaxError, + CreateStateFlow.ErrorTarget.ServiceSwallowErrors)) + val flowResult = flowHandle.returnValue + flowResult.getOrThrow(30.seconds) + } + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMessagingTest.kt b/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMessagingTest.kt index f23aeb1a4d..b6c7e8bcd2 100644 --- a/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMessagingTest.kt +++ b/node/src/integration-test/kotlin/net/corda/services/messaging/P2PMessagingTest.kt @@ -21,6 +21,7 @@ import net.corda.testing.driver.internal.internalServices import net.corda.testing.node.ClusterSpec import net.corda.testing.node.NotarySpec import org.assertj.core.api.Assertions.assertThat +import org.junit.Ignore import org.junit.Test import java.util.* import java.util.concurrent.atomic.AtomicBoolean @@ -31,6 +32,7 @@ class P2PMessagingTest { } @Test + @Ignore fun `communicating with a distributed service which we're part of`() { startDriverWithDistributedService { distributedService -> assertAllNodesAreUsed(distributedService, DISTRIBUTED_SERVICE_NAME, distributedService[0]) diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 8918ee1026..ebf1d1f7a0 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -1190,6 +1190,7 @@ class FlowStarterImpl(private val smm: StateMachineManager, private val flowLogi override val deduplicationHandler: DeduplicationHandler get() = this + override val flowId: StateMachineRunId = StateMachineRunId.createRandom() override val flowLogic: FlowLogic get() = logic override val context: InvocationContext @@ -1232,8 +1233,17 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig, @Suppress("DEPRECATION") org.hibernate.type.descriptor.java.JavaTypeDescriptorRegistry.INSTANCE.addDescriptor(AbstractPartyDescriptor(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) val attributeConverters = listOf(PublicKeyToTextConverter(), AbstractPartyToX500NameAsStringConverter(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) + val jdbcUrl = hikariProperties.getProperty("dataSource.url", "") - return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters, customClassLoader) + return CordaPersistence( + databaseConfig, + schemaService.schemaOptions.keys, + jdbcUrl, + cacheFactory, + attributeConverters, customClassLoader, + errorHandler = { t -> + FlowStateMachineImpl.currentStateMachine()?.scheduleEvent(Event.Error(t)) + }) } fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set, metricRegistry: MetricRegistry? = null, cordappLoader: CordappLoader? = null, currentDir: Path? = null, ourName: CordaX500Name) { diff --git a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt index 024fcf74ca..c7640267ab 100644 --- a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt +++ b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt @@ -29,6 +29,7 @@ object NodeInfoSchemaV1 : MappedSchema( @Column(name = "node_info_id", nullable = false) var id: Int, + @Suppress("MagicNumber") // database column width @Column(name = "node_info_hash", length = 64, nullable = false) val hash: String, diff --git a/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt b/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt index 0ea8fcacb3..57c3254cfc 100644 --- a/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt +++ b/node/src/main/kotlin/net/corda/node/services/events/NodeSchedulerService.kt @@ -11,6 +11,7 @@ import net.corda.core.contracts.ScheduledStateRef import net.corda.core.contracts.StateRef import net.corda.core.flows.FlowLogic import net.corda.core.flows.FlowLogicRefFactory +import net.corda.core.flows.StateMachineRunId import net.corda.core.internal.* import net.corda.core.internal.concurrent.flatMap import net.corda.core.internal.concurrent.openFuture @@ -239,6 +240,7 @@ class NodeSchedulerService(private val clock: CordaClock, } private inner class FlowStartDeduplicationHandler(val scheduledState: ScheduledStateRef, override val flowLogic: FlowLogic, override val context: InvocationContext) : DeduplicationHandler, ExternalEvent.ExternalStartFlowEvent { + override val flowId: StateMachineRunId = StateMachineRunId.createRandom() override val externalCause: ExternalEvent get() = this override val deduplicationHandler: FlowStartDeduplicationHandler diff --git a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt index b86355302c..6e896a6d22 100644 --- a/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt +++ b/node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt @@ -2,12 +2,16 @@ package net.corda.node.services.identity import net.corda.core.crypto.Crypto import net.corda.core.crypto.toStringShort -import net.corda.core.identity.* +import net.corda.core.identity.AbstractParty +import net.corda.core.identity.AnonymousParty +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party +import net.corda.core.identity.PartyAndCertificate +import net.corda.core.identity.x500Matches import net.corda.core.internal.CertRole import net.corda.core.internal.NamedCacheFactory import net.corda.core.internal.hash import net.corda.core.internal.toSet -import net.corda.core.node.services.IdentityService import net.corda.core.node.services.UnknownAnonymousPartyException import net.corda.core.serialization.SingletonSerializeAsToken import net.corda.core.utilities.MAX_HASH_HEX_SIZE @@ -29,13 +33,18 @@ import org.hibernate.annotations.Type import org.hibernate.internal.util.collections.ArrayHelper.EMPTY_BYTE_ARRAY import java.security.InvalidAlgorithmParameterException import java.security.PublicKey -import java.security.cert.* +import java.security.cert.CertPathValidatorException +import java.security.cert.CertStore +import java.security.cert.CertificateExpiredException +import java.security.cert.CertificateNotYetValidException +import java.security.cert.CollectionCertStoreParameters +import java.security.cert.TrustAnchor +import java.security.cert.X509Certificate import java.util.* import javax.annotation.concurrent.ThreadSafe import javax.persistence.Column import javax.persistence.Entity import javax.persistence.Id -import kotlin.IllegalStateException import kotlin.collections.HashSet import kotlin.streams.toList @@ -147,6 +156,7 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri @javax.persistence.Table(name = NAME_TO_HASH_TABLE_NAME) class PersistentPartyToPublicKeyHash( @Id + @Suppress("MagicNumber") // database column width @Column(name = NAME_COLUMN_NAME, length = 128, nullable = false) var name: String = "", diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessageDeduplicator.kt b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessageDeduplicator.kt index a5c2975e2a..4a63927fad 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessageDeduplicator.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessageDeduplicator.kt @@ -85,6 +85,7 @@ class P2PMessageDeduplicator(cacheFactory: NamedCacheFactory, private val databa } @Entity + @Suppress("MagicNumber") // database column width @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}message_ids") class ProcessedMessage( @Id diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt index c89589324c..5cb458a676 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/P2PMessagingClient.kt @@ -3,6 +3,7 @@ package net.corda.node.services.messaging import co.paralleluniverse.fibers.Suspendable import com.codahale.metrics.MetricRegistry import net.corda.core.crypto.toStringShort +import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.CordaX500Name import net.corda.core.internal.NamedCacheFactory import net.corda.core.internal.ThreadBox @@ -424,6 +425,7 @@ class P2PMessagingClient(val config: NodeConfiguration, private inner class MessageDeduplicationHandler(val artemisMessage: ClientMessage, override val receivedMessage: ReceivedMessage) : DeduplicationHandler, ExternalEvent.ExternalMessageEvent { override val externalCause: ExternalEvent get() = this + override val flowId: StateMachineRunId = StateMachineRunId.createRandom() override val deduplicationHandler: MessageDeduplicationHandler get() = this diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index d2f0d3163f..b1eec763f6 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -29,6 +29,7 @@ class DBCheckpointStorage : CheckpointStorage { @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}checkpoints") class DBCheckpoint( @Id + @Suppress("MagicNumber") // database column width @Column(name = "checkpoint_id", length = 64, nullable = false) var checkpointId: String = "", diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBTransactionStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBTransactionStorage.kt index b82f5f522b..14f7492139 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBTransactionStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBTransactionStorage.kt @@ -30,6 +30,7 @@ import kotlin.streams.toList class DBTransactionStorage(private val database: CordaPersistence, cacheFactory: NamedCacheFactory) : WritableTransactionStorage, SingletonSerializeAsToken() { + @Suppress("MagicNumber") // database column width @Entity @Table(name = "${NODE_DATABASE_PREFIX}transactions") class DBTransaction( diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt index e1551d5732..237b1097f7 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/ActionExecutorImpl.kt @@ -120,10 +120,21 @@ class ActionExecutorImpl( } } + @Suppress("TooGenericExceptionCaught") // this is fully intentional here, see comment in the catch clause @Suspendable private fun executeAcknowledgeMessages(action: Action.AcknowledgeMessages) { action.deduplicationHandlers.forEach { - it.afterDatabaseTransaction() + try { + it.afterDatabaseTransaction() + } catch (e: Exception) { + // Catch all exceptions that occur in the [DeduplicationHandler]s (although errors should be unlikely) + // It is deemed safe for errors to occur here + // Therefore the current transition should not fail if something does go wrong + log.info( + "An error occurred executing a deduplication post-database commit handler. Continuing, as it is safe to do so.", + e + ) + } } } @@ -218,17 +229,24 @@ class ActionExecutorImpl( } } + @Suppress("TooGenericExceptionCaught") // this is fully intentional here, see comment in the catch clause @Suspendable private fun executeAsyncOperation(fiber: FlowFiber, action: Action.ExecuteAsyncOperation) { - val operationFuture = action.operation.execute(action.deduplicationId) - operationFuture.thenMatch( - success = { result -> - fiber.scheduleEvent(Event.AsyncOperationCompletion(result)) - }, - failure = { exception -> - fiber.scheduleEvent(Event.Error(exception)) - } - ) + try { + val operationFuture = action.operation.execute(action.deduplicationId) + operationFuture.thenMatch( + success = { result -> + fiber.scheduleEvent(Event.AsyncOperationCompletion(result)) + }, + failure = { exception -> + fiber.scheduleEvent(Event.Error(exception)) + } + ) + } catch (e: Exception) { + // Catch and wrap any unexpected exceptions from the async operation + // Wrapping the exception allows it to be better handled by the flow hospital + throw AsyncOperationTransitionException(e) + } } private fun executeRetryFlowFromSafePoint(action: Action.RetryFlowFromSafePoint) { diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt index e85a7347b3..a8ba2dc4e6 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/SingleThreadedStateMachineManager.kt @@ -16,6 +16,7 @@ import net.corda.core.identity.Party import net.corda.core.internal.* import net.corda.core.internal.concurrent.OpenFuture import net.corda.core.internal.concurrent.map +import net.corda.core.internal.concurrent.mapError import net.corda.core.internal.concurrent.openFuture import net.corda.core.messaging.DataFeed import net.corda.core.serialization.SerializedBytes @@ -113,7 +114,7 @@ class SingleThreadedStateMachineManager( private var checkpointSerializationContext: CheckpointSerializationContext? = null private var actionExecutor: ActionExecutor? = null - override val flowHospital: StaffedFlowHospital = StaffedFlowHospital(flowMessaging, ourSenderUUID) + override val flowHospital: StaffedFlowHospital = makeFlowHospital() private val transitionExecutor = makeTransitionExecutor() override val allStateMachines: List> @@ -210,12 +211,14 @@ class SingleThreadedStateMachineManager( } private fun startFlow( + flowId: StateMachineRunId, flowLogic: FlowLogic, context: InvocationContext, ourIdentity: Party?, deduplicationHandler: DeduplicationHandler? ): CordaFuture> { return startFlowInternal( + flowId, invocationContext = context, flowLogic = flowLogic, flowStart = FlowStart.Explicit, @@ -230,7 +233,10 @@ class SingleThreadedStateMachineManager( cancelTimeoutIfScheduled(id) val flow = flows.remove(id) if (flow != null) { - logger.debug("Killing flow known to physical node.") + flow.fiber.transientState?.let { + flow.fiber.transientState = TransientReference(it.value.copy(isRemoved = true)) + } + logger.info("Killing flow $id known to this node.") decrementLiveFibers() totalFinishedFlows.inc() try { @@ -239,6 +245,7 @@ class SingleThreadedStateMachineManager( } finally { database.transaction { checkpointStorage.removeCheckpoint(id) + serviceHub.vaultService.softLockRelease(id.uuid) } transitionExecutor.forceRemoveFlow(id) unfinishedFibers.countDown() @@ -343,58 +350,71 @@ class SingleThreadedStateMachineManager( } } + @Suppress("TooGenericExceptionCaught", "ComplexMethod", "MaxLineLength") // this is fully intentional here, see comment in the catch clause override fun retryFlowFromSafePoint(currentState: StateMachineState) { // Get set of external events val flowId = currentState.flowLogic.runId - val oldFlowLeftOver = mutex.locked { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue - if (oldFlowLeftOver == null) { - logger.error("Unable to find flow for flow $flowId. Something is very wrong. The flow will not retry.") - return - } - val flow = if (currentState.isAnyCheckpointPersisted) { - // We intentionally grab the checkpoint from storage rather than relying on the one referenced by currentState. This is so that - // we mirror exactly what happens when restarting the node. - val serializedCheckpoint = checkpointStorage.getCheckpoint(flowId) - if (serializedCheckpoint == null) { - logger.error("Unable to find database checkpoint for flow $flowId. Something is very wrong. The flow will not retry.") + try { + val oldFlowLeftOver = mutex.locked { flows[flowId] }?.fiber?.transientValues?.value?.eventQueue + if (oldFlowLeftOver == null) { + logger.error("Unable to find flow for flow $flowId. Something is very wrong. The flow will not retry.") return } - // Resurrect flow - createFlowFromCheckpoint( + val flow = if (currentState.isAnyCheckpointPersisted) { + // We intentionally grab the checkpoint from storage rather than relying on the one referenced by currentState. This is so that + // we mirror exactly what happens when restarting the node. + val serializedCheckpoint = checkpointStorage.getCheckpoint(flowId) + if (serializedCheckpoint == null) { + logger.error("Unable to find database checkpoint for flow $flowId. Something is very wrong. The flow will not retry.") + return + } + // Resurrect flow + createFlowFromCheckpoint( id = flowId, serializedCheckpoint = serializedCheckpoint, initialDeduplicationHandler = null, isAnyCheckpointPersisted = true, isStartIdempotent = false - ) ?: return - } else { - // Just flow initiation message - null - } - mutex.locked { - if (stopping) { - return + ) ?: return + } else { + // Just flow initiation message + null } - // Remove any sessions the old flow has. - for (sessionId in getFlowSessionIds(currentState.checkpoint)) { - sessionToFlow.remove(sessionId) - } - if (flow != null) { - injectOldProgressTracker(currentState.flowLogic.progressTracker, flow.fiber.logic) - addAndStartFlow(flowId, flow) - } - // Deliver all the external events from the old flow instance. - val unprocessedExternalEvents = mutableListOf() - do { - val event = oldFlowLeftOver.tryReceive() - if (event is Event.GeneratedByExternalEvent) { - unprocessedExternalEvents += event.deduplicationHandler.externalCause + mutex.locked { + if (stopping) { + return + } + // Remove any sessions the old flow has. + for (sessionId in getFlowSessionIds(currentState.checkpoint)) { + sessionToFlow.remove(sessionId) + } + if (flow != null) { + injectOldProgressTracker(currentState.flowLogic.progressTracker, flow.fiber.logic) + addAndStartFlow(flowId, flow) + } + // Deliver all the external events from the old flow instance. + val unprocessedExternalEvents = mutableListOf() + do { + val event = oldFlowLeftOver.tryReceive() + if (event is Event.GeneratedByExternalEvent) { + unprocessedExternalEvents += event.deduplicationHandler.externalCause + } + } while (event != null) + val externalEvents = currentState.pendingDeduplicationHandlers.map { it.externalCause } + unprocessedExternalEvents + for (externalEvent in externalEvents) { + deliverExternalEvent(externalEvent) } - } while (event != null) - val externalEvents = currentState.pendingDeduplicationHandlers.map { it.externalCause } + unprocessedExternalEvents - for (externalEvent in externalEvents) { - deliverExternalEvent(externalEvent) } + } catch (e: Exception) { + // Failed to retry - manually put the flow in for observation rather than + // relying on the [HospitalisingInterceptor] to do so + val exceptions = (currentState.checkpoint.errorState as? ErrorState.Errored) + ?.errors + ?.map { it.exception } + ?.plus(e) ?: emptyList() + logger.info("Failed to retry flow $flowId, keeping in for observation and aborting") + flowHospital.forceIntoOvernightObservation(flowId, exceptions) + throw e } } @@ -410,7 +430,13 @@ class SingleThreadedStateMachineManager( } private fun onExternalStartFlow(event: ExternalEvent.ExternalStartFlowEvent) { - val future = startFlow(event.flowLogic, event.context, ourIdentity = null, deduplicationHandler = event.deduplicationHandler) + val future = startFlow( + event.flowId, + event.flowLogic, + event.context, + ourIdentity = null, + deduplicationHandler = event.deduplicationHandler + ) event.wireUpFuture(future) } @@ -476,7 +502,16 @@ class SingleThreadedStateMachineManager( is InitiatedFlowFactory.Core -> event.receivedMessage.platformVersion is InitiatedFlowFactory.CorDapp -> null } - startInitiatedFlow(flowLogic, event.deduplicationHandler, senderSession, initiatedSessionId, sessionMessage, senderCoreFlowVersion, initiatedFlowInfo) + startInitiatedFlow( + event.flowId, + flowLogic, + event.deduplicationHandler, + senderSession, + initiatedSessionId, + sessionMessage, + senderCoreFlowVersion, + initiatedFlowInfo + ) } catch (t: Throwable) { logger.warn("Unable to initiate flow from $sender (appName=${sessionMessage.appName} " + "flowVersion=${sessionMessage.flowVersion}), sending to the flow hospital", t) @@ -503,7 +538,9 @@ class SingleThreadedStateMachineManager( return serviceHub.getFlowFactory(initiatorFlowClass) ?: throw SessionRejectException.NotRegistered(initiatorFlowClass) } + @Suppress("LongParameterList") private fun startInitiatedFlow( + flowId: StateMachineRunId, flowLogic: FlowLogic, initiatingMessageDeduplicationHandler: DeduplicationHandler, peerSession: FlowSessionImpl, @@ -515,13 +552,19 @@ class SingleThreadedStateMachineManager( val flowStart = FlowStart.Initiated(peerSession, initiatedSessionId, initiatingMessage, senderCoreFlowVersion, initiatedFlowInfo) val ourIdentity = ourFirstIdentity startFlowInternal( - InvocationContext.peer(peerSession.counterparty.name), flowLogic, flowStart, ourIdentity, + flowId, + InvocationContext.peer(peerSession.counterparty.name), + flowLogic, + flowStart, + ourIdentity, initiatingMessageDeduplicationHandler, isStartIdempotent = false ) } + @Suppress("LongParameterList") private fun startFlowInternal( + flowId: StateMachineRunId, invocationContext: InvocationContext, flowLogic: FlowLogic, flowStart: FlowStart, @@ -529,7 +572,6 @@ class SingleThreadedStateMachineManager( deduplicationHandler: DeduplicationHandler?, isStartIdempotent: Boolean ): CordaFuture> { - val flowId = StateMachineRunId.createRandom() // Before we construct the state machine state by freezing the FlowLogic we need to make sure that lazy properties // have access to the fiber (and thereby the service hub) @@ -541,22 +583,44 @@ class SingleThreadedStateMachineManager( val flowCorDappVersion = createSubFlowVersion(serviceHub.cordappProvider.getCordappForFlow(flowLogic), serviceHub.myInfo.platformVersion) - val initialCheckpoint = Checkpoint.create( - invocationContext, - flowStart, - flowLogic.javaClass, - frozenFlowLogic, - ourIdentity, - flowCorDappVersion, - flowLogic.isEnabledTimedFlow() + val flowAlreadyExists = mutex.locked { flows[flowId] != null } + + val existingCheckpoint = if (flowAlreadyExists) { + // Load the flow's checkpoint + // The checkpoint will be missing if the flow failed before persisting the original checkpoint + // CORDA-3359 - Do not start/retry a flow that failed after deleting its checkpoint (the whole of the flow might replay) + checkpointStorage.getCheckpoint(flowId)?.let { serializedCheckpoint -> + val checkpoint = tryCheckpointDeserialize(serializedCheckpoint, flowId) + if (checkpoint == null) { + return openFuture>().mapError { + IllegalStateException("Unable to deserialize database checkpoint for flow $flowId. " + + "Something is very wrong. The flow will not retry.") + } + } else { + checkpoint + } + } + } else { + // This is a brand new flow + null + } + val checkpoint = existingCheckpoint ?: Checkpoint.create( + invocationContext, + flowStart, + flowLogic.javaClass, + frozenFlowLogic, + ourIdentity, + flowCorDappVersion, + flowLogic.isEnabledTimedFlow() ).getOrThrow() + val startedFuture = openFuture() val initialState = StateMachineState( - checkpoint = initialCheckpoint, + checkpoint = checkpoint, pendingDeduplicationHandlers = deduplicationHandler?.let { listOf(it) } ?: emptyList(), isFlowResumed = false, isTransactionTracked = false, - isAnyCheckpointPersisted = false, + isAnyCheckpointPersisted = existingCheckpoint != null, isStartIdempotent = isStartIdempotent, isRemoved = false, flowLogic = flowLogic, @@ -817,6 +881,12 @@ class SingleThreadedStateMachineManager( return interceptors.fold(transitionExecutor) { executor, interceptor -> interceptor(executor) } } + private fun makeFlowHospital() : StaffedFlowHospital { + // If the node is running as a notary service, we don't retain errored session initiation requests in case of missing Cordapps + // to avoid memory leaks if the notary is under heavy load. + return StaffedFlowHospital(flowMessaging, serviceHub.clock, ourSenderUUID) + } + private fun InnerState.removeFlowOrderly( flow: Flow, removalReason: FlowRemovalReason.OrderlyFinish, diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt index 476bd0829f..f8bdf269f8 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StaffedFlowHospital.kt @@ -10,48 +10,103 @@ import net.corda.core.identity.Party import net.corda.core.internal.DeclaredField import net.corda.core.internal.ThreadBox import net.corda.core.internal.TimedFlow +import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.bufferUntilSubscribed import net.corda.core.messaging.DataFeed import net.corda.core.utilities.contextLogger +import net.corda.core.utilities.debug +import net.corda.core.utilities.minutes import net.corda.core.utilities.seconds import net.corda.node.services.FinalityHandler import org.hibernate.exception.ConstraintViolationException import rx.subjects.PublishSubject import java.sql.SQLException import java.sql.SQLTransientConnectionException +import java.time.Clock import java.time.Duration import java.time.Instant import java.util.* +import java.util.concurrent.ConcurrentHashMap +import javax.persistence.PersistenceException +import kotlin.concurrent.timerTask import kotlin.math.pow /** * This hospital consults "staff" to see if they can automatically diagnose and treat flows. */ -class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val ourSenderUUID: String) { - private companion object { +class StaffedFlowHospital(private val flowMessaging: FlowMessaging, + private val clock: Clock, + private val ourSenderUUID: String) { + companion object { private val log = contextLogger() private val staff = listOf( - DeadlockNurse, - DuplicateInsertSpecialist, - DoctorTimeout, - FinalityDoctor, - TransientConnectionCardiologist + DeadlockNurse, + DuplicateInsertSpecialist, + DoctorTimeout, + FinalityDoctor, + TransientConnectionCardiologist, + DatabaseEndocrinologist, + TransitionErrorGeneralPractitioner ) + + @VisibleForTesting + val onFlowKeptForOvernightObservation = mutableListOf<(id: StateMachineRunId, by: List) -> Unit>() + + @VisibleForTesting + val onFlowDischarged = mutableListOf<(id: StateMachineRunId, by: List) -> Unit>() + + @VisibleForTesting + val onFlowAdmitted = mutableListOf<(id: StateMachineRunId) -> Unit>() } + private val hospitalJobTimer = Timer("FlowHospitalJobTimer", true) + + init { + // Register a task to log (at intervals) flows that are kept in hospital for overnight observation. + hospitalJobTimer.scheduleAtFixedRate(timerTask { + mutex.locked { + if (flowsInHospital.isNotEmpty()) { + // Get patients whose last record in their medical records is Outcome.OVERNIGHT_OBSERVATION. + val patientsUnderOvernightObservation = + flowsInHospital.filter { flowPatients[it.key]?.records?.last()?.outcome == Outcome.OVERNIGHT_OBSERVATION } + if (patientsUnderOvernightObservation.isNotEmpty()) + log.warn("There are ${patientsUnderOvernightObservation.count()} flows kept for overnight observation. " + + "Affected flow ids: ${patientsUnderOvernightObservation.map { it.key.uuid.toString() }.joinToString()}") + } + if (treatableSessionInits.isNotEmpty()) { + log.warn("There are ${treatableSessionInits.count()} erroneous session initiations kept for overnight observation. " + + "Erroneous session initiation ids: ${treatableSessionInits.map { it.key.toString() }.joinToString()}") + } + } + }, 1.minutes.toMillis(), 1.minutes.toMillis()) + } + + /** + * Represents the flows that have been admitted to the hospital for treatment. + * Flows should be removed from [flowsInHospital] when they have completed a successful transition. + */ + private val flowsInHospital = ConcurrentHashMap() + private val mutex = ThreadBox(object { + /** + * Contains medical history of every flow (a patient) that has entered the hospital. A flow can leave the hospital, + * but their medical history will be retained. + * + * Flows should be removed from [flowPatients] when they have completed successfully. Upon successful completion, + * the medical history of a flow is no longer relevant as that flow has been completely removed from the + * statemachine. + */ val flowPatients = HashMap() val treatableSessionInits = HashMap() val recordsPublisher = PublishSubject.create() }) private val secureRandom = newSecureRandom() - private val delayedDischargeTimer = Timer("FlowHospitalDelayedDischargeTimer", true) /** * The node was unable to initiate the [InitialSessionMessage] from [sender]. */ fun sessionInitErrored(sessionMessage: InitialSessionMessage, sender: Party, event: ExternalEvent.ExternalMessageEvent, error: Throwable) { - val time = Instant.now() + val time = clock.instant() val id = UUID.randomUUID() val outcome = if (error is SessionRejectException.UnknownClass) { // We probably don't have the CorDapp installed so let's pause the message in the hopes that the CorDapp is @@ -104,11 +159,48 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val } /** - * The flow running in [flowFiber] has errored. + * Forces the flow to be kept in for overnight observation by the hospital. A flow must already exist inside the hospital + * and have existing medical records for it to be moved to overnight observation. If it does not meet these criteria then + * an [IllegalArgumentException] will be thrown. + * + * @param id The [StateMachineRunId] of the flow that you are trying to force into observation + * @param errors The errors to include in the new medical record */ - fun flowErrored(flowFiber: FlowFiber, currentState: StateMachineState, errors: List) { - val time = Instant.now() + fun forceIntoOvernightObservation(id: StateMachineRunId, errors: List) { + mutex.locked { + // If a flow does not meet the criteria below, then it has moved into an invalid state or the function is being + // called from an incorrect location. The assertions below should error out the flow if they are not true. + requireNotNull(flowsInHospital[id]) { "Flow must already be in the hospital before forcing into overnight observation" } + val history = requireNotNull(flowPatients[id]) { "Flow must already have history before forcing into overnight observation" } + // Use the last staff member that last discharged the flow as the current staff member + val record = history.records.last().copy( + time = clock.instant(), + errors = errors, + outcome = Outcome.OVERNIGHT_OBSERVATION + ) + onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(id, record.by.map { it.toString() }) } + history.records += record + recordsPublisher.onNext(record) + } + } + + + /** + * Request treatment for the [flowFiber]. A flow can only be added to the hospital if they are not already being + * treated. + */ + fun requestTreatment(flowFiber: FlowFiber, currentState: StateMachineState, errors: List) { + // Only treat flows that are not already in the hospital + if (!currentState.isRemoved && flowsInHospital.putIfAbsent(flowFiber.id, flowFiber) == null) { + admit(flowFiber, currentState, errors) + } + } + + @Suppress("ComplexMethod") + private fun admit(flowFiber: FlowFiber, currentState: StateMachineState, errors: List) { + val time = clock.instant() log.info("Flow ${flowFiber.id} admitted to hospital in state $currentState") + onFlowAdmitted.forEach { it.invoke(flowFiber.id) } val (event, backOffForChronicCondition) = mutex.locked { val medicalHistory = flowPatients.computeIfAbsent(flowFiber.id) { FlowMedicalHistory() } @@ -119,15 +211,17 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val Diagnosis.DISCHARGE -> { val backOff = calculateBackOffForChronicCondition(report, medicalHistory, currentState) log.info("Flow error discharged from hospital (delay ${backOff.seconds}s) by ${report.by} (error was ${report.error.message})") + onFlowDischarged.forEach { hook -> hook.invoke(flowFiber.id, report.by.map{it.toString()}) } Triple(Outcome.DISCHARGE, Event.RetryFlowFromSafePoint, backOff) } Diagnosis.OVERNIGHT_OBSERVATION -> { log.info("Flow error kept for overnight observation by ${report.by} (error was ${report.error.message})") // We don't schedule a next event for the flow - it will automatically retry from its checkpoint on node restart + onFlowKeptForOvernightObservation.forEach { hook -> hook.invoke(flowFiber.id, report.by.map{it.toString()}) } Triple(Outcome.OVERNIGHT_OBSERVATION, null, 0.seconds) } - Diagnosis.NOT_MY_SPECIALTY -> { - // None of the staff care for these errors so we let them propagate + Diagnosis.NOT_MY_SPECIALTY, Diagnosis.TERMINAL -> { + // None of the staff care for these errors, or someone decided it is a terminal condition, so we let them propagate log.info("Flow error allowed to propagate", report.error) Triple(Outcome.UNTREATABLE, Event.StartErrorPropagation, 0.seconds) } @@ -143,10 +237,8 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val if (backOffForChronicCondition.isZero) { flowFiber.scheduleEvent(event) } else { - delayedDischargeTimer.schedule(object : TimerTask() { - override fun run() { - flowFiber.scheduleEvent(event) - } + hospitalJobTimer.schedule(timerTask { + flowFiber.scheduleEvent(event) }, backOffForChronicCondition.toMillis()) } } @@ -185,12 +277,19 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val private data class ConsultationReport(val error: Throwable, val diagnosis: Diagnosis, val by: List) /** - * The flow has been removed from the state machine. + * Remove the flow's medical history from the hospital. */ - fun flowRemoved(flowId: StateMachineRunId) { + fun removeMedicalHistory(flowId: StateMachineRunId) { mutex.locked { flowPatients.remove(flowId) } } + /** + * Remove the flow from the hospital as it is not currently being treated. + */ + fun leave(id: StateMachineRunId) { + flowsInHospital.remove(id) + } + // TODO MedicalRecord subtypes can expose the Staff class, something which we probably don't want when wiring this method to RPC /** Returns a stream of medical records as flows pass through the hospital. */ fun track(): DataFeed, MedicalRecord> { @@ -251,6 +350,8 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val /** The order of the enum values are in priority order. */ enum class Diagnosis { + /** The flow should not see other staff members */ + TERMINAL, /** Retry from last safe point. */ DISCHARGE, /** Park and await intervention. */ @@ -259,7 +360,6 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val NOT_MY_SPECIALTY } - interface Staff { fun consult(flowFiber: FlowFiber, currentState: StateMachineState, newError: Throwable, history: FlowMedicalHistory): Diagnosis } @@ -288,7 +388,8 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val */ object DuplicateInsertSpecialist : Staff { override fun consult(flowFiber: FlowFiber, currentState: StateMachineState, newError: Throwable, history: FlowMedicalHistory): Diagnosis { - return if (newError.mentionsThrowable(ConstraintViolationException::class.java) && history.notDischargedForTheSameThingMoreThan(3, this, currentState)) { + return if (newError.mentionsThrowable(ConstraintViolationException::class.java) + && history.notDischargedForTheSameThingMoreThan(2, this, currentState)) { Diagnosis.DISCHARGE } else { Diagnosis.NOT_MY_SPECIALTY @@ -334,7 +435,7 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val } private fun isErrorPropagatedFromCounterparty(error: Throwable): Boolean { - return when(error) { + return when (error) { is UnexpectedFlowEndException -> { val peer = DeclaredField(UnexpectedFlowEndException::class.java, "peer", error).value peer != null @@ -358,17 +459,21 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val val strippedStacktrace = error.stackTrace .filterNot { it?.className?.contains("counter-flow exception from peer") ?: false } .filterNot { it?.className?.startsWith("net.corda.node.services.statemachine.") ?: false } - return strippedStacktrace.isNotEmpty() && - strippedStacktrace.first().className.startsWith(ReceiveTransactionFlow::class.qualifiedName!! ) + return strippedStacktrace.isNotEmpty() + && strippedStacktrace.first().className.startsWith(ReceiveTransactionFlow::class.qualifiedName!!) } - } /** * [SQLTransientConnectionException] detection that arise from failing to connect the underlying database/datasource */ object TransientConnectionCardiologist : Staff { - override fun consult(flowFiber: FlowFiber, currentState: StateMachineState, newError: Throwable, history: FlowMedicalHistory): Diagnosis { + override fun consult( + flowFiber: FlowFiber, + currentState: StateMachineState, + newError: Throwable, + history: FlowMedicalHistory + ): Diagnosis { return if (mentionsTransientConnection(newError)) { if (history.notDischargedForTheSameThingMoreThan(2, this, currentState)) { Diagnosis.DISCHARGE @@ -384,6 +489,72 @@ class StaffedFlowHospital(private val flowMessaging: FlowMessaging, private val return exception.mentionsThrowable(SQLTransientConnectionException::class.java, "connection is not available") } } + + /** + * Hospitalise any database (SQL and Persistence) exception that wasn't handled otherwise, unless on the configurable whitelist + * Note that retry decisions from other specialists will not be affected as retries take precedence over hospitalisation. + */ + object DatabaseEndocrinologist : Staff { + override fun consult( + flowFiber: FlowFiber, + currentState: StateMachineState, + newError: Throwable, + history: FlowMedicalHistory + ): Diagnosis { + return if ((newError is SQLException || newError is PersistenceException) && !customConditions.any { it(newError) }) { + Diagnosis.OVERNIGHT_OBSERVATION + } else { + Diagnosis.NOT_MY_SPECIALTY + } + } + + @VisibleForTesting + val customConditions = mutableSetOf<(t: Throwable) -> Boolean>() + } + + /** + * Handles exceptions from internal state transitions that are not dealt with by the rest of the staff. + * + * [InterruptedException]s are diagnosed as [Diagnosis.TERMINAL] so they are never retried + * (can occur when a flow is killed - `killFlow`). + * [AsyncOperationTransitionException]s ares ignored as the error is likely to have originated in user async code rather than inside + * of a transition. + * All other exceptions are retried a maximum of 3 times before being kept in for observation. + */ + object TransitionErrorGeneralPractitioner : Staff { + override fun consult( + flowFiber: FlowFiber, + currentState: StateMachineState, + newError: Throwable, + history: FlowMedicalHistory + ): Diagnosis { + return if (newError.mentionsThrowable(StateTransitionException::class.java)) { + when { + newError.mentionsThrowable(InterruptedException::class.java) -> Diagnosis.TERMINAL + newError.mentionsThrowable(AsyncOperationTransitionException::class.java) -> Diagnosis.NOT_MY_SPECIALTY + history.notDischargedForTheSameThingMoreThan(2, this, currentState) -> Diagnosis.DISCHARGE + else -> Diagnosis.OVERNIGHT_OBSERVATION + } + } else { + Diagnosis.NOT_MY_SPECIALTY + }.also { logDiagnosis(it, newError, flowFiber, history) } + } + + private fun logDiagnosis(diagnosis: Diagnosis, newError: Throwable, flowFiber: FlowFiber, history: FlowMedicalHistory) { + if (diagnosis != Diagnosis.NOT_MY_SPECIALTY) { + log.debug { + """ + Flow ${flowFiber.id} given $diagnosis diagnosis due to a transition error + - Exception: ${newError.message} + - History: $history + ${(newError as? StateTransitionException)?.transitionAction?.let { "- Action: $it" }} + ${(newError as? StateTransitionException)?.transitionEvent?.let { "- Event: $it" }} + """.trimIndent() + } + } + } + } + } private fun Throwable?.mentionsThrowable(exceptionType: Class, errorMessage: String? = null): Boolean { @@ -396,4 +567,5 @@ private fun Throwable?.mentionsThrowable(exceptionType: Class true } return (exceptionType.isAssignableFrom(this::class.java) && containsMessage) || cause.mentionsThrowable(exceptionType, errorMessage) -} \ No newline at end of file +} + diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt index 3617da084d..f7c10d9ecd 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateMachineManager.kt @@ -114,6 +114,7 @@ interface ExternalEvent { * An external P2P message event. */ interface ExternalMessageEvent : ExternalEvent { + val flowId: StateMachineRunId val receivedMessage: ReceivedMessage } @@ -121,6 +122,7 @@ interface ExternalEvent { * An external request to start a flow, from the scheduler for example. */ interface ExternalStartFlowEvent : ExternalEvent { + val flowId: StateMachineRunId val flowLogic: FlowLogic val context: InvocationContext diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt new file mode 100644 index 0000000000..e32014ab18 --- /dev/null +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/StateTransitionExceptions.kt @@ -0,0 +1,18 @@ +package net.corda.node.services.statemachine + +import net.corda.core.CordaException +import net.corda.core.serialization.ConstructorForDeserialization + +// CORDA-3353 - These exceptions should not be propagated up to rpc as they suppress the real exceptions + +class StateTransitionException( + val transitionAction: Action?, + val transitionEvent: Event?, + val exception: Exception +) : CordaException(exception.message, exception) { + + @ConstructorForDeserialization + constructor(exception: Exception): this(null, null, exception) +} + +class AsyncOperationTransitionException(exception: Exception) : CordaException(exception.message, exception) diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt index 1ae80f8f5a..a9929522ac 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/TransitionExecutorImpl.kt @@ -9,6 +9,7 @@ import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.contextDatabase import net.corda.nodeapi.internal.persistence.contextTransactionOrNull import java.security.SecureRandom +import javax.persistence.OptimisticLockException /** * This [TransitionExecutor] runs the transition actions using the passed in [ActionExecutor] and manually dirties the @@ -27,6 +28,7 @@ class TransitionExecutorImpl( val log = contextLogger() } + @Suppress("NestedBlockDepth", "ReturnCount") @Suspendable override fun executeTransition( fiber: FlowFiber, @@ -47,15 +49,24 @@ class TransitionExecutorImpl( // Instead we just keep around the old error state and wait for a new schedule, perhaps // triggered from a flow hospital log.warn("Error while executing $action during transition to errored state, aborting transition", exception) + // CORDA-3354 - Go to the hospital with the new error that has occurred + // while already in a error state (as this error could be for a different reason) return Pair(FlowContinuation.Abort, previousState.copy(isFlowResumed = false)) } else { // Otherwise error the state manually keeping the old flow state and schedule a DoRemainingWork // to trigger error propagation - log.info("Error while executing $action, erroring state", exception) + log.info("Error while executing $action, with event $event, erroring state", exception) + if(previousState.isRemoved && exception is OptimisticLockException) { + log.debug("Flow has been killed and the following error is likely due to the flow's checkpoint being deleted. " + + "Occurred while executing $action, with event $event", exception) + } else { + log.info("Error while executing $action, with event $event, erroring state", exception) + } val newState = previousState.copy( checkpoint = previousState.checkpoint.copy( errorState = previousState.checkpoint.errorState.addErrors( - listOf(FlowError(secureRandom.nextLong(), exception)) + // Wrap the exception with [StateTransitionException] for handling by the flow hospital + listOf(FlowError(secureRandom.nextLong(), StateTransitionException(action, event, exception))) ) ), isFlowResumed = false @@ -67,4 +78,4 @@ class TransitionExecutorImpl( } return Pair(transition.continuation, transition.newState) } -} +} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/DumpHistoryOnErrorInterceptor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/DumpHistoryOnErrorInterceptor.kt index ecef21157b..88a4f76d22 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/DumpHistoryOnErrorInterceptor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/DumpHistoryOnErrorInterceptor.kt @@ -19,6 +19,7 @@ import java.util.concurrent.ConcurrentHashMap * This interceptor records a trace of all of the flows' states and transitions. If the flow dirties it dumps the trace * transition to the logger. */ +@Suppress("MaxLineLength") // detekt confusing the whole if statement for a line class DumpHistoryOnErrorInterceptor(val delegate: TransitionExecutor) : TransitionExecutor { companion object { private val log = contextLogger() @@ -34,18 +35,23 @@ class DumpHistoryOnErrorInterceptor(val delegate: TransitionExecutor) : Transiti transition: TransitionResult, actionExecutor: ActionExecutor ): Pair { - val (continuation, nextState) = delegate.executeTransition(fiber, previousState, event, transition, actionExecutor) - val transitionRecord = TransitionDiagnosticRecord(Instant.now(), fiber.id, previousState, nextState, event, transition, continuation) - val record = records.compute(fiber.id) { _, record -> - (record ?: ArrayList()).apply { add(transitionRecord) } - } + val (continuation, nextState) + = delegate.executeTransition(fiber, previousState, event, transition, actionExecutor) - // Just if we decide to propagate, and not if just on the way to the hospital. Only log at debug level here - the flow transition - // information is often unhelpful in the logs, and the actual cause of the problem will be logged elsewhere. - if (nextState.checkpoint.errorState is ErrorState.Errored && nextState.checkpoint.errorState.propagating) { - log.warn("Flow ${fiber.id} errored, dumping all transitions:\n${record!!.joinToString("\n")}") - for (error in nextState.checkpoint.errorState.errors) { - log.warn("Flow ${fiber.id} error", error.exception) + if (!previousState.isRemoved) { + val transitionRecord = + TransitionDiagnosticRecord(Instant.now(), fiber.id, previousState, nextState, event, transition, continuation) + val record = records.compute(fiber.id) { _, record -> + (record ?: ArrayList()).apply { add(transitionRecord) } + } + + // Just if we decide to propagate, and not if just on the way to the hospital. Only log at debug level here - the flow transition + // information is often unhelpful in the logs, and the actual cause of the problem will be logged elsewhere. + if (nextState.checkpoint.errorState is ErrorState.Errored && nextState.checkpoint.errorState.propagating) { + log.warn("Flow ${fiber.id} errored, dumping all transitions:\n${record!!.joinToString("\n")}") + for (error in nextState.checkpoint.errorState.errors) { + log.warn("Flow ${fiber.id} error", error.exception) + } } } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt index 93f78f3692..41798a3b3f 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/interceptors/HospitalisingInterceptor.kt @@ -11,7 +11,6 @@ import net.corda.node.services.statemachine.StateMachineState import net.corda.node.services.statemachine.TransitionExecutor import net.corda.node.services.statemachine.transitions.FlowContinuation import net.corda.node.services.statemachine.transitions.TransitionResult -import java.util.concurrent.ConcurrentHashMap /** * This interceptor notifies the passed in [flowHospital] in case a flow went through a clean->errored or a errored->clean @@ -27,12 +26,10 @@ class HospitalisingInterceptor( } private fun removeFlow(id: StateMachineRunId) { - hospitalisedFlows.remove(id) - flowHospital.flowRemoved(id) + flowHospital.leave(id) + flowHospital.removeMedicalHistory(id) } - private val hospitalisedFlows = ConcurrentHashMap() - @Suspendable override fun executeTransition( fiber: FlowFiber, @@ -41,19 +38,19 @@ class HospitalisingInterceptor( transition: TransitionResult, actionExecutor: ActionExecutor ): Pair { + + // If the fiber's previous state was clean then remove it from the hospital + // This is important for retrying a flow that has errored during a state machine transition + if (previousState.checkpoint.errorState is ErrorState.Clean) { + flowHospital.leave(fiber.id) + } + val (continuation, nextState) = delegate.executeTransition(fiber, previousState, event, transition, actionExecutor) - when (nextState.checkpoint.errorState) { - is ErrorState.Clean -> { - hospitalisedFlows.remove(fiber.id) - } - is ErrorState.Errored -> { - val exceptionsToHandle = nextState.checkpoint.errorState.errors.map { it.exception } - if (hospitalisedFlows.putIfAbsent(fiber.id, fiber) == null) { - flowHospital.flowErrored(fiber, previousState, exceptionsToHandle) - } - } - } + if (nextState.checkpoint.errorState is ErrorState.Errored && previousState.checkpoint.errorState is ErrorState.Clean) { + val exceptionsToHandle = nextState.checkpoint.errorState.errors.map { it.exception } + flowHospital.requestTreatment(fiber, previousState, exceptionsToHandle) + } if (nextState.isRemoved) { removeFlow(fiber.id) } diff --git a/node/src/main/kotlin/net/corda/node/services/transactions/PersistentUniquenessProvider.kt b/node/src/main/kotlin/net/corda/node/services/transactions/PersistentUniquenessProvider.kt index 73b5f79aa6..c093aad06a 100644 --- a/node/src/main/kotlin/net/corda/node/services/transactions/PersistentUniquenessProvider.kt +++ b/node/src/main/kotlin/net/corda/node/services/transactions/PersistentUniquenessProvider.kt @@ -72,6 +72,7 @@ class PersistentUniquenessProvider(val clock: Clock, val database: CordaPersiste var requestDate: Instant ) + @Suppress("MagicNumber") // database column length @Entity @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}notary_committed_txs") class CommittedTransaction( diff --git a/node/src/main/kotlin/net/corda/node/services/upgrade/ContractUpgradeServiceImpl.kt b/node/src/main/kotlin/net/corda/node/services/upgrade/ContractUpgradeServiceImpl.kt index 9aa5739a61..b28ef53b8a 100644 --- a/node/src/main/kotlin/net/corda/node/services/upgrade/ContractUpgradeServiceImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/upgrade/ContractUpgradeServiceImpl.kt @@ -12,6 +12,7 @@ import javax.persistence.Entity import javax.persistence.Id import javax.persistence.Table +@Suppress("MagicNumber") // database column length class ContractUpgradeServiceImpl(cacheFactory: NamedCacheFactory) : ContractUpgradeService, SingletonSerializeAsToken() { @Entity diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index 4e7e6915b7..c91a2363f7 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -23,6 +23,7 @@ import net.corda.node.services.statemachine.FlowStateMachineImpl import net.corda.nodeapi.internal.persistence.* import org.hibernate.Session import rx.Observable +import rx.exceptions.OnErrorNotImplementedException import rx.subjects.PublishSubject import java.security.PublicKey import java.time.Clock @@ -390,7 +391,15 @@ class NodeVaultService( } } persistentStateService.persist(vaultUpdate.produced + vaultUpdate.references) - updatesPublisher.onNext(vaultUpdate) + try { + updatesPublisher.onNext(vaultUpdate) + } catch (e: OnErrorNotImplementedException) { + log.warn("Caught an Rx.OnErrorNotImplementedException " + + "- caused by an exception in an RX observer that was unhandled " + + "- the observer has been unsubscribed! The underlying exception will be rethrown.", e) + // if the observer code threw, unwrap their exception from the RX wrapper + throw e.cause ?: e + } } } } diff --git a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt index 27fa6a8fa7..6b5b5fe0c5 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt @@ -26,6 +26,7 @@ object VaultSchema /** * First version of the Vault ORM schema */ +@Suppress("MagicNumber") // database column length @CordaSerializable object VaultSchemaV1 : MappedSchema( schemaFamily = VaultSchema.javaClass, diff --git a/node/src/main/kotlin/net/corda/notary/experimental/bftsmart/BFTSmartNotaryService.kt b/node/src/main/kotlin/net/corda/notary/experimental/bftsmart/BFTSmartNotaryService.kt index 168caa5d1e..f663af3fd1 100644 --- a/node/src/main/kotlin/net/corda/notary/experimental/bftsmart/BFTSmartNotaryService.kt +++ b/node/src/main/kotlin/net/corda/notary/experimental/bftsmart/BFTSmartNotaryService.kt @@ -127,6 +127,7 @@ class BFTSmartNotaryService( } } + @Suppress("MagicNumber") // database column length @Entity @Table(name = "${NODE_DATABASE_PREFIX}bft_committed_txs") class CommittedTransaction( diff --git a/node/src/main/kotlin/net/corda/notary/experimental/raft/RaftUniquenessProvider.kt b/node/src/main/kotlin/net/corda/notary/experimental/raft/RaftUniquenessProvider.kt index 5c43294eed..18761102f9 100644 --- a/node/src/main/kotlin/net/corda/notary/experimental/raft/RaftUniquenessProvider.kt +++ b/node/src/main/kotlin/net/corda/notary/experimental/raft/RaftUniquenessProvider.kt @@ -104,6 +104,7 @@ class RaftUniquenessProvider( var index: Long = 0 ) + @Suppress("MagicNumber") // database column length @Entity @Table(name = "${NODE_DATABASE_PREFIX}raft_committed_txs") class CommittedTransaction( diff --git a/node/src/main/resources/migration/vault-schema.changelog-master.xml b/node/src/main/resources/migration/vault-schema.changelog-master.xml index e934760df8..3ba9d52575 100644 --- a/node/src/main/resources/migration/vault-schema.changelog-master.xml +++ b/node/src/main/resources/migration/vault-schema.changelog-master.xml @@ -11,4 +11,5 @@ + diff --git a/node/src/main/resources/migration/vault-schema.changelog-v11.xml b/node/src/main/resources/migration/vault-schema.changelog-v11.xml new file mode 100644 index 0000000000..d095e19bce --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-v11.xml @@ -0,0 +1,11 @@ + + + + + + + \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt index 7a30fad6ee..8e1ebc4aa5 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt @@ -37,7 +37,8 @@ import net.corda.testing.internal.LogHelper import net.corda.testing.node.InMemoryMessagingNetwork.MessageTransfer import net.corda.testing.node.InMemoryMessagingNetwork.ServicePeerAllocationStrategy.RoundRobin import net.corda.testing.node.internal.* -import org.assertj.core.api.Assertions.* +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatIllegalArgumentException import org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType import org.assertj.core.api.Condition import org.junit.After @@ -115,18 +116,16 @@ class FlowFrameworkTests { } @Test - fun `exception while fiber suspended`() { + fun `exception while fiber suspended is retried and completes successfully`() { bobNode.registerCordappFlowFactory(ReceiveFlow::class) { InitiatedSendFlow("Hello", it) } val flow = ReceiveFlow(bob) val fiber = aliceNode.services.startFlow(flow) as FlowStateMachineImpl // Before the flow runs change the suspend action to throw an exception - val exceptionDuringSuspend = Exception("Thrown during suspend") - val throwingActionExecutor = SuspendThrowingActionExecutor(exceptionDuringSuspend, fiber.transientValues!!.value.actionExecutor) + val throwingActionExecutor = SuspendThrowingActionExecutor(Exception("Thrown during suspend"), + fiber.transientValues!!.value.actionExecutor) fiber.transientValues = TransientReference(fiber.transientValues!!.value.copy(actionExecutor = throwingActionExecutor)) mockNet.runNetwork() - assertThatThrownBy { - fiber.resultFuture.getOrThrow() - }.isSameAs(exceptionDuringSuspend) + fiber.resultFuture.getOrThrow() assertThat(aliceNode.smm.allStateMachines).isEmpty() // Make sure the fiber does actually terminate assertThat(fiber.state).isEqualTo(Strand.State.WAITING) diff --git a/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt b/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt index 95e2db570b..53ca92c77e 100644 --- a/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/statemachine/RetryFlowMockTest.kt @@ -2,14 +2,18 @@ package net.corda.node.services.statemachine import co.paralleluniverse.fibers.Suspendable import net.corda.core.concurrent.CordaFuture -import net.corda.core.flows.* +import net.corda.core.flows.Destination +import net.corda.core.flows.FlowInfo +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.FlowSession +import net.corda.core.flows.InitiatedBy +import net.corda.core.flows.InitiatingFlow import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party import net.corda.core.internal.FlowStateMachine import net.corda.core.internal.concurrent.flatMap import net.corda.core.messaging.MessageRecipients import net.corda.core.utilities.UntrustworthyData -import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.unwrap import net.corda.node.services.FinalityHandler import net.corda.node.services.messaging.Message @@ -17,11 +21,13 @@ import net.corda.node.services.persistence.DBTransactionStorage import net.corda.nodeapi.internal.persistence.contextTransaction import net.corda.testing.common.internal.eventually import net.corda.testing.core.TestIdentity -import net.corda.testing.node.internal.* +import net.corda.testing.node.internal.InternalMockNetwork +import net.corda.testing.node.internal.MessagingServiceSpy +import net.corda.testing.node.internal.TestStartedNode +import net.corda.testing.node.internal.enclosedCordapp +import net.corda.testing.node.internal.newContext import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatThrownBy import org.h2.util.Utils -import org.hibernate.exception.ConstraintViolationException import org.junit.After import org.junit.Assert.assertTrue import org.junit.Before @@ -49,6 +55,8 @@ class RetryFlowMockTest { SendAndRetryFlow.count = 0 RetryInsertFlow.count = 0 KeepSendingFlow.count.set(0) + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { t -> t is LimitedRetryCausingError } + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add { t -> t is RetryCausingError } } private fun TestStartedNode.startFlow(logic: FlowLogic): CordaFuture { @@ -58,6 +66,7 @@ class RetryFlowMockTest { @After fun cleanUp() { mockNet.stopNodes() + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.clear() } @Test @@ -66,14 +75,6 @@ class RetryFlowMockTest { assertEquals(2, RetryFlow.count) } - @Test - fun `Retry forever`() { - assertThatThrownBy { - nodeA.startFlow(RetryFlow(Int.MAX_VALUE)).getOrThrow() - }.isInstanceOf(LimitedRetryCausingError::class.java) - assertEquals(5, RetryFlow.count) - } - @Test fun `Retry does not set senderUUID`() { val messagesSent = Collections.synchronizedList(mutableListOf()) @@ -184,8 +185,7 @@ class RetryFlowMockTest { assertThat(nodeA.smm.flowHospital.track().snapshot).isEmpty() } - - class LimitedRetryCausingError : ConstraintViolationException("Test message", SQLException(), "Test constraint") + class LimitedRetryCausingError : IllegalStateException("I am going to live forever") class RetryCausingError : SQLException("deadlock") diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultFlowTest.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultFlowTest.kt index 23359b1cfd..42237ee304 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultFlowTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultFlowTest.kt @@ -6,6 +6,7 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party import net.corda.core.node.services.queryBy import net.corda.core.transactions.TransactionBuilder +import net.corda.node.services.statemachine.StaffedFlowHospital import net.corda.testing.core.DummyCommandData import net.corda.testing.core.singleIdentity import net.corda.testing.internal.vault.DUMMY_DEAL_PROGRAM_ID @@ -16,12 +17,13 @@ import net.corda.testing.node.MockNetwork import net.corda.testing.node.MockNetworkNotarySpec import net.corda.testing.node.MockNodeParameters import net.corda.testing.node.StartedMockNode -import org.assertj.core.api.Assertions import org.junit.After import org.junit.Before import org.junit.Test -import java.util.concurrent.ExecutionException +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import kotlin.test.assertEquals +import kotlin.test.assertTrue class VaultFlowTest { private lateinit var mockNetwork: MockNetwork @@ -48,14 +50,19 @@ class VaultFlowTest { @After fun tearDown() { mockNetwork.stopNodes() + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.clear() + StaffedFlowHospital.onFlowKeptForOvernightObservation.clear() } @Test fun `Unique column constraint failing causes states to not persist to vaults`() { + StaffedFlowHospital.DatabaseEndocrinologist.customConditions.add( { t: Throwable -> t is javax.persistence.PersistenceException }) partyA.startFlow(Initiator(listOf(partyA.info.singleIdentity(), partyB.info.singleIdentity()))).get() - Assertions.assertThatExceptionOfType(ExecutionException::class.java).isThrownBy { - partyA.startFlow(Initiator(listOf(partyA.info.singleIdentity(), partyB.info.singleIdentity()))).get() - } + val hospitalLatch = CountDownLatch(1) + StaffedFlowHospital.onFlowKeptForOvernightObservation.add { _, _ -> hospitalLatch.countDown() } + partyA.startFlow(Initiator(listOf(partyA.info.singleIdentity(), partyB.info.singleIdentity()))) + assertTrue(hospitalLatch.await(10, TimeUnit.SECONDS), "Flow not hospitalised") + assertEquals(1, partyA.transaction { partyA.services.vaultService.queryBy().states.size }) diff --git a/samples/README.md b/samples/README.md index fff62b2e0b..b293da63a2 100644 --- a/samples/README.md +++ b/samples/README.md @@ -5,7 +5,7 @@ Please refer to `README.md` in the individual project folders. There are the fo * **attachment-demo** A simple demonstration of sending a transaction with an attachment from one node to another, and then accessing the attachment on the remote node. * **irs-demo** A demo showing two nodes agreeing to an interest rate swap and doing fixings using an oracle. * **trader-demo** A simple driver for exercising the two party trading flow. In this scenario, a buyer wants to purchase some commercial paper by swapping his cash for commercial paper. The seller learns that the buyer exists, and sends them a message to kick off the trade. The seller, having obtained his CP, then quits and the buyer goes back to waiting. The buyer will sell as much CP as he can! **We recommend starting with this demo.** -* **Network-visualiser** A tool that uses a simulation to visualise the interaction and messages between nodes on the Corda network. Currently only works for the IRS demo. * **simm-valuation-demo** A demo showing two nodes reaching agreement on the valuation of a derivatives portfolio. * **notary-demo** A simple demonstration of a node getting multiple transactions notarised by a single or distributed (Raft or BFT SMaRt) notary. * **bank-of-corda-demo** A demo showing a node acting as an issuer of fungible assets (initially Cash) +* **network-verifier** A very simple CorDapp that can be used to test that communication over a Corda network works. diff --git a/samples/network-verifier/README.md b/samples/network-verifier/README.md new file mode 100644 index 0000000000..62d0266ca9 --- /dev/null +++ b/samples/network-verifier/README.md @@ -0,0 +1,13 @@ +Network verifier +---------------- + +Simple CorDapp that can be used to verify the setup of a Corda network. +It contacts every other network participant and receives a reply from them. +It also creates a transaction and finalizes it. + +This makes sure that all basic Corda functionality works. + +*Usage:* + +- From the rpc just run the ``TestCommsFlowInitiator`` flow and inspect the result. There should be a "Hello" message from every ohter participant. + \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 28c395e119..6e8ff4ba1f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -81,6 +81,8 @@ include 'serialization' include 'serialization-djvm' include 'serialization-djvm:deserializers' include 'serialization-tests' +include 'testing:cordapps:dbfailure:dbfcontracts' +include 'testing:cordapps:dbfailure:dbfworkflows' // Common libraries - start include 'common-validation' diff --git a/testing/cordapps/dbfailure/dbfcontracts/build.gradle b/testing/cordapps/dbfailure/dbfcontracts/build.gradle new file mode 100644 index 0000000000..76dd8b8082 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/build.gradle @@ -0,0 +1,18 @@ +apply plugin: 'kotlin' +//apply plugin: 'net.corda.plugins.cordapp' +//apply plugin: 'net.corda.plugins.quasar-utils' + +repositories { + mavenLocal() + mavenCentral() + maven { url "$artifactory_contextUrl/corda-dependencies" } + maven { url "$artifactory_contextUrl/corda" } +} + +dependencies { + compile project(":core") +} + +jar{ + baseName "testing-dbfailure-contracts" +} \ No newline at end of file diff --git a/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/contracts/DbFailureContract.kt b/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/contracts/DbFailureContract.kt new file mode 100644 index 0000000000..c344badebb --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/contracts/DbFailureContract.kt @@ -0,0 +1,50 @@ +package com.r3.dbfailure.contracts + +import com.r3.dbfailure.schemas.DbFailureSchemaV1 +import net.corda.core.contracts.CommandData +import net.corda.core.contracts.Contract +import net.corda.core.contracts.LinearState +import net.corda.core.contracts.UniqueIdentifier +import net.corda.core.identity.AbstractParty +import net.corda.core.identity.Party +import net.corda.core.schemas.MappedSchema +import net.corda.core.schemas.PersistentState +import net.corda.core.schemas.QueryableState +import net.corda.core.transactions.LedgerTransaction +import java.lang.IllegalArgumentException + +class DbFailureContract : Contract { + companion object { + @JvmStatic + val ID = "com.r3.dbfailure.contracts.DbFailureContract" + } + + class TestState( + override val linearId: UniqueIdentifier, + val particpant: Party, + val randomValue: String?, + val errorTarget: Int = 0 + ) : LinearState, QueryableState { + + override val participants: List = listOf(particpant) + + override fun supportedSchemas(): Iterable = listOf(DbFailureSchemaV1) + + override fun generateMappedObject(schema: MappedSchema): PersistentState { + return if (schema is DbFailureSchemaV1){ + DbFailureSchemaV1.PersistentTestState( particpant.name.toString(), randomValue, errorTarget, linearId.id) + } + else { + throw IllegalArgumentException("Unsupported schema $schema") + } + } + } + + override fun verify(tx: LedgerTransaction) { + // no op - don't care for now + } + + interface Commands : CommandData{ + class Create: Commands + } +} \ No newline at end of file diff --git a/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/schemas/DbFailureSchema.kt b/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/schemas/DbFailureSchema.kt new file mode 100644 index 0000000000..b047098f1c --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/src/main/kotlin/com/r3/dbfailure/schemas/DbFailureSchema.kt @@ -0,0 +1,35 @@ +package com.r3.dbfailure.schemas + +import net.corda.core.schemas.MappedSchema +import net.corda.core.schemas.PersistentState +import java.util.* +import javax.persistence.Column +import javax.persistence.Entity +import javax.persistence.Table + +object DbFailureSchema + +object DbFailureSchemaV1 : MappedSchema( + schemaFamily = DbFailureSchema.javaClass, + version = 1, + mappedTypes = listOf(DbFailureSchemaV1.PersistentTestState::class.java)){ + override val migrationResource = "dbfailure.changelog-master" + + @Entity + @Table( name = "fail_test_states") + class PersistentTestState( + @Column( name = "participant") + var participantName: String, + + @Column( name = "random_value", nullable = false) + var randomValue: String?, + + @Column( name = "error_target") + var errorTarget: Int, + + @Column( name = "linear_id") + var linearId: UUID + ) : PersistentState() { + constructor() : this( "", "", 0, UUID.randomUUID()) + } +} diff --git a/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-errortarget.xml b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-errortarget.xml new file mode 100644 index 0000000000..ebc2450e11 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-errortarget.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-init.xml b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-init.xml new file mode 100644 index 0000000000..c65f013bb1 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-init.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-master.xml b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-master.xml new file mode 100644 index 0000000000..22b9ffda33 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfcontracts/src/main/resources/migration/dbfailure.changelog-master.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/testing/cordapps/dbfailure/dbfworkflows/build.gradle b/testing/cordapps/dbfailure/dbfworkflows/build.gradle new file mode 100644 index 0000000000..571a3cb3a5 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfworkflows/build.gradle @@ -0,0 +1,12 @@ +apply plugin: 'kotlin' +//apply plugin: 'net.corda.plugins.cordapp' +//apply plugin: 'net.corda.plugins.quasar-utils' + +dependencies { + compile project(":core") + compile project(":testing:cordapps:dbfailure:dbfcontracts") +} + +jar{ + baseName "testing-dbfailure-workflows" +} \ No newline at end of file diff --git a/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/CreateStateFlow.kt b/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/CreateStateFlow.kt new file mode 100644 index 0000000000..6fd4a49b63 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/CreateStateFlow.kt @@ -0,0 +1,99 @@ +package com.r3.dbfailure.workflows + +import co.paralleluniverse.fibers.Suspendable +import com.r3.dbfailure.contracts.DbFailureContract +import net.corda.core.contracts.Command +import net.corda.core.contracts.UniqueIdentifier +import net.corda.core.flows.FlowLogic +import net.corda.core.flows.InitiatingFlow +import net.corda.core.flows.StartableByRPC +import net.corda.core.transactions.TransactionBuilder + +// There is a bit of number fiddling in this class to encode/decode the error target instructions +@Suppress("MagicNumber") +object CreateStateFlow { + + // Encoding of error targets + // 1s are errors actions to be taken in the vault listener in the service + // 10s are errors caused in the flow + // 100s control exception handling in the flow + // 1000s control exception handlling in the service/vault listener + enum class ErrorTarget(val targetNumber: Int) { + NoError(0), + ServiceSqlSyntaxError(1), + ServiceNullConstraintViolation(2), + ServiceValidUpdate(3), + ServiceReadState(4), + ServiceCheckForState(5), + ServiceThrowInvalidParameter(6), + TxInvalidState(10), + FlowSwallowErrors(100), + ServiceSwallowErrors(1000) + } + + fun errorTargetsToNum(vararg targets: ErrorTarget): Int { + return targets.map { it.targetNumber }.sum() + } + + private val targetMap = ErrorTarget.values().associateBy(ErrorTarget::targetNumber) + + fun getServiceTarget(target: Int?): ErrorTarget { + return target?.let { targetMap.getValue(it % 10) } ?: CreateStateFlow.ErrorTarget.NoError + } + + fun getServiceExceptionHandlingTarget(target: Int?): ErrorTarget { + return target?.let { targetMap.getValue(((it / 1000) % 10) * 1000) } ?: CreateStateFlow.ErrorTarget.NoError + } + + fun getTxTarget(target: Int?): ErrorTarget { + return target?.let { targetMap.getValue(((it / 10) % 10) * 10) } ?: CreateStateFlow.ErrorTarget.NoError + } + + fun getFlowTarget(target: Int?): ErrorTarget { + return target?.let { targetMap.getValue(((it / 100) % 10) * 100) } ?: CreateStateFlow.ErrorTarget.NoError + } + + @InitiatingFlow + @StartableByRPC + class Initiator(private val randomValue: String, private val errorTarget: Int) : FlowLogic() { + + @Suspendable + override fun call(): UniqueIdentifier { + logger.info("Test flow: starting") + val notary = serviceHub.networkMapCache.notaryIdentities[0] + val txTarget = getTxTarget(errorTarget) + logger.info("Test flow: The tx error target is $txTarget") + val state = DbFailureContract.TestState( + UniqueIdentifier(), + ourIdentity, + if (txTarget == CreateStateFlow.ErrorTarget.TxInvalidState) null else randomValue, + errorTarget) + val txCommand = Command(DbFailureContract.Commands.Create(), ourIdentity.owningKey) + + logger.info("Test flow: tx builder") + val txBuilder = TransactionBuilder(notary) + .addOutputState(state) + .addCommand(txCommand) + + logger.info("Test flow: verify") + txBuilder.verify(serviceHub) + + val signedTx = serviceHub.signInitialTransaction(txBuilder) + + @Suppress("TooGenericExceptionCaught") // this is fully intentional here, to allow twiddling with exceptions according to config + try { + logger.info("Test flow: recording transaction") + serviceHub.recordTransactions(signedTx) + } catch (t: Throwable) { + if (getFlowTarget(errorTarget) == CreateStateFlow.ErrorTarget.FlowSwallowErrors) { + logger.info("Test flow: Swallowing all exception! Muahahaha!", t) + } else { + logger.info("Test flow: caught exception - rethrowing") + throw t + } + } + logger.info("Test flow: returning") + return state.linearId + } + } +} \ No newline at end of file diff --git a/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/DbListenerService.kt b/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/DbListenerService.kt new file mode 100644 index 0000000000..412510b2f1 --- /dev/null +++ b/testing/cordapps/dbfailure/dbfworkflows/src/main/kotlin/com/r3/dbfailure/workflows/DbListenerService.kt @@ -0,0 +1,98 @@ +package com.r3.dbfailure.workflows + +import com.r3.dbfailure.contracts.DbFailureContract +import net.corda.core.node.AppServiceHub +import net.corda.core.node.services.CordaService +import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.utilities.contextLogger +import java.security.InvalidParameterException + +@CordaService +class DbListenerService(services: AppServiceHub) : SingletonSerializeAsToken() { + + companion object { + val log = contextLogger() + } + + init { + services.vaultService.rawUpdates.subscribe { (_, produced) -> + produced.forEach { + val contractState = it.state.data as? DbFailureContract.TestState + @Suppress("TooGenericExceptionCaught") // this is fully intentional here, to allow twiddling with exceptions + try { + when (CreateStateFlow.getServiceTarget(contractState?.errorTarget)) { + CreateStateFlow.ErrorTarget.ServiceSqlSyntaxError -> { + log.info("Fail with syntax error on raw statement") + val session = services.jdbcSession() + val statement = session.createStatement() + statement.execute( + "UPDATE FAIL_TEST_STATES \n" + + "BLAAA RANDOM_VALUE = NULL\n" + + "WHERE transaction_id = '${it.ref.txhash}' AND output_index = ${it.ref.index};" + ) + log.info("SQL result: ${statement.resultSet}") + } + CreateStateFlow.ErrorTarget.ServiceNullConstraintViolation -> { + log.info("Fail with null constraint violation on raw statement") + val session = services.jdbcSession() + val statement = session.createStatement() + statement.execute( + "UPDATE FAIL_TEST_STATES \n" + + "SET RANDOM_VALUE = NULL\n" + + "WHERE transaction_id = '${it.ref.txhash}' AND output_index = ${it.ref.index};" + ) + log.info("SQL result: ${statement.resultSet}") + } + CreateStateFlow.ErrorTarget.ServiceValidUpdate -> { + log.info("Update current statement") + val session = services.jdbcSession() + val statement = session.createStatement() + statement.execute( + "UPDATE FAIL_TEST_STATES \n" + + "SET RANDOM_VALUE = '${contractState!!.randomValue} Updated by service'\n" + + "WHERE transaction_id = '${it.ref.txhash}' AND output_index = ${it.ref.index};" + ) + log.info("SQL result: ${statement.resultSet}") + } + CreateStateFlow.ErrorTarget.ServiceReadState -> { + log.info("Read current state from db") + val session = services.jdbcSession() + val statement = session.createStatement() + statement.execute( + "SELECT * FROM FAIL_TEST_STATES \n" + + "WHERE transaction_id = '${it.ref.txhash}' AND output_index = ${it.ref.index};" + ) + log.info("SQL result: ${statement.resultSet}") + } + CreateStateFlow.ErrorTarget.ServiceCheckForState -> { + log.info("Check for currently written state in the db") + val session = services.jdbcSession() + val statement = session.createStatement() + val rs = statement.executeQuery( + "SELECT COUNT(*) FROM FAIL_TEST_STATES \n" + + "WHERE transaction_id = '${it.ref.txhash}' AND output_index = ${it.ref.index};" + ) + val numOfRows = if (rs.next()) rs.getInt("COUNT(*)") else 0 + log.info("Found a state with tx:ind ${it.ref.txhash}:${it.ref.index} in " + + "TEST_FAIL_STATES: ${if (numOfRows > 0) "Yes" else "No"}") + } + CreateStateFlow.ErrorTarget.ServiceThrowInvalidParameter -> { + log.info("Throw InvalidParameterException") + throw InvalidParameterException("Toys out of pram") + } + else -> { + // do nothing, everything else must be handled elsewhere + } + } + } catch (t: Throwable) { + if (CreateStateFlow.getServiceExceptionHandlingTarget(contractState?.errorTarget) + == CreateStateFlow.ErrorTarget.ServiceSwallowErrors) { + log.warn("Service not letting errors escape", t) + } else { + throw t + } + } + } + } + } +} \ No newline at end of file diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/CustomCordapp.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/CustomCordapp.kt index e2665e96dc..cba5d9a523 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/CustomCordapp.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/CustomCordapp.kt @@ -7,6 +7,7 @@ import net.corda.core.internal.cordapp.set import net.corda.core.utilities.contextLogger import net.corda.core.utilities.debug import net.corda.testing.core.internal.JarSignatureTestUtils.generateKey +import net.corda.testing.core.internal.JarSignatureTestUtils.containsKey import net.corda.testing.core.internal.JarSignatureTestUtils.signJar import java.nio.file.Path import java.nio.file.Paths @@ -43,7 +44,8 @@ data class CustomCordapp( override fun withOnlyJarContents(): CustomCordapp = CustomCordapp(packages = packages, classes = classes) - fun signed(keyStorePath: Path? = null): CustomCordapp = copy(signingInfo = SigningInfo(keyStorePath)) + fun signed(keyStorePath: Path? = null, numberOfSignatures: Int = 1, keyAlgorithm: String = "RSA"): CustomCordapp = + copy(signingInfo = SigningInfo(keyStorePath, numberOfSignatures, keyAlgorithm)) @VisibleForTesting internal fun packageAsJar(file: Path) { @@ -73,20 +75,21 @@ data class CustomCordapp( private fun signJar(jarFile: Path) { if (signingInfo != null) { - val testKeystore = "_teststore" - val alias = "Test" - val pwd = "secret!" val keyStorePathToUse = if (signingInfo.keyStorePath != null) { signingInfo.keyStorePath } else { defaultJarSignerDirectory.createDirectories() - if (!(defaultJarSignerDirectory / testKeystore).exists()) { - defaultJarSignerDirectory.generateKey(alias, pwd, "O=Test Company Ltd,OU=Test,L=London,C=GB") - } defaultJarSignerDirectory } - val pk = keyStorePathToUse.signJar(jarFile.toString(), alias, pwd) - logger.debug { "Signed Jar: $jarFile with public key $pk" } + + for (i in 1 .. signingInfo.numberOfSignatures) { + val alias = "alias$i" + val pwd = "secret!" + if (!keyStorePathToUse.containsKey(alias, pwd)) + keyStorePathToUse.generateKey(alias, pwd, "O=Test Company Ltd $i,OU=Test,L=London,C=GB", signingInfo.keyAlgorithm) + val pk = keyStorePathToUse.signJar(jarFile.toString(), alias, pwd) + logger.debug { "Signed Jar: $jarFile with public key $pk" } + } } else { logger.debug { "Unsigned Jar: $jarFile" } } @@ -111,7 +114,7 @@ data class CustomCordapp( return ZipEntry(name).setCreationTime(epochFileTime).setLastAccessTime(epochFileTime).setLastModifiedTime(epochFileTime) } - data class SigningInfo(val keyStorePath: Path? = null) + data class SigningInfo(val keyStorePath: Path?, val numberOfSignatures: Int, val keyAlgorithm: String) companion object { private val logger = contextLogger() diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index 8bb6a4ad3e..9293e6274b 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -34,6 +34,7 @@ import net.corda.core.node.NetworkParameters import net.corda.core.node.NotaryInfo import net.corda.core.node.services.NetworkMapCache import net.corda.core.utilities.NetworkHostAndPort +import net.corda.core.utilities.Try import net.corda.core.utilities.contextLogger import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.millis @@ -82,6 +83,7 @@ import rx.schedulers.Schedulers import java.io.File import java.net.ConnectException import java.net.URL +import java.net.URLClassLoader import java.nio.file.Path import java.security.cert.X509Certificate import java.time.Duration @@ -152,6 +154,14 @@ class DriverDSLImpl( //TODO: remove this once we can bundle quasar properly. private val quasarJarPath: String by lazy { resolveJar("co.paralleluniverse.fibers.Suspendable") } + private val bytemanJarPath: String? by lazy { + try { + resolveJar("org.jboss.byteman.agent.Transformer") + } catch (e: Exception) { + null + } + } + private fun NodeConfig.checkAndOverrideForInMemoryDB(): NodeConfig = this.run { if (inMemoryDB && corda.dataSourceProperties.getProperty("dataSource.url").startsWith("jdbc:h2:")) { val jdbcUrl = "jdbc:h2:mem:persistence${inMemoryCounter.getAndIncrement()};DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=100" @@ -206,7 +216,9 @@ class DriverDSLImpl( } } - override fun startNode(parameters: NodeParameters): CordaFuture { + override fun startNode(parameters: NodeParameters): CordaFuture = startNode(parameters, bytemanPort = null) + + override fun startNode(parameters: NodeParameters, bytemanPort: Int?): CordaFuture { val p2pAddress = portAllocation.nextHostAndPort() // TODO: Derive name from the full picked name, don't just wrap the common name val name = parameters.providedName ?: CordaX500Name("${oneOf(names).organisation}-${p2pAddress.port}", "London", "GB") @@ -221,15 +233,17 @@ class DriverDSLImpl( return registrationFuture.flatMap { networkMapAvailability.flatMap { // But starting the node proper does require the network map - startRegisteredNode(name, it, parameters, p2pAddress) + startRegisteredNode(name, it, parameters, p2pAddress, bytemanPort) } } } + @Suppress("ComplexMethod") private fun startRegisteredNode(name: CordaX500Name, localNetworkMap: LocalNetworkMap?, parameters: NodeParameters, - p2pAddress: NetworkHostAndPort = portAllocation.nextHostAndPort()): CordaFuture { + p2pAddress: NetworkHostAndPort = portAllocation.nextHostAndPort(), + bytemanPort: Int? = null): CordaFuture { val rpcAddress = portAllocation.nextHostAndPort() val rpcAdminAddress = portAllocation.nextHostAndPort() val webAddress = portAllocation.nextHostAndPort() @@ -264,13 +278,13 @@ class DriverDSLImpl( NodeConfiguration::additionalNodeInfoPollingFrequencyMsec.name to 1000 ) + czUrlConfig + jmxConfig + parameters.customOverrides val config = NodeConfig( - ConfigHelper.loadConfig( - baseDirectory = baseDirectory(name), - allowMissingConfig = true, - configOverrides = if (overrides.hasPath("devMode")) overrides else overrides + mapOf("devMode" to true) - ).withDJVMConfig(djvmBootstrapSource, djvmCordaSource) + ConfigHelper.loadConfig( + baseDirectory = baseDirectory(name), + allowMissingConfig = true, + configOverrides = if (overrides.hasPath("devMode")) overrides else overrides + mapOf("devMode" to true) + ).withDJVMConfig(djvmBootstrapSource, djvmCordaSource) ).checkAndOverrideForInMemoryDB() - return startNodeInternal(config, webAddress, localNetworkMap, parameters) + return startNodeInternal(config, webAddress, localNetworkMap, parameters, bytemanPort) } private fun startNodeRegistration( @@ -574,6 +588,8 @@ class DriverDSLImpl( config, quasarJarPath, debugPort, + bytemanJarPath, + null, systemProperties, "512m", null, @@ -585,10 +601,12 @@ class DriverDSLImpl( } } + @Suppress("ComplexMethod") private fun startNodeInternal(config: NodeConfig, webAddress: NetworkHostAndPort, localNetworkMap: LocalNetworkMap?, - parameters: NodeParameters): CordaFuture { + parameters: NodeParameters, + bytemanPort: Int?): CordaFuture { val visibilityHandle = networkVisibilityController.register(config.corda.myLegalName) val baseDirectory = config.corda.baseDirectory.createDirectories() localNetworkMap?.networkParametersCopier?.install(baseDirectory) @@ -634,7 +652,16 @@ class DriverDSLImpl( nodeFuture } else { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null - val process = startOutOfProcessNode(config, quasarJarPath, debugPort, systemProperties, parameters.maximumHeapSize, parameters.logLevelOverride) + val process = startOutOfProcessNode( + config, + quasarJarPath, + debugPort, + bytemanJarPath, + bytemanPort, + systemProperties, + parameters.maximumHeapSize, + parameters.logLevelOverride + ) // Destroy the child process when the parent exits.This is needed even when `waitForAllNodesToFinish` is // true because we don't want orphaned processes in the case that the parent process is terminated by the @@ -786,16 +813,21 @@ class DriverDSLImpl( } } + @Suppress("ComplexMethod", "MaxLineLength") private fun startOutOfProcessNode( config: NodeConfig, quasarJarPath: String, debugPort: Int?, + bytemanJarPath: String?, + bytemanPort: Int?, overriddenSystemProperties: Map, maximumHeapSize: String, logLevelOverride: String?, vararg extraCmdLineFlag: String ): Process { - log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled")) + log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, " + + "debug port is " + (debugPort ?: "not enabled") + ", " + + "byteMan: " + if (bytemanJarPath == null) "not in classpath" else "port is " + (bytemanPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) @@ -839,6 +871,20 @@ class DriverDSLImpl( it += extraCmdLineFlag }.toList() + val bytemanJvmArgs = { + val bytemanAgent = bytemanJarPath?.let { + bytemanPort?.let { + "-javaagent:$bytemanJarPath=port:$bytemanPort,listener:true" + } + } + listOfNotNull(bytemanAgent) + + if (bytemanAgent != null && debugPort != null) listOf( + "-Dorg.jboss.byteman.verbose=true", + "-Dorg.jboss.byteman.debug=true" + ) + else emptyList() + }.invoke() + // The following dependencies are excluded from the classpath of the created JVM, // so that the environment resembles a real one as close as possible. // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.) @@ -853,7 +899,7 @@ class DriverDSLImpl( className = "net.corda.node.Corda", // cannot directly get class for this, so just use string arguments = arguments, jdwpPort = debugPort, - extraJvmArguments = extraJvmArguments, + extraJvmArguments = extraJvmArguments + bytemanJvmArgs, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, classPath = cp @@ -1016,6 +1062,11 @@ interface InternalDriverDSL : DriverDSL { fun start() fun shutdown() + + fun startNode( + parameters: NodeParameters = NodeParameters(), + bytemanPort: Int? = null + ): CordaFuture } /** diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt index 89c60509ec..bebe27afbd 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockNodeMessagingService.kt @@ -1,5 +1,6 @@ package net.corda.testing.node.internal +import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.CordaX500Name import net.corda.core.identity.PartyAndCertificate import net.corda.core.internal.PLATFORM_VERSION @@ -268,6 +269,7 @@ class MockNodeMessagingService(private val configuration: NodeConfiguration, private inner class InMemoryDeduplicationHandler(override val receivedMessage: ReceivedMessage, val transfer: InMemoryMessagingNetwork.MessageTransfer) : DeduplicationHandler, ExternalEvent.ExternalMessageEvent { override val externalCause: ExternalEvent get() = this + override val flowId: StateMachineRunId = StateMachineRunId.createRandom() override val deduplicationHandler: DeduplicationHandler get() = this diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/core/internal/JarSignatureTestUtils.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/core/internal/JarSignatureTestUtils.kt index b5f27fe0a8..00a827c1e1 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/core/internal/JarSignatureTestUtils.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/core/internal/JarSignatureTestUtils.kt @@ -9,6 +9,7 @@ import java.io.Closeable import java.io.FileInputStream import java.io.FileOutputStream import java.nio.file.Files +import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.Paths import java.security.PublicKey @@ -72,6 +73,15 @@ object JarSignatureTestUtils { return ks.getCertificate(alias).publicKey } + fun Path.containsKey(alias: String, storePassword: String, storeName: String = "_teststore"): Boolean { + return try { + val ks = loadKeyStore(this.resolve(storeName), storePassword) + ks.containsAlias(alias) + } catch (e: NoSuchFileException) { + false + } + } + fun Path.getPublicKey(alias: String, storePassword: String) = getPublicKey(alias, "_teststore", storePassword) fun Path.getJarSigners(fileName: String) = diff --git a/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeController.kt b/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeController.kt index 8c69365022..9b2b924c7c 100644 --- a/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeController.kt +++ b/tools/demobench/src/main/kotlin/net/corda/demobench/model/NodeController.kt @@ -135,6 +135,7 @@ class NodeController(check: atRuntime = ::checkExists) : Controller() { } } + @Suppress("MagicNumber") // initialising to max value private fun makeNetworkParametersCopier(config: NodeConfigWrapper): NetworkParametersCopier { val identity = getNotaryIdentity(config) val parametersCopier = NetworkParametersCopier(NetworkParameters( diff --git a/tools/demobench/src/main/kotlin/net/corda/demobench/views/NodeTabView.kt b/tools/demobench/src/main/kotlin/net/corda/demobench/views/NodeTabView.kt index e270d27731..dd867fc1e0 100644 --- a/tools/demobench/src/main/kotlin/net/corda/demobench/views/NodeTabView.kt +++ b/tools/demobench/src/main/kotlin/net/corda/demobench/views/NodeTabView.kt @@ -241,6 +241,7 @@ class NodeTabView : Fragment() { CityDatabase.cityMap.values.map { it.countryCode }.toSet().map { it to Image(resources["/net/corda/demobench/flags/$it.png"]) }.toMap() } + @Suppress("MagicNumber") // demobench UI magic private fun Pane.nearestCityField(): ComboBox { return combobox(model.nearestCity, CityDatabase.cityMap.values.toList().sortedBy { it.description }) { minWidth = textWidth diff --git a/tools/loadtest/src/main/kotlin/net/corda/loadtest/Main.kt b/tools/loadtest/src/main/kotlin/net/corda/loadtest/Main.kt index eb4e103c73..a13d12e44b 100644 --- a/tools/loadtest/src/main/kotlin/net/corda/loadtest/Main.kt +++ b/tools/loadtest/src/main/kotlin/net/corda/loadtest/Main.kt @@ -70,6 +70,7 @@ fun main(args: Array) { } } +@Suppress("MagicNumber") // test constants private fun runLoadTest(loadTestConfiguration: LoadTestConfiguration) { runLoadTests(loadTestConfiguration, listOf( selfIssueTest to LoadTest.RunParameters( @@ -131,6 +132,7 @@ private fun runLoadTest(loadTestConfiguration: LoadTestConfiguration) { )) } +@Suppress("MagicNumber") // test constants private fun runStabilityTest(loadTestConfiguration: LoadTestConfiguration) { runLoadTests(loadTestConfiguration, listOf( // Self issue cash. This is a pre test step to make sure vault have enough cash to work with. diff --git a/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/volumes/Volume.kt b/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/volumes/Volume.kt index d9c7c8d7bf..d1918c5431 100644 --- a/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/volumes/Volume.kt +++ b/tools/network-builder/src/main/kotlin/net/corda/networkbuilder/volumes/Volume.kt @@ -38,6 +38,7 @@ interface Volume { nodeInfoFile.readBytes().deserialize().verified().let { NotaryInfo(it.legalIdentities.first(), validating) } } + @Suppress("MagicNumber") // default config constants return notaryInfos.let { NetworkParameters( minimumPlatformVersion = 1,