Webook test branch (#5528)

* add ability to index by class OR method

* disable unit tests again

* pending commit

* stream output of build for debugging

* remove git from the base image

* re-enable NodeRPCTests

* add ability to distribute tests based on TeamCity CSV output for test times

* try enabling unit tests again after test distribution has been fixed

* refactor BucketingAllocator to be a class, which bucketingAllocatorTask delegates to.

* finishing touches for improved test distribution

* create new pipelines for builds

.ci/dev/integration/Jenkinsfile

@@ -0,0 +1,59 @@
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+@Library('existing-build-control')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'k8s' }
+    options { timestamps() }
+
+    environment {
+        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
+        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
+        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
+    }
+
+    stages {
+        stage('Corda - Generate Build Image') {
+            steps {
+                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                    sh "./gradlew " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " clean pushBuildImage"
+                }
+                sh "kubectl auth can-i get pods"
+            }
+        }
+
+        stage('Corda - Run Tests') {
+            stage('Integration Tests') {
+                steps {
+                    sh "./gradlew " +
+                            "-DbuildId=\"\${BUILD_ID}\" " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " allParallelIntegrationTest"
+                    if (env.CHANGE_ID) {
+                        pullRequest.createStatus(status: 'success',
+                                context: 'continuous-integration/jenkins/pr-merge/integrationTest',
+                                description: 'Integration Tests Passed',
+                                targetUrl: "${env.JOB_URL}/testResults")
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            junit '**/build/test-results-xml/**/*.xml'
+        }
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

.ci/dev/regression/Jenkinsfile

@@ -0,0 +1,61 @@
+@Library('existing-build-control')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'k8s' }
+    options { timestamps() }
+
+    environment {
+        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
+        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
+        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
+    }
+
+    stages {
+        stage('Corda Pull Request - Generate Build Image') {
+            steps {
+                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                    sh "./gradlew " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " clean pushBuildImage"
+                }
+                sh "kubectl auth can-i get pods"
+            }
+        }
+
+        stage('Corda Pull Request - Run Tests') {
+            stage('Unit and Integration Tests') {
+                steps {
+                    sh "./gradlew " +
+                            "-DbuildId=\"\${BUILD_ID}\" " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " allParallelUnitAndIntegrationTest"
+                }
+            }
+            stage('Slow Integration Tests') {
+                steps {
+                    sh "./gradlew " +
+                            "-DbuildId=\"\${BUILD_ID}\" " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " allParallelSlowIntegrationTest"
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            junit '**/build/test-results-xml/**/*.xml'
+        }
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

.ci/dev/unit/Jenkinsfile

@@ -0,0 +1,59 @@
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+@Library('existing-build-control')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'k8s' }
+    options { timestamps() }
+
+    environment {
+        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
+        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
+        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
+    }
+
+    stages {
+        stage('Corda Pull Request - Generate Build Image') {
+            steps {
+                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                    sh "./gradlew " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " clean pushBuildImage"
+                }
+                sh "kubectl auth can-i get pods"
+            }
+        }
+
+        stage('Corda Pull Request - Run Tests') {
+            stage('Unit Tests') {
+                steps {
+                    sh "./gradlew " +
+                            "-DbuildId=\"\${BUILD_ID}\" " +
+                            "-Dkubenetize=true " +
+                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                            " allParallelUnitTest"
+                    if (env.CHANGE_ID) {
+                        pullRequest.createStatus(status: 'success',
+                                context: 'continuous-integration/jenkins/pr-merge/unitTest',
+                                description: 'Unit Tests Passed',
+                                targetUrl: "${env.JOB_URL}/testResults")
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            junit '**/build/test-results-xml/**/*.xml'
+        }
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

Jenkinsfile

@@ -1,4 +1,7 @@
-killall_jobs()
+@Library('existing-build-control')
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
 pipeline {
     agent { label 'k8s' }
@@ -35,48 +38,27 @@ pipeline {
                                 "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                                 " allParallelIntegrationTest"
                     }
-                    post {
+                }
-                        always {
+                stage('Unit Tests') {
-                            junit '**/build/test-results-xml/**/*.xml'
+                    steps {
-                        }
+                        sh "./gradlew " +
+                                "-DbuildId=\"\${BUILD_ID}\" " +
+                                "-Dkubenetize=true " +
+                                "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                                " allParallelUnitTest"
                     }
                 }
-//                stage('Unit Tests') {
-//                    steps {
-//                        sh "./gradlew " +
-//                                "-DbuildId=\"\${BUILD_ID}\" " +
-//                                "-Dkubenetize=true " +
-//                                "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-//                                " allParallelUnitTest"
-//                    }
-//                    post {
-//                        always {
-//                            junit '**/build/test-results-xml/**/*.xml'
-//                        }
-//                    }
-//                }
             }
 
         }
     }
-}
 
-@NonCPS
+    post {
-def killall_jobs() {
+        always {
-    def jobname = env.JOB_NAME
+            junit '**/build/test-results-xml/**/*.xml'
-    def buildnum = env.BUILD_NUMBER.toInteger()
-
-    def job = Jenkins.instance.getItemByFullName(jobname)
-    for (build in job.builds) {
-        if (!build.isBuilding()) {
-            continue;
         }
-
+        cleanup {
-        if (buildnum == build.getNumber().toInteger()) {
+            deleteDir() /* clean up our workspace */
-            continue
         }
-
-        echo "Killing task = ${build}"
-        build.doStop();
     }
 }

build.gradle

@@ -1,8 +1,10 @@
 import net.corda.testing.DistributedTesting
+import net.corda.testing.ImageBuilding
+import net.corda.testing.Distribution
 import net.corda.testing.ParallelTestGroup
 
-import static org.gradle.api.JavaVersion.VERSION_1_8
 import static org.gradle.api.JavaVersion.VERSION_11
+import static org.gradle.api.JavaVersion.VERSION_1_8
 
 buildscript {
     // For sharing constants between builds
@@ -123,8 +125,7 @@ buildscript {
         // has been compiled by a more recent version of the Java Runtime (class file version 55.0)
         ext.fontawesomefx_commons_version = '11.0'
         ext.fontawesomefx_fontawesome_version = '4.7.0-11'
-    }
+    } else {
-    else {
         ext.fontawesomefx_commons_version = '8.15'
         ext.fontawesomefx_fontawesome_version = '4.7.0-5'
     }
@@ -597,25 +598,37 @@ buildScan {
 
 task allParallelIntegrationTest(type: ParallelTestGroup) {
     testGroups "integrationTest"
-    numberOfShards 15
+    numberOfShards 10
     streamOutput false
     coresPerFork 6
     memoryInGbPerFork 10
+    distribute Distribution.CLASS
+}
+task allParallelSlowIntegrationTest(type: ParallelTestGroup) {
+    testGroups "slowIntegrationTest"
+    numberOfShards 4
+    streamOutput false
+    coresPerFork 6
+    memoryInGbPerFork 10
+    distribute Distribution.CLASS
 }
 task allParallelUnitTest(type: ParallelTestGroup) {
     testGroups "test"
-    numberOfShards 15
+    numberOfShards 10
     streamOutput false
-    coresPerFork 3
+    coresPerFork 5
     memoryInGbPerFork 6
+    distribute Distribution.CLASS
 }
 task allParallelUnitAndIntegrationTest(type: ParallelTestGroup) {
     testGroups "test", "integrationTest"
-    numberOfShards 20
+    numberOfShards 15
-    streamOutput false
+    streamOutput true
     coresPerFork 6
     memoryInGbPerFork 10
+    distribute Distribution.CLASS
 }
+apply plugin: ImageBuilding
 apply plugin: DistributedTesting
 
 

buildSrc/build.gradle

@@ -39,5 +39,8 @@ dependencies {
     compile 'commons-codec:commons-codec:1.13'
     compile "io.github.classgraph:classgraph:$class_graph_version"
     compile "com.bmuschko:gradle-docker-plugin:5.0.0"
+    compile 'org.apache.commons:commons-csv:1.1'
+    compile group: 'org.jetbrains', name: 'annotations', version: '13.0'
     testCompile "junit:junit:$junit_version"
+    testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
 }

buildSrc/src/main/groovy/net/corda/testing/BucketingAllocator.java

@@ -0,0 +1,164 @@
+package net.corda.testing;
+
+//Why Java?! because sometimes types are useful.
+
+import groovy.lang.Tuple2;
+import org.gradle.api.tasks.TaskAction;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.*;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class BucketingAllocator {
+
+    private List<Tuple2<TestLister, Object>> sources = new ArrayList<>();
+    private final List<TestsForForkContainer> forkContainers;
+    private final Supplier<List<Tuple2<String, Double>>> timedTestsProvider;
+
+
+    public BucketingAllocator(Integer forkCount, Supplier<List<Tuple2<String, Double>>> timedTestsProvider) {
+        this.forkContainers = IntStream.range(0, forkCount).mapToObj(TestsForForkContainer::new).collect(Collectors.toList());
+        this.timedTestsProvider = timedTestsProvider;
+    }
+
+    public void addSource(TestLister source, Object testTask) {
+        sources.add(new Tuple2<>(source, testTask));
+    }
+
+    public List<String> getTestsForForkAndTestTask(Integer fork, Object testTask) {
+        return forkContainers.get(fork).getTestsForTask(testTask);
+    }
+
+    @TaskAction
+    public void generateTestPlan() {
+        List<Tuple2<String, Double>> allTestsFromCSV = timedTestsProvider.get();
+        List<Tuple2<String, Object>> allDiscoveredTests = getTestsOnClasspathOfTestingTasks();
+        List<TestBucket> matchedTests = matchClasspathTestsToCSV(allTestsFromCSV, allDiscoveredTests);
+
+        //use greedy algo - for each testbucket find the currently smallest container and add to it
+        allocateTestsToForks(matchedTests);
+        forkContainers.forEach(TestsForForkContainer::freeze);
+
+        printSummary();
+    }
+
+    private void printSummary() {
+        forkContainers.forEach(container -> {
+            System.out.println("####### TEST PLAN SUMMARY ( " + container.forkIdx + " ) #######");
+            System.out.println("Duration: " + container.getCurrentDuration());
+            System.out.println("Number of tests: " + container.testsForFork.stream().mapToInt(b -> b.foundTests.size()).sum());
+            System.out.println("Tests to Run: ");
+            container.testsForFork.forEach(tb -> {
+                System.out.println(tb.nameWithAsterix);
+                tb.foundTests.forEach(ft -> System.out.println("\t" + ft.getFirst() + ", " + ft.getSecond()));
+            });
+        });
+    }
+
+
+    private void allocateTestsToForks(@NotNull List<TestBucket> matchedTests) {
+        matchedTests.forEach(matchedTestBucket -> {
+            TestsForForkContainer smallestContainer = Collections.min(forkContainers, Comparator.comparing(TestsForForkContainer::getCurrentDuration));
+            smallestContainer.addBucket(matchedTestBucket);
+        });
+    }
+
+    private List<TestBucket> matchClasspathTestsToCSV(List<Tuple2<String, Double>> allTestsFromCSV, @NotNull List<Tuple2<String, Object>> allDiscoveredTests) {
+        return allDiscoveredTests.stream().map(tuple -> {
+            String testName = tuple.getFirst();
+            Object task = tuple.getSecond();
+            String noAsterixName = testName.substring(0, testName.length() - 1);
+            //2DO [can this filtering algorithm be improved - the test names are sorted, it should be possible to do something using binary search]
+            List<Tuple2<String, Double>> matchingTests = allTestsFromCSV.stream().filter(testFromCSV -> testFromCSV.getFirst().startsWith(noAsterixName)).collect(Collectors.toList());
+            return new TestBucket(task, testName, noAsterixName, matchingTests);
+        }).sorted(Comparator.comparing(TestBucket::getDuration).reversed()).collect(Collectors.toList());
+    }
+
+    private List<Tuple2<String, Object>> getTestsOnClasspathOfTestingTasks() {
+        return sources.stream().map(source -> {
+            TestLister lister = source.getFirst();
+            Object testTask = source.getSecond();
+            return lister.getAllTestsDiscovered().stream().map(test -> new Tuple2<>(test, testTask)).collect(Collectors.toList());
+        }).flatMap(Collection::stream).collect(Collectors.toList());
+    }
+
+    public static class TestBucket {
+        final Object testTask;
+        final String nameWithAsterix;
+        final String nameWithoutAsterix;
+        final List<Tuple2<String, Double>> foundTests;
+        final Double duration;
+
+        public TestBucket(Object testTask, String nameWithAsterix, String nameWithoutAsterix, List<Tuple2<String, Double>> foundTests) {
+            this.testTask = testTask;
+            this.nameWithAsterix = nameWithAsterix;
+            this.nameWithoutAsterix = nameWithoutAsterix;
+            this.foundTests = foundTests;
+            duration = Math.max(foundTests.stream().mapToDouble(tp -> Math.max(tp.getSecond(), 10)).sum(), 10);
+        }
+
+        public Double getDuration() {
+            return duration;
+        }
+
+        @Override
+        public String toString() {
+            return "TestBucket{" +
+                    "testTask=" + testTask +
+                    ", nameWithAsterix='" + nameWithAsterix + '\'' +
+                    ", nameWithoutAsterix='" + nameWithoutAsterix + '\'' +
+                    ", foundTests=" + foundTests +
+                    ", duration=" + duration +
+                    '}';
+        }
+    }
+
+    public static class TestsForForkContainer {
+        private Double runningDuration = 0.0;
+        private final Integer forkIdx;
+
+        private final List<TestBucket> testsForFork = Collections.synchronizedList(new ArrayList<>());
+        private final Map<Object, List<TestBucket>> frozenTests = new HashMap<>();
+
+        public TestsForForkContainer(Integer forkIdx) {
+            this.forkIdx = forkIdx;
+        }
+
+        public void addBucket(TestBucket tb) {
+            this.testsForFork.add(tb);
+            this.runningDuration = runningDuration + tb.duration;
+        }
+
+        public Double getCurrentDuration() {
+            return runningDuration;
+        }
+
+        public void freeze() {
+            testsForFork.forEach(tb -> {
+                frozenTests.computeIfAbsent(tb.testTask, i -> new ArrayList<>()).add(tb);
+            });
+        }
+
+        public List<String> getTestsForTask(Object task) {
+            return frozenTests.getOrDefault(task, Collections.emptyList()).stream().map(it -> it.nameWithAsterix).collect(Collectors.toList());
+        }
+
+        public List<TestBucket> getBucketsForFork() {
+            return new ArrayList<>(testsForFork);
+        }
+
+        @Override
+        public String toString() {
+            return "TestsForForkContainer{" +
+                    "runningDuration=" + runningDuration +
+                    ", forkIdx=" + forkIdx +
+                    ", testsForFork=" + testsForFork +
+                    ", frozenTests=" + frozenTests +
+                    '}';
+        }
+    }
+
+
+}

buildSrc/src/main/groovy/net/corda/testing/BucketingAllocatorTask.java

@@ -0,0 +1,69 @@
+package net.corda.testing;
+
+import groovy.lang.Tuple2;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVRecord;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.testing.Test;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+public class BucketingAllocatorTask extends DefaultTask {
+    private static final String DEFAULT_TESTING_TEST_TIMES_CSV = "testing/test-times.csv";
+    private final BucketingAllocator allocator;
+
+    @Inject
+    public BucketingAllocatorTask(Integer forkCount) {
+        Supplier<List<Tuple2<String, Double>>> defaultTestCSV = () -> {
+            try {
+                FileReader csvSource = new FileReader(new File(BucketingAllocatorTask.this.getProject().getRootDir(), DEFAULT_TESTING_TEST_TIMES_CSV));
+                return fromCSV(csvSource);
+            } catch (IOException e) {
+                return Collections.emptyList();
+            }
+        };
+        this.allocator = new BucketingAllocator(forkCount, defaultTestCSV);
+    }
+
+    public void addSource(TestLister source, Test testTask) {
+        allocator.addSource(source, testTask);
+        this.dependsOn(source);
+    }
+
+    public List<String> getTestsForForkAndTestTask(Integer fork, Test testTask) {
+        return allocator.getTestsForForkAndTestTask(fork, testTask);
+    }
+
+    @TaskAction
+    public void allocate() {
+        allocator.generateTestPlan();
+    }
+
+
+    public static List<Tuple2<String, Double>> fromCSV(Reader reader) throws IOException {
+        String name = "Test Name";
+        String duration = "Duration(ms)";
+        List<CSVRecord> records = CSVFormat.DEFAULT.withHeader().parse(reader).getRecords();
+        return records.stream().map(record -> {
+            try{
+                String testName = record.get(name);
+                String testDuration = record.get(duration);
+                return new Tuple2<>(testName, Math.max(Double.parseDouble(testDuration), 10));
+            }catch (IllegalArgumentException | IllegalStateException  e){
+                return null;
+            }
+        }).filter(Objects::nonNull).sorted(Comparator.comparing(Tuple2::getFirst)).collect(Collectors.toList());
+    }
+
+}

buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy

@@ -10,6 +10,7 @@ import org.gradle.api.tasks.testing.Test
  */
 class DistributedTesting implements Plugin<Project> {
 
+
     static def getPropertyAsInt(Project proj, String property, Integer defaultValue) {
         return proj.hasProperty(property) ? Integer.parseInt(proj.property(property).toString()) : defaultValue
     }
@@ -17,10 +18,16 @@ class DistributedTesting implements Plugin<Project> {
     @Override
     void apply(Project project) {
         if (System.getProperty("kubenetize") != null) {
+
+            def forks = getPropertyAsInt(project, "dockerForks", 1)
+
             ensureImagePluginIsApplied(project)
             ImageBuilding imagePlugin = project.plugins.getPlugin(ImageBuilding)
             DockerPushImage imageBuildingTask = imagePlugin.pushTask
             String providedTag = System.getProperty("docker.tag")
+            BucketingAllocatorTask globalAllocator = project.tasks.create("bucketingAllocator", BucketingAllocatorTask, forks)
+
+            def requestedTasks = project.gradle.startParameter.taskNames.collect { project.tasks.findByPath(it) }
 
             //in each subproject
             //1. add the task to determine all tests within the module
@@ -28,9 +35,19 @@ class DistributedTesting implements Plugin<Project> {
             //3. KubesTest will invoke these test tasks in a parallel fashion on a remote k8s cluster
             project.subprojects { Project subProject ->
                 subProject.tasks.withType(Test) { Test task ->
-                    ListTests testListerTask = createTestListingTasks(task, subProject)
+                    println "Evaluating ${task.getPath()}"
-                    Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, testListerTask)
+                    if (task in requestedTasks && !task.hasProperty("ignoreForDistribution")) {
-                    KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
+                        println "Modifying ${task.getPath()}"
+                        ListTests testListerTask = createTestListingTasks(task, subProject)
+                        globalAllocator.addSource(testListerTask, task)
+                        Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, globalAllocator)
+                    } else {
+                        println "Skipping modification of ${task.getPath()} as it's not scheduled for execution"
+                    }
+                    if (!task.hasProperty("ignoreForDistribution")) {
+                        KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
+                    }
+
                 }
             }
 
@@ -62,6 +79,7 @@ class DistributedTesting implements Plugin<Project> {
                     taskToExecuteName = testGrouping.groups.join("And")
                     memoryGbPerFork = testGrouping.gbOfMemory
                     numberOfCoresPerFork = testGrouping.coresToUse
+                    distribution = testGrouping.distribution
                     doFirst {
                         dockerTag = dockerTag = providedTag ? ImageBuilding.registryName + ":" + providedTag : (imageBuildingTask.imageName.get() + ":" + imageBuildingTask.tag.get())
                     }
@@ -101,24 +119,21 @@ class DistributedTesting implements Plugin<Project> {
         return createdParallelTestTask as KubesTest
     }
 
-    private Test modifyTestTaskForParallelExecution(Project subProject, Test task, ListTests testListerTask) {
+    private Test modifyTestTaskForParallelExecution(Project subProject, Test task, BucketingAllocatorTask globalAllocator) {
-        subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${testListerTask.getPath()}")
+        subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${globalAllocator.getPath()}")
         def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name)
         task.configure {
-            dependsOn testListerTask
+            dependsOn globalAllocator
             binResultsDir new File(reportsDir, "binary")
             reports.junitXml.destination new File(reportsDir, "xml")
             maxHeapSize = "6g"
             doFirst {
                 filter {
                     def fork = getPropertyAsInt(subProject, "dockerFork", 0)
-                    def forks = getPropertyAsInt(subProject, "dockerForks", 1)
+                    subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (idx: ${fork})")
-                    def shuffleSeed = 42
+                    List<String> includes = globalAllocator.getTestsForForkAndTestTask(
-                    subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (${fork}, ${forks}, ${shuffleSeed})")
-                    List<String> includes = testListerTask.getTestsForFork(
                             fork,
-                            forks,
+                            task)
-                            shuffleSeed)
                     subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}"
 
                     if (includes.size() == 0) {

buildSrc/src/main/groovy/net/corda/testing/ImageBuilding.groovy

@@ -48,6 +48,8 @@ class ImageBuilding implements Plugin<Project> {
                 if (!mavenDir.exists()) {
                     mavenDir.mkdirs()
                 }
+
+                logger.info("Will use: ${gradleDir.absolutePath} for caching gradle artifacts")
             }
 
             dependsOn buildDockerImageForSource

buildSrc/src/main/groovy/net/corda/testing/KubesTest.groovy

@@ -39,6 +39,8 @@ class KubesTest extends DefaultTask {
     int numberOfPods = 20
     int timeoutInMinutesForPodToStart = 60
 
+    Distribution distribution = Distribution.METHOD
+
     @TaskAction
     void runTestsOnKubes() {
 
@@ -78,7 +80,8 @@ class KubesTest extends DefaultTask {
         }
 
         List<CompletableFuture<KubePodResult>> futures = IntStream.range(0, numberOfPods).mapToObj({ i ->
-            String podName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
+            String potentialPodName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
+            String podName = potentialPodName.substring(0, Math.min(potentialPodName.size(), 62))
             runBuild(client, namespace, numberOfPods, i, podName, printOutput, 3)
         }).collect(Collectors.toList())
         this.testOutput = Collections.synchronizedList(futures.collect { it -> it.get().binaryResults }.flatten())
@@ -115,10 +118,11 @@ class KubesTest extends DefaultTask {
                     CompletableFuture<KubePodResult> waiter = new CompletableFuture<>()
                     ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter, result)
                     stdOutIs.connect(stdOutOs)
+                    String[] buildCommand = getBuildCommand(numberOfPods, podIdx)
                     ExecWatch execWatch = client.pods().inNamespace(namespace).withName(podName)
                             .writingOutput(stdOutOs)
                             .writingErrorChannel(errChannelStream)
-                            .usingListener(execListener).exec(getBuildCommand(numberOfPods, podIdx))
+                            .usingListener(execListener).exec(buildCommand)
 
                     startLogPumping(outputFile, stdOutIs, podIdx, printOutput)
                     KubePodResult execResult = waiter.join()
@@ -183,20 +187,21 @@ class KubesTest extends DefaultTask {
     ExecListener buildExecListenerForPod(podName, errChannelStream, CompletableFuture<KubePodResult> waitingFuture, KubePodResult result) {
 
         new ExecListener() {
+            final Long start = System.currentTimeMillis()
             @Override
             void onOpen(Response response) {
-                project.logger.lifecycle("Build started on pod " + podName)
+                project.logger.lifecycle("Build started on pod $podName")
             }
 
             @Override
             void onFailure(Throwable t, Response response) {
-                project.logger.lifecycle("Received error from rom pod " + podName)
+                project.logger.lifecycle("Received error from rom pod $podName")
                 waitingFuture.completeExceptionally(t)
             }
 
             @Override
             void onClose(int code, String reason) {
-                project.logger.lifecycle("Received onClose() from pod " + podName + " with returnCode=" + code)
+                project.logger.lifecycle("Received onClose() from pod ${podName}, build took: ${(System.currentTimeMillis() - start) / 1000} seconds")
                 try {
                     def errChannelContents = errChannelStream.toString()
                     Status status = Serialization.unmarshal(errChannelContents, Status.class);
@@ -277,7 +282,7 @@ class KubesTest extends DefaultTask {
                 "let x=1 ; while [ \${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=\$? ; sleep 1 ; done ; " +
                         "cd /tmp/source ; " +
                         "let y=1 ; while [ \${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=\$? ; sleep 1 ; done ;" +
-                        "./gradlew -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
+                        "./gradlew -D${ListTests.DISTRIBUTION_PROPERTY}=${distribution.name()} -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
                         "let rs=\$? ; sleep 10 ; exit \${rs}"]
     }
 

buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy

@@ -37,10 +37,17 @@ class ListShufflerAndAllocator {
     }
 }
 
-class ListTests extends DefaultTask {
+interface TestLister {
+    List<String> getAllTestsDiscovered()
+}
+
+class ListTests extends DefaultTask implements TestLister {
+
+    public static final String DISTRIBUTION_PROPERTY = "distributeBy"
 
     FileCollection scanClassPath
     List<String> allTests
+    Distribution distribution = System.getProperty(DISTRIBUTION_PROPERTY) ? Distribution.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : Distribution.METHOD
 
     def getTestsForFork(int fork, int forks, Integer seed) {
         def gitSha = new BigInteger(project.hasProperty("corda_revision") ? project.property("corda_revision").toString() : "0", 36)
@@ -51,24 +58,53 @@ class ListTests extends DefaultTask {
         return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse)
     }
 
+    @Override
+    public List<String> getAllTestsDiscovered() {
+        return new ArrayList<>(allTests)
+    }
+
     @TaskAction
     def discoverTests() {
-        Collection<String> results = new ClassGraph()
+        switch (distribution) {
-                .enableClassInfo()
+            case Distribution.METHOD:
-                .enableMethodInfo()
+                Collection<String> results = new ClassGraph()
-                .ignoreClassVisibility()
+                        .enableClassInfo()
-                .ignoreMethodVisibility()
+                        .enableMethodInfo()
-                .enableAnnotationInfo()
+                        .ignoreClassVisibility()
-                .overrideClasspath(scanClassPath)
+                        .ignoreMethodVisibility()
-                .scan()
+                        .enableAnnotationInfo()
-                .getClassesWithMethodAnnotation("org.junit.Test")
+                        .overrideClasspath(scanClassPath)
-                .collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
+                        .scan()
-                .flatten()
+                        .getClassesWithMethodAnnotation("org.junit.Test")
-                .collect { ClassInfo c ->
+                        .collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
-                    c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
+                        .flatten()
-                }.flatten()
+                        .collect { ClassInfo c ->
-                .toSet()
+                            c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
+                        }.flatten()
+                        .toSet()
 
-        this.allTests = results.stream().sorted().collect(Collectors.toList())
+                this.allTests = results.stream().sorted().collect(Collectors.toList())
+                break
+            case Distribution.CLASS:
+                Collection<String> results = new ClassGraph()
+                        .enableClassInfo()
+                        .enableMethodInfo()
+                        .ignoreClassVisibility()
+                        .ignoreMethodVisibility()
+                        .enableAnnotationInfo()
+                        .overrideClasspath(scanClassPath)
+                        .scan()
+                        .getClassesWithMethodAnnotation("org.junit.Test")
+                        .collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
+                        .flatten()
+                        .collect { ClassInfo c -> c.name + "*" }.flatten()
+                        .toSet()
+                this.allTests = results.stream().sorted().collect(Collectors.toList())
+                break
+        }
     }
 }
+
+public enum Distribution {
+    CLASS, METHOD
+}

buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy

@@ -1,9 +1,9 @@
 package net.corda.testing
 
 import org.gradle.api.DefaultTask
-import org.gradle.api.tasks.TaskAction
 
 class ParallelTestGroup extends DefaultTask {
+    Distribution distribution = Distribution.METHOD
 
     List<String> groups = new ArrayList<>()
     int shardCount = 20
@@ -11,20 +11,24 @@ class ParallelTestGroup extends DefaultTask {
     int gbOfMemory = 4
     boolean printToStdOut = true
 
-    void numberOfShards(int shards){
+    void numberOfShards(int shards) {
         this.shardCount = shards
     }
 
-    void coresPerFork(int cores){
+    void distribute(Distribution dist){
+        this.distribution = dist
+    }
+
+    void coresPerFork(int cores) {
         this.coresToUse = cores
     }
 
-    void memoryInGbPerFork(int gb){
+    void memoryInGbPerFork(int gb) {
         this.gbOfMemory = gb
     }
 
     //when this is false, only containers will "failed" exit codes will be printed to stdout
-    void streamOutput(boolean print){
+    void streamOutput(boolean print) {
         this.printToStdOut = print
     }
 

buildSrc/src/test/java/net/corda/testing/BucketingAllocatorTest.java

@@ -0,0 +1,54 @@
+package net.corda.testing;
+
+import org.hamcrest.collection.IsIterableContainingInAnyOrder;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.hamcrest.CoreMatchers.is;
+
+public class BucketingAllocatorTest {
+
+    @Test
+    public void shouldAlwaysBucketTestsEvenIfNotInTimedFile() {
+
+        BucketingAllocator bucketingAllocator = new BucketingAllocator(1, Collections::emptyList);
+
+        Object task = new Object();
+        bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
+
+        bucketingAllocator.generateTestPlan();
+        List<String> testsForForkAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
+
+        Assert.assertThat(testsForForkAndTestTask, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
+
+    }
+
+
+    @Test
+    public void shouldAllocateTestsAcrossForksEvenIfNoMatchingTestsFound() {
+
+        BucketingAllocator bucketingAllocator = new BucketingAllocator(2, Collections::emptyList);
+
+        Object task = new Object();
+        bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
+
+        bucketingAllocator.generateTestPlan();
+        List<String> testsForForkOneAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
+        List<String> testsForForkTwoAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(1, task);
+
+        Assert.assertThat(testsForForkOneAndTestTask.size(), is(1));
+        Assert.assertThat(testsForForkTwoAndTestTask.size(), is(1));
+
+        List<String> allTests = Stream.of(testsForForkOneAndTestTask, testsForForkTwoAndTestTask).flatMap(Collection::stream).collect(Collectors.toList());
+
+        Assert.assertThat(allTests, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
+
+    }
+}

core-deterministic/testing/data/build.gradle

@@ -22,6 +22,9 @@ dependencies {
 jar.enabled = false
 
 test {
+    ext {
+        ignoreForDistribution = true
+    }
     filter {
         // Running this class is the whole point, so include it explicitly.
         includeTestsMatching "net.corda.deterministic.data.GenerateData"

testing/Dockerfile

@@ -1,4 +1,4 @@
 FROM stefanotestingcr.azurecr.io/buildbase:latest
 COPY . /tmp/source
-CMD ls /tmp/gradle && cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info
+CMD cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info
 

testing/DockerfileBase

@@ -2,7 +2,7 @@ FROM ubuntu:18.04
 ENV GRADLE_USER_HOME=/tmp/gradle
 RUN mkdir /tmp/gradle && mkdir -p /home/root/.m2/repository
 
-RUN apt-get update && apt-get install -y curl && \
+RUN apt-get update && apt-get install -y curl libatomic1 && \
     curl -O https://d3pxv6yz143wms.cloudfront.net/8.222.10.1/java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
     apt-get install -y java-common && dpkg -i java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
     apt-get clean && \