ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2024-12-18 20:47:57 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Webook test branch (#5528)

Browse Source 
* add ability to index by class OR method

* disable unit tests again

* pending commit

* stream output of build for debugging

* remove git from the base image

* re-enable NodeRPCTests

* add ability to distribute tests based on TeamCity CSV output for test times

* try enabling unit tests again after test distribution has been fixed

* refactor BucketingAllocator to be a class, which bucketingAllocatorTask delegates to.

* finishing touches for improved test distribution

* create new pipelines for builds
This commit is contained in:
Stefano Franz 2019-10-08 16:33:24 +02:00 committed by GitHub
parent ca0c7e02ae
commit 298c91ce82
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 3838 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
.ci/dev/integration/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,59 @@
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda - Run Tests') {
stage('Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelIntegrationTest"
if (env.CHANGE_ID) {
pullRequest.createStatus(status: 'success',
context: 'continuous-integration/jenkins/pr-merge/integrationTest',
description: 'Integration Tests Passed',
targetUrl: "${env.JOB_URL}/testResults")
}
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

61
.ci/dev/regression/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,61 @@
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda Pull Request - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda Pull Request - Run Tests') {
stage('Unit and Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitAndIntegrationTest"
}
}
stage('Slow Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelSlowIntegrationTest"
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

59
.ci/dev/unit/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,59 @@
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda Pull Request - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda Pull Request - Run Tests') {
stage('Unit Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitTest"
if (env.CHANGE_ID) {
pullRequest.createStatus(status: 'success',
context: 'continuous-integration/jenkins/pr-merge/unitTest',
description: 'Unit Tests Passed',
targetUrl: "${env.JOB_URL}/testResults")
}
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

52
Jenkinsfile vendored
View File

@ -1,4 +1,7 @@
killall_jobs()
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
@ -35,48 +38,27 @@ pipeline {
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelIntegrationTest"
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
}
stage('Unit Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitTest"
}
}
// stage('Unit Tests') {
// steps {
// sh "./gradlew " +
// "-DbuildId=\"\${BUILD_ID}\" " +
// "-Dkubenetize=true " +
// "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
// " allParallelUnitTest"
// }
// post {
// always {
// junit '**/build/test-results-xml/**/*.xml'
// }
// }
// }
}
}
}
}
@NonCPS
def killall_jobs() {
def jobname = env.JOB_NAME
def buildnum = env.BUILD_NUMBER.toInteger()
def job = Jenkins.instance.getItemByFullName(jobname)
for (build in job.builds) {
if (!build.isBuilding()) {
continue;
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
if (buildnum == build.getNumber().toInteger()) {
continue
cleanup {
deleteDir() /* clean up our workspace */
}
echo "Killing task = ${build}"
build.doStop();
}
}

29
build.gradle
View File

@ -1,8 +1,10 @@
import net.corda.testing.DistributedTesting
import net.corda.testing.ImageBuilding
import net.corda.testing.Distribution
import net.corda.testing.ParallelTestGroup
import static org.gradle.api.JavaVersion.VERSION_1_8
import static org.gradle.api.JavaVersion.VERSION_11
import static org.gradle.api.JavaVersion.VERSION_1_8
buildscript {
// For sharing constants between builds
@ -123,8 +125,7 @@ buildscript {
// has been compiled by a more recent version of the Java Runtime (class file version 55.0)
ext.fontawesomefx_commons_version = '11.0'
ext.fontawesomefx_fontawesome_version = '4.7.0-11'
}
else {
} else {
ext.fontawesomefx_commons_version = '8.15'
ext.fontawesomefx_fontawesome_version = '4.7.0-5'
}
@ -597,25 +598,37 @@ buildScan {
task allParallelIntegrationTest(type: ParallelTestGroup) {
testGroups "integrationTest"
numberOfShards 15
numberOfShards 10
streamOutput false
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
task allParallelSlowIntegrationTest(type: ParallelTestGroup) {
testGroups "slowIntegrationTest"
numberOfShards 4
streamOutput false
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
task allParallelUnitTest(type: ParallelTestGroup) {
testGroups "test"
numberOfShards 15
numberOfShards 10
streamOutput false
coresPerFork 3
coresPerFork 5
memoryInGbPerFork 6
distribute Distribution.CLASS
}
task allParallelUnitAndIntegrationTest(type: ParallelTestGroup) {
testGroups "test", "integrationTest"
numberOfShards 20
streamOutput false
numberOfShards 15
streamOutput true
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
apply plugin: ImageBuilding
apply plugin: DistributedTesting

3
buildSrc/build.gradle
View File

@ -39,5 +39,8 @@ dependencies {
compile 'commons-codec:commons-codec:1.13'
compile "io.github.classgraph:classgraph:$class_graph_version"
compile "com.bmuschko:gradle-docker-plugin:5.0.0"
compile 'org.apache.commons:commons-csv:1.1'
compile group: 'org.jetbrains', name: 'annotations', version: '13.0'
testCompile "junit:junit:$junit_version"
testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
}

164
buildSrc/src/main/groovy/net/corda/testing/BucketingAllocator.java Normal file
View File

@ -0,0 +1,164 @@
package net.corda.testing;
//Why Java?! because sometimes types are useful.
import groovy.lang.Tuple2;
import org.gradle.api.tasks.TaskAction;
import org.jetbrains.annotations.NotNull;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public class BucketingAllocator {
private List<Tuple2<TestLister, Object>> sources = new ArrayList<>();
private final List<TestsForForkContainer> forkContainers;
private final Supplier<List<Tuple2<String, Double>>> timedTestsProvider;
public BucketingAllocator(Integer forkCount, Supplier<List<Tuple2<String, Double>>> timedTestsProvider) {
this.forkContainers = IntStream.range(0, forkCount).mapToObj(TestsForForkContainer::new).collect(Collectors.toList());
this.timedTestsProvider = timedTestsProvider;
}
public void addSource(TestLister source, Object testTask) {
sources.add(new Tuple2<>(source, testTask));
}
public List<String> getTestsForForkAndTestTask(Integer fork, Object testTask) {
return forkContainers.get(fork).getTestsForTask(testTask);
}
@TaskAction
public void generateTestPlan() {
List<Tuple2<String, Double>> allTestsFromCSV = timedTestsProvider.get();
List<Tuple2<String, Object>> allDiscoveredTests = getTestsOnClasspathOfTestingTasks();
List<TestBucket> matchedTests = matchClasspathTestsToCSV(allTestsFromCSV, allDiscoveredTests);
//use greedy algo - for each testbucket find the currently smallest container and add to it
allocateTestsToForks(matchedTests);
forkContainers.forEach(TestsForForkContainer::freeze);
printSummary();
}
private void printSummary() {
forkContainers.forEach(container -> {
System.out.println("####### TEST PLAN SUMMARY ( " + container.forkIdx + " ) #######");
System.out.println("Duration: " + container.getCurrentDuration());
System.out.println("Number of tests: " + container.testsForFork.stream().mapToInt(b -> b.foundTests.size()).sum());
System.out.println("Tests to Run: ");
container.testsForFork.forEach(tb -> {
System.out.println(tb.nameWithAsterix);
tb.foundTests.forEach(ft -> System.out.println("\t" + ft.getFirst() + ", " + ft.getSecond()));
});
});
}
private void allocateTestsToForks(@NotNull List<TestBucket> matchedTests) {
matchedTests.forEach(matchedTestBucket -> {
TestsForForkContainer smallestContainer = Collections.min(forkContainers, Comparator.comparing(TestsForForkContainer::getCurrentDuration));
smallestContainer.addBucket(matchedTestBucket);
});
}
private List<TestBucket> matchClasspathTestsToCSV(List<Tuple2<String, Double>> allTestsFromCSV, @NotNull List<Tuple2<String, Object>> allDiscoveredTests) {
return allDiscoveredTests.stream().map(tuple -> {
String testName = tuple.getFirst();
Object task = tuple.getSecond();
String noAsterixName = testName.substring(0, testName.length() - 1);
//2DO [can this filtering algorithm be improved - the test names are sorted, it should be possible to do something using binary search]
List<Tuple2<String, Double>> matchingTests = allTestsFromCSV.stream().filter(testFromCSV -> testFromCSV.getFirst().startsWith(noAsterixName)).collect(Collectors.toList());
return new TestBucket(task, testName, noAsterixName, matchingTests);
}).sorted(Comparator.comparing(TestBucket::getDuration).reversed()).collect(Collectors.toList());
}
private List<Tuple2<String, Object>> getTestsOnClasspathOfTestingTasks() {
return sources.stream().map(source -> {
TestLister lister = source.getFirst();
Object testTask = source.getSecond();
return lister.getAllTestsDiscovered().stream().map(test -> new Tuple2<>(test, testTask)).collect(Collectors.toList());
}).flatMap(Collection::stream).collect(Collectors.toList());
}
public static class TestBucket {
final Object testTask;
final String nameWithAsterix;
final String nameWithoutAsterix;
final List<Tuple2<String, Double>> foundTests;
final Double duration;
public TestBucket(Object testTask, String nameWithAsterix, String nameWithoutAsterix, List<Tuple2<String, Double>> foundTests) {
this.testTask = testTask;
this.nameWithAsterix = nameWithAsterix;
this.nameWithoutAsterix = nameWithoutAsterix;
this.foundTests = foundTests;
duration = Math.max(foundTests.stream().mapToDouble(tp -> Math.max(tp.getSecond(), 10)).sum(), 10);
}
public Double getDuration() {
return duration;
}
@Override
public String toString() {
return "TestBucket{" +
"testTask=" + testTask +
", nameWithAsterix='" + nameWithAsterix + '\'' +
", nameWithoutAsterix='" + nameWithoutAsterix + '\'' +
", foundTests=" + foundTests +
", duration=" + duration +
'}';
}
}
public static class TestsForForkContainer {
private Double runningDuration = 0.0;
private final Integer forkIdx;
private final List<TestBucket> testsForFork = Collections.synchronizedList(new ArrayList<>());
private final Map<Object, List<TestBucket>> frozenTests = new HashMap<>();
public TestsForForkContainer(Integer forkIdx) {
this.forkIdx = forkIdx;
}
public void addBucket(TestBucket tb) {
this.testsForFork.add(tb);
this.runningDuration = runningDuration + tb.duration;
}
public Double getCurrentDuration() {
return runningDuration;
}
public void freeze() {
testsForFork.forEach(tb -> {
frozenTests.computeIfAbsent(tb.testTask, i -> new ArrayList<>()).add(tb);
});
}
public List<String> getTestsForTask(Object task) {
return frozenTests.getOrDefault(task, Collections.emptyList()).stream().map(it -> it.nameWithAsterix).collect(Collectors.toList());
}
public List<TestBucket> getBucketsForFork() {
return new ArrayList<>(testsForFork);
}
@Override
public String toString() {
return "TestsForForkContainer{" +
"runningDuration=" + runningDuration +
", forkIdx=" + forkIdx +
", testsForFork=" + testsForFork +
", frozenTests=" + frozenTests +
'}';
}
}
}

69
buildSrc/src/main/groovy/net/corda/testing/BucketingAllocatorTask.java Normal file
View File

@ -0,0 +1,69 @@
package net.corda.testing;
import groovy.lang.Tuple2;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.TaskAction;
import org.gradle.api.tasks.testing.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.stream.Collectors;
public class BucketingAllocatorTask extends DefaultTask {
private static final String DEFAULT_TESTING_TEST_TIMES_CSV = "testing/test-times.csv";
private final BucketingAllocator allocator;
@Inject
public BucketingAllocatorTask(Integer forkCount) {
Supplier<List<Tuple2<String, Double>>> defaultTestCSV = () -> {
try {
FileReader csvSource = new FileReader(new File(BucketingAllocatorTask.this.getProject().getRootDir(), DEFAULT_TESTING_TEST_TIMES_CSV));
return fromCSV(csvSource);
} catch (IOException e) {
return Collections.emptyList();
}
};
this.allocator = new BucketingAllocator(forkCount, defaultTestCSV);
}
public void addSource(TestLister source, Test testTask) {
allocator.addSource(source, testTask);
this.dependsOn(source);
}
public List<String> getTestsForForkAndTestTask(Integer fork, Test testTask) {
return allocator.getTestsForForkAndTestTask(fork, testTask);
}
@TaskAction
public void allocate() {
allocator.generateTestPlan();
}
public static List<Tuple2<String, Double>> fromCSV(Reader reader) throws IOException {
String name = "Test Name";
String duration = "Duration(ms)";
List<CSVRecord> records = CSVFormat.DEFAULT.withHeader().parse(reader).getRecords();
return records.stream().map(record -> {
try{
String testName = record.get(name);
String testDuration = record.get(duration);
return new Tuple2<>(testName, Math.max(Double.parseDouble(testDuration), 10));
}catch (IllegalArgumentException | IllegalStateException e){
return null;
}
}).filter(Objects::nonNull).sorted(Comparator.comparing(Tuple2::getFirst)).collect(Collectors.toList());
}
}

39
buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy
View File

@ -10,6 +10,7 @@ import org.gradle.api.tasks.testing.Test
*/
class DistributedTesting implements Plugin<Project> {
static def getPropertyAsInt(Project proj, String property, Integer defaultValue) {
return proj.hasProperty(property) ? Integer.parseInt(proj.property(property).toString()) : defaultValue
}
@ -17,10 +18,16 @@ class DistributedTesting implements Plugin<Project> {
@Override
void apply(Project project) {
if (System.getProperty("kubenetize") != null) {
def forks = getPropertyAsInt(project, "dockerForks", 1)
ensureImagePluginIsApplied(project)
ImageBuilding imagePlugin = project.plugins.getPlugin(ImageBuilding)
DockerPushImage imageBuildingTask = imagePlugin.pushTask
String providedTag = System.getProperty("docker.tag")
BucketingAllocatorTask globalAllocator = project.tasks.create("bucketingAllocator", BucketingAllocatorTask, forks)
def requestedTasks = project.gradle.startParameter.taskNames.collect { project.tasks.findByPath(it) }
//in each subproject
//1. add the task to determine all tests within the module
@ -28,9 +35,19 @@ class DistributedTesting implements Plugin<Project> {
//3. KubesTest will invoke these test tasks in a parallel fashion on a remote k8s cluster
project.subprojects { Project subProject ->
subProject.tasks.withType(Test) { Test task ->
ListTests testListerTask = createTestListingTasks(task, subProject)
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, testListerTask)
KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
println "Evaluating ${task.getPath()}"
if (task in requestedTasks && !task.hasProperty("ignoreForDistribution")) {
println "Modifying ${task.getPath()}"
ListTests testListerTask = createTestListingTasks(task, subProject)
globalAllocator.addSource(testListerTask, task)
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, globalAllocator)
} else {
println "Skipping modification of ${task.getPath()} as it's not scheduled for execution"
}
if (!task.hasProperty("ignoreForDistribution")) {
KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
}
}
}
@ -62,6 +79,7 @@ class DistributedTesting implements Plugin<Project> {
taskToExecuteName = testGrouping.groups.join("And")
memoryGbPerFork = testGrouping.gbOfMemory
numberOfCoresPerFork = testGrouping.coresToUse
distribution = testGrouping.distribution
doFirst {
dockerTag = dockerTag = providedTag ? ImageBuilding.registryName + ":" + providedTag : (imageBuildingTask.imageName.get() + ":" + imageBuildingTask.tag.get())
}
@ -101,24 +119,21 @@ class DistributedTesting implements Plugin<Project> {
return createdParallelTestTask as KubesTest
}
private Test modifyTestTaskForParallelExecution(Project subProject, Test task, ListTests testListerTask) {
subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${testListerTask.getPath()}")
private Test modifyTestTaskForParallelExecution(Project subProject, Test task, BucketingAllocatorTask globalAllocator) {
subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${globalAllocator.getPath()}")
def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name)
task.configure {
dependsOn testListerTask
dependsOn globalAllocator
binResultsDir new File(reportsDir, "binary")
reports.junitXml.destination new File(reportsDir, "xml")
maxHeapSize = "6g"
doFirst {
filter {
def fork = getPropertyAsInt(subProject, "dockerFork", 0)
def forks = getPropertyAsInt(subProject, "dockerForks", 1)
def shuffleSeed = 42
subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (${fork}, ${forks}, ${shuffleSeed})")
List<String> includes = testListerTask.getTestsForFork(
subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (idx: ${fork})")
List<String> includes = globalAllocator.getTestsForForkAndTestTask(
fork,
forks,
shuffleSeed)
task)
subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}"
if (includes.size() == 0) {

2
buildSrc/src/main/groovy/net/corda/testing/ImageBuilding.groovy
View File

@ -48,6 +48,8 @@ class ImageBuilding implements Plugin<Project> {
if (!mavenDir.exists()) {
mavenDir.mkdirs()
}
logger.info("Will use: ${gradleDir.absolutePath} for caching gradle artifacts")
}
dependsOn buildDockerImageForSource

17
buildSrc/src/main/groovy/net/corda/testing/KubesTest.groovy
View File

@ -39,6 +39,8 @@ class KubesTest extends DefaultTask {
int numberOfPods = 20
int timeoutInMinutesForPodToStart = 60
Distribution distribution = Distribution.METHOD
@TaskAction
void runTestsOnKubes() {
@ -78,7 +80,8 @@ class KubesTest extends DefaultTask {
}
List<CompletableFuture<KubePodResult>> futures = IntStream.range(0, numberOfPods).mapToObj({ i ->
String podName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
String potentialPodName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
String podName = potentialPodName.substring(0, Math.min(potentialPodName.size(), 62))
runBuild(client, namespace, numberOfPods, i, podName, printOutput, 3)
}).collect(Collectors.toList())
this.testOutput = Collections.synchronizedList(futures.collect { it -> it.get().binaryResults }.flatten())
@ -115,10 +118,11 @@ class KubesTest extends DefaultTask {
CompletableFuture<KubePodResult> waiter = new CompletableFuture<>()
ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter, result)
stdOutIs.connect(stdOutOs)
String[] buildCommand = getBuildCommand(numberOfPods, podIdx)
ExecWatch execWatch = client.pods().inNamespace(namespace).withName(podName)
.writingOutput(stdOutOs)
.writingErrorChannel(errChannelStream)
.usingListener(execListener).exec(getBuildCommand(numberOfPods, podIdx))
.usingListener(execListener).exec(buildCommand)
startLogPumping(outputFile, stdOutIs, podIdx, printOutput)
KubePodResult execResult = waiter.join()
@ -183,20 +187,21 @@ class KubesTest extends DefaultTask {
ExecListener buildExecListenerForPod(podName, errChannelStream, CompletableFuture<KubePodResult> waitingFuture, KubePodResult result) {
new ExecListener() {
final Long start = System.currentTimeMillis()
@Override
void onOpen(Response response) {
project.logger.lifecycle("Build started on pod " + podName)
project.logger.lifecycle("Build started on pod $podName")
}
@Override
void onFailure(Throwable t, Response response) {
project.logger.lifecycle("Received error from rom pod " + podName)
project.logger.lifecycle("Received error from rom pod $podName")
waitingFuture.completeExceptionally(t)
}
@Override
void onClose(int code, String reason) {
project.logger.lifecycle("Received onClose() from pod " + podName + " with returnCode=" + code)
project.logger.lifecycle("Received onClose() from pod ${podName}, build took: ${(System.currentTimeMillis() - start) / 1000} seconds")
try {
def errChannelContents = errChannelStream.toString()
Status status = Serialization.unmarshal(errChannelContents, Status.class);
@ -277,7 +282,7 @@ class KubesTest extends DefaultTask {
"let x=1 ; while [ \${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=\$? ; sleep 1 ; done ; " +
"cd /tmp/source ; " +
"let y=1 ; while [ \${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=\$? ; sleep 1 ; done ;" +
"./gradlew -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
"./gradlew -D${ListTests.DISTRIBUTION_PROPERTY}=${distribution.name()} -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
"let rs=\$? ; sleep 10 ; exit \${rs}"]
}

70
buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy
View File

@ -37,10 +37,17 @@ class ListShufflerAndAllocator {
}
}
class ListTests extends DefaultTask {
interface TestLister {
List<String> getAllTestsDiscovered()
}
class ListTests extends DefaultTask implements TestLister {
public static final String DISTRIBUTION_PROPERTY = "distributeBy"
FileCollection scanClassPath
List<String> allTests
Distribution distribution = System.getProperty(DISTRIBUTION_PROPERTY) ? Distribution.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : Distribution.METHOD
def getTestsForFork(int fork, int forks, Integer seed) {
def gitSha = new BigInteger(project.hasProperty("corda_revision") ? project.property("corda_revision").toString() : "0", 36)
@ -51,24 +58,53 @@ class ListTests extends DefaultTask {
return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse)
}
@Override
public List<String> getAllTestsDiscovered() {
return new ArrayList<>(allTests)
}
@TaskAction
def discoverTests() {
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c ->
c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
}.flatten()
.toSet()
switch (distribution) {
case Distribution.METHOD:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c ->
c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
}.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
case Distribution.CLASS:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c -> c.name + "*" }.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
}
}
}
public enum Distribution {
CLASS, METHOD
}

14
buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy
View File

@ -1,9 +1,9 @@
package net.corda.testing
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
class ParallelTestGroup extends DefaultTask {
Distribution distribution = Distribution.METHOD
List<String> groups = new ArrayList<>()
int shardCount = 20
@ -11,20 +11,24 @@ class ParallelTestGroup extends DefaultTask {
int gbOfMemory = 4
boolean printToStdOut = true
void numberOfShards(int shards){
void numberOfShards(int shards) {
this.shardCount = shards
}
void coresPerFork(int cores){
void distribute(Distribution dist){
this.distribution = dist
}
void coresPerFork(int cores) {
this.coresToUse = cores
}
void memoryInGbPerFork(int gb){
void memoryInGbPerFork(int gb) {
this.gbOfMemory = gb
}
//when this is false, only containers will "failed" exit codes will be printed to stdout
void streamOutput(boolean print){
void streamOutput(boolean print) {
this.printToStdOut = print
}

54
buildSrc/src/test/java/net/corda/testing/BucketingAllocatorTest.java Normal file
View File

@ -0,0 +1,54 @@
package net.corda.testing;
import org.hamcrest.collection.IsIterableContainingInAnyOrder;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.hamcrest.CoreMatchers.is;
public class BucketingAllocatorTest {
@Test
public void shouldAlwaysBucketTestsEvenIfNotInTimedFile() {
BucketingAllocator bucketingAllocator = new BucketingAllocator(1, Collections::emptyList);
Object task = new Object();
bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
bucketingAllocator.generateTestPlan();
List<String> testsForForkAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
Assert.assertThat(testsForForkAndTestTask, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
}
@Test
public void shouldAllocateTestsAcrossForksEvenIfNoMatchingTestsFound() {
BucketingAllocator bucketingAllocator = new BucketingAllocator(2, Collections::emptyList);
Object task = new Object();
bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
bucketingAllocator.generateTestPlan();
List<String> testsForForkOneAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
List<String> testsForForkTwoAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(1, task);
Assert.assertThat(testsForForkOneAndTestTask.size(), is(1));
Assert.assertThat(testsForForkTwoAndTestTask.size(), is(1));
List<String> allTests = Stream.of(testsForForkOneAndTestTask, testsForForkTwoAndTestTask).flatMap(Collection::stream).collect(Collectors.toList());
Assert.assertThat(allTests, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
}
}

3
core-deterministic/testing/data/build.gradle
View File

@ -22,6 +22,9 @@ dependencies {
jar.enabled = false
test {
ext {
ignoreForDistribution = true
}
filter {
// Running this class is the whole point, so include it explicitly.
includeTestsMatching "net.corda.deterministic.data.GenerateData"

2
testing/Dockerfile
View File

@ -1,4 +1,4 @@
FROM stefanotestingcr.azurecr.io/buildbase:latest
COPY . /tmp/source
CMD ls /tmp/gradle && cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info
CMD cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info

2
testing/DockerfileBase
View File

@ -2,7 +2,7 @@ FROM ubuntu:18.04
ENV GRADLE_USER_HOME=/tmp/gradle
RUN mkdir /tmp/gradle && mkdir -p /home/root/.m2/repository
RUN apt-get update && apt-get install -y curl && \
RUN apt-get update && apt-get install -y curl libatomic1 && \
curl -O https://d3pxv6yz143wms.cloudfront.net/8.222.10.1/java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
apt-get install -y java-common && dpkg -i java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
apt-get clean && \

3224
testing/test-times.csv Normal file
View File

File diff suppressed because it is too large Load Diff