ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2024-12-18 20:47:57 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Converting Groovy to Java in the Distributed testing plugin (#5642)

Browse Source 
* TM-74 Converting ParallelTestGroup

* TM-74 private fields, getters, distributedTesting refactoring

* TM-74 More conversions

* TM-74 reinstating ListTests groovy class

* TM-74 extracting enum and clas from ListTests

* TM-74 switching to java ListTests

* TM-74 null check

* TM-74 new access modifiers

* TM-74 minor changes

* TM-74 scrapping the conversions of ListTests to java

* TM-74 reverting build.gradle

* TM-74 formatting main gradle

* TM-74 removing the commented out code. It has been linked in the jira

* TM-74 adding list tests back for investigation

* TM-74 collecting after every operation

* TM-74 collecting after every operation

* TM-74 more debugging

* TM-74 more debugging with lifecycle now

* TM-74 debugging on the existing groovy file

* TM-74 with stream output

* TM-74 switching back to java

* TM-74 reverting java code, fixing cast in distributed testing file?

* TM-74 nailing down the line causing the problem

* TM-74 casting

* TM-74 changing types

* TM-74 stacktrace at pod lvl

* TM-74 stacktrace at pod lvl

* TM-74 fix issue with immutable list

* TM-74 reverting changes now that fix is in place

* TM-74 switching to generic task

* TM-74 turning off debugging

* TM-74 allocating by method
This commit is contained in:
Razvan Codreanu 2019-11-05 16:59:19 +00:00 committed by Stefano Franz
parent ed50016ed0
commit 7106b0f1b9
11 changed files with 259 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.gradle
View File

@ -1,5 +1,5 @@
import net.corda.testing.DistributedTesting
import net.corda.testing.DistributeTestsBy
import net.corda.testing.DistributedTesting
import net.corda.testing.ImageBuilding
import net.corda.testing.ParallelTestGroup
import net.corda.testing.PodLogLevel
@ -602,7 +602,7 @@ task allParallelIntegrationTest(type: ParallelTestGroup) {
streamOutput false
coresPerFork 5
memoryInGbPerFork 10
distribute DistributeTestsBy.CLASS
distribute DistributeTestsBy.METHOD
}
task allParallelUnitTest(type: ParallelTestGroup) {
podLogLevel PodLogLevel.INFO
@ -639,5 +639,3 @@ task allParallelSmokeTest(type: ParallelTestGroup) {
}
apply plugin: ImageBuilding
apply plugin: DistributedTesting

5
buildSrc/src/main/groovy/net/corda/testing/DistributeTestsBy.java Normal file
View File

@ -0,0 +1,5 @@
package net.corda.testing;
public enum DistributeTestsBy {
CLASS, METHOD
}

38
buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy
View File

@ -50,7 +50,7 @@ class DistributedTesting implements Plugin<Project> {
project.logger.info("Evaluating ${task.getPath()}")
if (task in requestedTasks && !task.hasProperty("ignoreForDistribution")) {
project.logger.info "Modifying ${task.getPath()}"
ListTests testListerTask = createTestListingTasks(task, subProject)
Task testListerTask = createTestListingTasks(task, subProject)
globalAllocator.addSource(testListerTask, task)
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, globalAllocator)
} else {
@ -79,7 +79,7 @@ class DistributedTesting implements Plugin<Project> {
userGroups.forEach { testGrouping ->
//for each "group" (ie: test, integrationTest) within the grouping find all the Test tasks which have the same name.
List<Test> testTasksToRunInGroup = ((ParallelTestGroup) testGrouping).groups.collect {
List<Test> testTasksToRunInGroup = ((ParallelTestGroup) testGrouping).getGroups().collect {
allTestTasksGroupedByType.get(it)
}.flatten()
@ -95,7 +95,7 @@ class DistributedTesting implements Plugin<Project> {
imageBuildTask.dependsOn preAllocateTask
}
def userDefinedParallelTask = project.rootProject.tasks.create("userDefined" + testGrouping.name.capitalize(), KubesTest) {
def userDefinedParallelTask = project.rootProject.tasks.create("userDefined" + testGrouping.getName().capitalize(), KubesTest) {
group = GRADLE_GROUP
if (!tagToUseForRunningTests) {
@ -106,24 +106,24 @@ class DistributedTesting implements Plugin<Project> {
dependsOn deAllocateTask
}
numberOfPods = testGrouping.getShardCount()
printOutput = testGrouping.printToStdOut
printOutput = testGrouping.getPrintToStdOut()
fullTaskToExecutePath = superListOfTasks
taskToExecuteName = testGrouping.groups.join("And")
memoryGbPerFork = testGrouping.gbOfMemory
numberOfCoresPerFork = testGrouping.coresToUse
distribution = testGrouping.distribution
podLogLevel = testGrouping.logLevel
taskToExecuteName = testGrouping.getGroups().join("And")
memoryGbPerFork = testGrouping.getGbOfMemory()
numberOfCoresPerFork = testGrouping.getCoresToUse()
distribution = testGrouping.getDistribution()
podLogLevel = testGrouping.getLogLevel()
doFirst {
dockerTag = tagToUseForRunningTests ? (ImageBuilding.registryName + ":" + tagToUseForRunningTests) : (imagePushTask.imageName.get() + ":" + imagePushTask.tag.get())
}
}
def reportOnAllTask = project.rootProject.tasks.create("userDefinedReports${testGrouping.name.capitalize()}", KubesReporting) {
def reportOnAllTask = project.rootProject.tasks.create("userDefinedReports${testGrouping.getName().capitalize()}", KubesReporting) {
group = GRADLE_GROUP
dependsOn userDefinedParallelTask
destinationDir new File(project.rootProject.getBuildDir(), "userDefinedReports${testGrouping.name.capitalize()}")
destinationDir new File(project.rootProject.getBuildDir(), "userDefinedReports${testGrouping.getName().capitalize()}")
doFirst {
destinationDir.deleteDir()
shouldPrintOutput = !testGrouping.printToStdOut
shouldPrintOutput = !testGrouping.getPrintToStdOut()
podResults = userDefinedParallelTask.containerResults
reportOn(userDefinedParallelTask.testOutput)
}
@ -145,14 +145,14 @@ class DistributedTesting implements Plugin<Project> {
private List<Task> generatePreAllocateAndDeAllocateTasksForGrouping(Project project, ParallelTestGroup testGrouping) {
PodAllocator allocator = new PodAllocator(project.getLogger())
Task preAllocateTask = project.rootProject.tasks.create("preAllocateFor" + testGrouping.name.capitalize()) {
Task preAllocateTask = project.rootProject.tasks.create("preAllocateFor" + testGrouping.getName().capitalize()) {
group = GRADLE_GROUP
doFirst {
String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_BUILDING_PROPERTY)
if (dockerTag == null) {
throw new GradleException("pre allocation cannot be used without a stable docker tag - please provide one using -D" + ImageBuilding.PROVIDE_TAG_FOR_BUILDING_PROPERTY)
}
int seed = (dockerTag.hashCode() + testGrouping.name.hashCode())
int seed = (dockerTag.hashCode() + testGrouping.getName().hashCode())
String podPrefix = new BigInteger(64, new Random(seed)).toString(36)
//here we will pre-request the correct number of pods for this testGroup
int numberOfPodsToRequest = testGrouping.getShardCount()
@ -162,14 +162,14 @@ class DistributedTesting implements Plugin<Project> {
}
}
Task deAllocateTask = project.rootProject.tasks.create("deAllocateFor" + testGrouping.name.capitalize()) {
Task deAllocateTask = project.rootProject.tasks.create("deAllocateFor" + testGrouping.getName().capitalize()) {
group = GRADLE_GROUP
doFirst {
String dockerTag = System.getProperty(ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY)
if (dockerTag == null) {
throw new GradleException("pre allocation cannot be used without a stable docker tag - please provide one using -D" + ImageBuilding.PROVIDE_TAG_FOR_RUNNING_PROPERTY)
}
int seed = (dockerTag.hashCode() + testGrouping.name.hashCode())
int seed = (dockerTag.hashCode() + testGrouping.getName().hashCode())
String podPrefix = new BigInteger(64, new Random(seed)).toString(36);
allocator.tearDownPods(podPrefix)
}
@ -249,12 +249,12 @@ class DistributedTesting implements Plugin<Project> {
project.plugins.apply(ImageBuilding)
}
private ListTests createTestListingTasks(Test task, Project subProject) {
private Task createTestListingTasks(Test task, Project subProject) {
def taskName = task.getName()
def capitalizedTaskName = task.getName().capitalize()
//determine all the tests which are present in this test task.
//this list will then be shared between the various worker forks
def createdListTask = subProject.tasks.create("listTestsFor" + capitalizedTaskName, ListTests) {
ListTests createdListTask = subProject.tasks.create("listTestsFor" + capitalizedTaskName, ListTests) {
group = GRADLE_GROUP
//the convention is that a testing task is backed by a sourceSet with the same name
dependsOn subProject.getTasks().getByName("${taskName}Classes")
@ -281,7 +281,7 @@ class DistributedTesting implements Plugin<Project> {
subProject.logger.info("created task: " + createdListTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdListTask.dependsOn)
subProject.logger.info("created task: " + createdPrintTask.getPath() + " in project: " + subProject + " it dependsOn: " + createdPrintTask.dependsOn)
return createdListTask as ListTests
return createdListTask
}
}

37
buildSrc/src/main/groovy/net/corda/testing/ListShufflerAndAllocator.java Normal file
View File

@ -0,0 +1,37 @@
package net.corda.testing;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Random;
import java.util.stream.Collectors;
class ListShufflerAndAllocator {
private final List<String> tests;
public ListShufflerAndAllocator(List<String> tests) {
this.tests = new ArrayList<>(tests);
}
public List<String> getTestsForFork(int fork, int forks, Integer seed) {
final Random shuffler = new Random(seed);
final List<String> copy = new ArrayList<>(tests);
while (copy.size() < forks) {
//pad the list
copy.add(null);
}
Collections.shuffle(copy, shuffler);
final int numberOfTestsPerFork = Math.max((copy.size() / forks), 1);
final int consumedTests = numberOfTestsPerFork * forks;
final int ourStartIdx = numberOfTestsPerFork * fork;
final int ourEndIdx = ourStartIdx + numberOfTestsPerFork;
final int ourSupplementaryIdx = consumedTests + fork;
final ArrayList<String> toReturn = new ArrayList<>(copy.subList(ourStartIdx, ourEndIdx));
if (ourSupplementaryIdx < copy.size()) {
toReturn.add(copy.get(ourSupplementaryIdx));
}
return toReturn.stream().filter(Objects::nonNull).collect(Collectors.toList());
}
}

110
buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy
View File

@ -1,110 +0,0 @@
package net.corda.testing
import io.github.classgraph.ClassGraph
import io.github.classgraph.ClassInfo
import org.gradle.api.DefaultTask
import org.gradle.api.file.FileCollection
import org.gradle.api.tasks.TaskAction
import java.util.stream.Collectors
class ListShufflerAndAllocator {
private final List<String> tests
public ListShufflerAndAllocator(List<String> tests) {
this.tests = new ArrayList<>(tests)
}
List<String> getTestsForFork(int fork, int forks, Integer seed) {
Random shuffler = new Random(seed);
List<String> copy = new ArrayList<>(tests);
while (copy.size() < forks) {
//pad the list
copy.add(null);
}
Collections.shuffle(copy, shuffler);
int numberOfTestsPerFork = Math.max((copy.size() / forks).intValue(), 1);
int consumedTests = numberOfTestsPerFork * forks;
int ourStartIdx = numberOfTestsPerFork * fork;
int ourEndIdx = ourStartIdx + numberOfTestsPerFork;
int ourSupplementaryIdx = consumedTests + fork;
ArrayList<String> toReturn = new ArrayList<>(copy.subList(ourStartIdx, ourEndIdx));
if (ourSupplementaryIdx < copy.size()) {
toReturn.add(copy.get(ourSupplementaryIdx));
}
return toReturn.stream().filter { it -> it != null }.collect(Collectors.toList());
}
}
interface TestLister {
List<String> getAllTestsDiscovered()
}
class ListTests extends DefaultTask implements TestLister {
public static final String DISTRIBUTION_PROPERTY = "distributeBy"
FileCollection scanClassPath
List<String> allTests
DistributeTestsBy distribution = System.getProperty(DISTRIBUTION_PROPERTY) ? DistributeTestsBy.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : DistributeTestsBy.METHOD
def getTestsForFork(int fork, int forks, Integer seed) {
def gitSha = new BigInteger(project.hasProperty("corda_revision") ? project.property("corda_revision").toString() : "0", 36)
if (fork >= forks) {
throw new IllegalArgumentException("requested shard ${fork + 1} for total shards ${forks}")
}
def seedToUse = seed ? (seed + ((String) this.getPath()).hashCode() + gitSha.intValue()) : 0
return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse)
}
@Override
public List<String> getAllTestsDiscovered() {
return new ArrayList<>(allTests)
}
@TaskAction
def discoverTests() {
switch (distribution) {
case DistributeTestsBy.METHOD:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c ->
c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name }
}.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
case DistributeTestsBy.CLASS:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c -> c.name }.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
}
}
}
public enum DistributeTestsBy {
CLASS, METHOD
}

99
buildSrc/src/main/groovy/net/corda/testing/ListTests.java Normal file
View File

@ -0,0 +1,99 @@
package net.corda.testing;
import io.github.classgraph.ClassGraph;
import io.github.classgraph.ClassInfo;
import io.github.classgraph.ClassInfoList;
import org.gradle.api.DefaultTask;
import org.gradle.api.file.FileCollection;
import org.gradle.api.tasks.TaskAction;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
interface TestLister {
List<String> getAllTestsDiscovered();
}
public class ListTests extends DefaultTask implements TestLister {
public static final String DISTRIBUTION_PROPERTY = "distributeBy";
public FileCollection scanClassPath;
private List<String> allTests;
private DistributeTestsBy distribution = System.getProperty(DISTRIBUTION_PROPERTY) != null && !System.getProperty(DISTRIBUTION_PROPERTY).isEmpty() ?
DistributeTestsBy.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : DistributeTestsBy.METHOD;
public List<String> getTestsForFork(int fork, int forks, Integer seed) {
BigInteger gitSha = new BigInteger(getProject().hasProperty("corda_revision") ?
getProject().property("corda_revision").toString() : "0", 36);
if (fork >= forks) {
throw new IllegalArgumentException("requested shard ${fork + 1} for total shards ${forks}");
}
int seedToUse = seed != null ? (seed + (this.getPath()).hashCode() + gitSha.intValue()) : 0;
return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse);
}
@Override
public List<String> getAllTestsDiscovered() {
return new ArrayList<>(allTests);
}
@TaskAction
void discoverTests() {
Collection<String> results;
switch (distribution) {
case METHOD:
results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.stream()
.map(classInfo -> {
ClassInfoList returnList = new ClassInfoList();
returnList.add(classInfo);
returnList.addAll(classInfo.getSubclasses());
return returnList;
})
.flatMap(ClassInfoList::stream)
.map(classInfo -> classInfo.getMethodInfo().filter(methodInfo -> methodInfo.hasAnnotation("org.junit.Test"))
.stream().map(methodInfo -> classInfo.getName() + "." + methodInfo.getName()))
.flatMap(Function.identity())
.collect(Collectors.toSet());
this.allTests = results.stream().sorted().collect(Collectors.toList());
break;
case CLASS:
results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.stream()
.map(classInfo -> {
ClassInfoList returnList = new ClassInfoList();
returnList.add(classInfo);
returnList.addAll(classInfo.getSubclasses());
return returnList;
})
.flatMap(ClassInfoList::stream)
.map(ClassInfo::getName)
.collect(Collectors.toSet());
this.allTests = results.stream().sorted().collect(Collectors.toList());
break;
}
getProject().getLogger().lifecycle("THESE ARE ALL THE TESTSSS!!!!!!!!: " + allTests.toString());
}
}

51
buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy
View File

@ -1,51 +0,0 @@
package net.corda.testing
import org.gradle.api.DefaultTask
class ParallelTestGroup extends DefaultTask {
DistributeTestsBy distribution = DistributeTestsBy.METHOD
List<String> groups = new ArrayList<>()
int shardCount = 20
int coresToUse = 4
int gbOfMemory = 4
boolean printToStdOut = true
PodLogLevel logLevel = PodLogLevel.INFO
void numberOfShards(int shards) {
this.shardCount = shards
}
void podLogLevel(PodLogLevel level) {
this.logLevel = level
}
void distribute(DistributeTestsBy dist) {
this.distribution = dist
}
void coresPerFork(int cores) {
this.coresToUse = cores
}
void memoryInGbPerFork(int gb) {
this.gbOfMemory = gb
}
//when this is false, only containers will "failed" exit codes will be printed to stdout
void streamOutput(boolean print) {
this.printToStdOut = print
}
void testGroups(String... group) {
testGroups(group.toList())
}
void testGroups(List<String> group) {
group.forEach {
groups.add(it)
}
}
}

80
buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.java Normal file
View File

@ -0,0 +1,80 @@
package net.corda.testing;
import org.gradle.api.DefaultTask;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class ParallelTestGroup extends DefaultTask {
private DistributeTestsBy distribution = DistributeTestsBy.METHOD;
private List<String> groups = new ArrayList<>();
private int shardCount = 20;
private int coresToUse = 4;
private int gbOfMemory = 4;
private boolean printToStdOut = true;
private PodLogLevel logLevel = PodLogLevel.INFO;
public DistributeTestsBy getDistribution() {
return distribution;
}
public List<String> getGroups() {
return groups;
}
public int getShardCount() {
return shardCount;
}
public int getCoresToUse() {
return coresToUse;
}
public int getGbOfMemory() {
return gbOfMemory;
}
public boolean getPrintToStdOut() {
return printToStdOut;
}
public PodLogLevel getLogLevel() {
return logLevel;
}
public void numberOfShards(int shards) {
this.shardCount = shards;
}
public void podLogLevel(PodLogLevel level) {
this.logLevel = level;
}
public void distribute(DistributeTestsBy dist) {
this.distribution = dist;
}
public void coresPerFork(int cores) {
this.coresToUse = cores;
}
public void memoryInGbPerFork(int gb) {
this.gbOfMemory = gb;
}
//when this is false, only containers will "failed" exit codes will be printed to stdout
public void streamOutput(boolean print) {
this.printToStdOut = print;
}
public void testGroups(String... group) {
testGroups(Arrays.asList(group));
}
private void testGroups(List<String> group) {
groups.addAll(group);
}
}

32
buildSrc/src/main/groovy/net/corda/testing/RunInParallel.groovy
View File

@ -1,32 +0,0 @@
package net.corda.testing
import org.gradle.api.Action
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.tasks.TaskAction
import java.util.concurrent.CompletableFuture
class RunInParallel extends DefaultTask {
private List<Task> tasksToRunInParallel = new ArrayList<>()
public RunInParallel runInParallel(Task... tasks) {
for (Task task : tasks) {
tasksToRunInParallel.add(task)
}
return this;
}
@TaskAction
def void run() {
tasksToRunInParallel.collect { t ->
CompletableFuture.runAsync {
def actions = t.getActions()
for (Action action : actions) {
action.execute(t)
}
}
}.join()
}
}

27
buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.groovy → buildSrc/src/test/groovy/net/corda/testing/ListTestsTest.java
View File

@ -1,25 +1,30 @@
package net.corda.testing
package net.corda.testing;
import org.hamcrest.CoreMatchers
import org.junit.Assert
import org.junit.Test
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Test;
import java.util.stream.Collectors
import java.util.stream.IntStream
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.hamcrest.core.Is.is
import static org.hamcrest.core.IsEqual.equalTo
import static org.hamcrest.core.Is.is;
import static org.hamcrest.core.IsEqual.equalTo;
class ListTestsTest {
public class ListTestsTest {
@Test
void shouldAllocateTests() {
public void shouldAllocateTests() {
for (int numberOfTests = 0; numberOfTests < 100; numberOfTests++) {
for (int numberOfForks = 1; numberOfForks < 100; numberOfForks++) {
List<String> tests = IntStream.range(0, numberOfTests).collect { z -> "Test.method" + z }
List<String> tests = IntStream.range(0, numberOfTests).boxed()
.map(integer -> "Test.method" + integer.toString())
.collect(Collectors.toList());
ListShufflerAndAllocator testLister = new ListShufflerAndAllocator(tests);
List<String> listOfLists = new ArrayList<>();

2
detekt-config.yml
View File

@ -170,7 +170,7 @@ style:
ignoreCompanionObjectPropertyDeclaration: true
ignoreAnnotation: false
ignoreNamedArgument: true
ignoreEnums: false
ignoreEnums: true
MaxLineLength:
active: true
excludes: "**/buildSrc/**"