From d5462a2afecdfdddbdb1cbf541131abcd8b4a95a Mon Sep 17 00:00:00 2001 From: Razvan Codreanu <52859362+Schife@users.noreply.github.com> Date: Tue, 29 Oct 2019 16:23:22 +0000 Subject: [PATCH] Re enabling persistent volume claims (#5628) * TM-68 reenabling persistent volume claims using azure files * TM-68 jenkins stackstracee * TM-68 removing duplicate volume * TM-68 pushing storage class yaml file * TM-68 writing all results to the new persistent volume * TM-68 fix wrong directory * TM-68 fix wrong directory * reapply lost merge commit * investigate missing POD from test results * more investigations around pods not executing their tests * make Pod command line more strict with regards to sub command failure * make logs an artifact within jenkins * tidy up command line --- .ci/dev/azureStorageClass.yml | 8 ++ Jenkinsfile | 1 + .../corda/testing/DistributedTesting.groovy | 34 +++---- .../groovy/net/corda/testing/KubesTest.java | 90 +++++++++++-------- 4 files changed, 73 insertions(+), 60 deletions(-) create mode 100644 .ci/dev/azureStorageClass.yml diff --git a/.ci/dev/azureStorageClass.yml b/.ci/dev/azureStorageClass.yml new file mode 100644 index 0000000000..7e978151b5 --- /dev/null +++ b/.ci/dev/azureStorageClass.yml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: testing-storage +provisioner: kubernetes.io/azure-file +parameters: + storageAccount: testrestart + location: westeurope \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index ab2862b88d..28d55398f8 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -57,6 +57,7 @@ pipeline { post { always { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false junit '**/build/test-results-xml/**/*.xml' } cleanup { diff --git a/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy b/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy index c8eab20f08..723e9bac08 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy +++ b/buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy @@ -176,55 +176,47 @@ class DistributedTesting implements Plugin { private Test modifyTestTaskForParallelExecution(Project subProject, Test task, BucketingAllocatorTask globalAllocator) { subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${globalAllocator.getPath()}") - def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name) + def reportsDir = new File(new File(KubesTest.TEST_RUN_DIR, "test-reports"), subProject.name + "-" + task.name) + reportsDir.mkdirs() + File executedTestsFile = new File(KubesTest.TEST_RUN_DIR + "/executedTests.txt") task.configure { dependsOn globalAllocator binResultsDir new File(reportsDir, "binary") reports.junitXml.destination new File(reportsDir, "xml") - maxHeapSize = "6g" + maxHeapSize = "10g" + doFirst { + executedTestsFile.createNewFile() filter { - List executedTests = [] - File executedTestsFile = new File(KubesTest.TEST_RUN_DIR + "/executedTests.txt") - try { - executedTests = executedTestsFile.readLines() - } catch (FileNotFoundException e) { - executedTestsFile.createNewFile() - } - - task.afterTest { desc, result -> - executedTestsFile.withWriterAppend { writer -> - writer.writeLine(desc.getClassName() + "." + desc.getName()) - } - } - + List executedTests = executedTestsFile.readLines() def fork = getPropertyAsInt(subProject, "dockerFork", 0) subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (idx: ${fork})") List includes = globalAllocator.getTestIncludesForForkAndTestTask( fork, task) subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}" - if (includes.size() == 0) { subProject.logger.info "Disabling test execution for testing task ${task.getPath()}" excludeTestsMatching "*" } - includes.removeAll(executedTests) - executedTests.forEach { exclude -> subProject.logger.info "excluding: $exclude for testing task ${task.getPath()}" excludeTestsMatching exclude } - includes.forEach { include -> subProject.logger.info "including: $include for testing task ${task.getPath()}" includeTestsMatching include } - failOnNoMatchingTests false } } + + afterTest { desc, result -> + executedTestsFile.withWriterAppend { writer -> + writer.writeLine(desc.getClassName() + "." + desc.getName()) + } + } } return task diff --git a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java index db7d139333..b15aa0791f 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java +++ b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java @@ -18,6 +18,7 @@ import io.fabric8.kubernetes.client.dsl.PodResource; import io.fabric8.kubernetes.client.utils.Serialization; import net.corda.testing.retry.Retry; import okhttp3.Response; +import org.apache.commons.compress.utils.IOUtils; import org.gradle.api.DefaultTask; import org.gradle.api.tasks.TaskAction; import org.jetbrains.annotations.NotNull; @@ -26,6 +27,8 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; @@ -150,7 +153,8 @@ public class KubesTest extends DefaultTask { int numberOfRetries ) { return CompletableFuture.supplyAsync(() -> { - return buildRunPodWithRetriesOrThrow(namespace, numberOfPods, podIdx, podName, printOutput, numberOfRetries); + PersistentVolumeClaim pvc = createPvc(podName); + return buildRunPodWithRetriesOrThrow(namespace, numberOfPods, podIdx, podName, printOutput, numberOfRetries, pvc); }, executorService); } @@ -158,20 +162,26 @@ public class KubesTest extends DefaultTask { Runtime.getRuntime().addShutdownHook(new Thread(hook)); } - private PersistentVolumeClaim createPvc(KubernetesClient client, String name) { - PersistentVolumeClaim pvc = client.persistentVolumeClaims() - .inNamespace(NAMESPACE) - .createNew() - .editOrNewMetadata().withName(name).endMetadata() - .editOrNewSpec() - .withAccessModes("ReadWriteOnce") - .editOrNewResources().addToRequests("storage", new Quantity("100Mi")).endResources() - .endSpec() - .done(); + private PersistentVolumeClaim createPvc(String name) { + PersistentVolumeClaim pvc; + try (KubernetesClient client = getKubernetesClient()) { + pvc = client.persistentVolumeClaims() + .inNamespace(NAMESPACE) + .createNew() + .editOrNewMetadata().withName(name).endMetadata() + .editOrNewSpec() + .withAccessModes("ReadWriteOnce") + .editOrNewResources().addToRequests("storage", new Quantity("100Mi")).endResources() + .withStorageClassName("testing-storage") + .endSpec() + .done(); + } addShutdownHook(() -> { - System.out.println("Deleting PVC: " + pvc.getMetadata().getName()); - client.persistentVolumeClaims().delete(pvc); + try (KubernetesClient client = getKubernetesClient()) { + System.out.println("Deleting PVC: " + pvc.getMetadata().getName()); + client.persistentVolumeClaims().delete(pvc); + } }); return pvc; } @@ -182,8 +192,8 @@ public class KubesTest extends DefaultTask { int podIdx, String podName, boolean printOutput, - int numberOfRetries - ) { + int numberOfRetries, + PersistentVolumeClaim pvc) { addShutdownHook(() -> { System.out.println("deleting pod: " + podName); try (KubernetesClient client = getKubernetesClient()) { @@ -207,7 +217,7 @@ public class KubesTest extends DefaultTask { } } getProject().getLogger().lifecycle("creating pod: " + podName); - createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName)); + createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName, pvc)); getProject().getLogger().lifecycle("scheduled pod: " + podName); } @@ -221,12 +231,22 @@ public class KubesTest extends DefaultTask { CompletableFuture waiter = new CompletableFuture<>(); File podOutput = executeBuild(namespace, numberOfPods, podIdx, podName, printOutput, stdOutOs, stdOutIs, errChannelStream, waiter); + int resCode = waiter.join(); - getProject().getLogger().lifecycle("build has ended on on pod " + podName + " (" + podIdx + "/" + numberOfPods + "), gathering results"); + getProject().getLogger().lifecycle("build has ended on on pod " + podName + " (" + podIdx + "/" + numberOfPods + ") with result " + resCode + " , gathering results"); Collection binaryResults = downloadTestXmlFromPod(namespace, createdPod); getLogger().lifecycle("removing pod " + podName + " (" + podIdx + "/" + numberOfPods + ") after completed build"); + File podLogsDirectory = new File(getProject().getBuildDir(), "pod-logs"); + if (!podLogsDirectory.exists()) { + podLogsDirectory.mkdirs(); + } + File logFileToArchive = new File(podLogsDirectory, podName + ".log"); + try (FileInputStream logIn = new FileInputStream(podOutput); FileOutputStream logOut = new FileOutputStream(logFileToArchive)) { + IOUtils.copy(logIn, logOut); + } try (KubernetesClient client = getKubernetesClient()) { client.pods().delete(createdPod); + client.persistentVolumeClaims().delete(pvc); } return new KubePodResult(resCode, podOutput, binaryResults); }); @@ -246,7 +266,7 @@ public class KubesTest extends DefaultTask { ByteArrayOutputStream errChannelStream, CompletableFuture waiter) throws IOException { KubernetesClient client = getKubernetesClient(); - ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter); + ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter, client); stdOutIs.connect(stdOutOs); String[] buildCommand = getBuildCommand(numberOfPods, podIdx); @@ -260,7 +280,7 @@ public class KubesTest extends DefaultTask { return startLogPumping(stdOutIs, podIdx, printOutput); } - private Pod buildPodRequest(String podName) { + private Pod buildPodRequest(String podName, PersistentVolumeClaim pvc) { return new PodBuilder() .withNewMetadata().withName(podName).endMetadata() @@ -273,23 +293,12 @@ public class KubesTest extends DefaultTask { .withPath("/tmp/gradle") .endHostPath() .endVolume() - .addNewVolume() .withName("testruns") - .withNewHostPath() - .withType("DirectoryOrCreate") - .withPath("/tmp/testruns") - .endHostPath() + .withNewPersistentVolumeClaim() + .withClaimName(pvc.getMetadata().getName()) + .endPersistentVolumeClaim() .endVolume() - - -// .addNewVolume() -// .withName("testruns") -// .withNewPersistentVolumeClaim() -// .withClaimName(pvc.getMetadata().getName()) -// .endPersistentVolumeClaim() -// .endVolume() - .addNewContainer() .withImage(dockerTag) .withCommand("bash") @@ -367,7 +376,7 @@ public class KubesTest extends DefaultTask { } private Collection downloadTestXmlFromPod(String namespace, Pod cp) { - String resultsInContainerPath = "/tmp/source/build/test-reports"; + String resultsInContainerPath = TEST_RUN_DIR + "/test-reports"; String binaryResultsFile = "results.bin"; String podName = cp.getMetadata().getName(); Path tempDir = new File(new File(getProject().getBuildDir(), "test-results-xml"), podName).toPath(); @@ -387,9 +396,10 @@ public class KubesTest extends DefaultTask { } private String[] getBuildCommand(int numberOfPods, int podIdx) { - String shellScript = "let x=1 ; while [ ${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=$? ; sleep 1 ; done ; " + "cd /tmp/source ; " + - "let y=1 ; while [ ${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=$? ; sleep 1 ; done ;" + - "./gradlew -D" + ListTests.DISTRIBUTION_PROPERTY + "=" + distribution.name() + " -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + getLoggingLevel() + " 2>&1 ;" + + String shellScript = "(let x=1 ; while [ ${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=$? ; sleep 1 ; done ) && " + + " cd /tmp/source && " + + "(let y=1 ; while [ ${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=$? ; sleep 1 ; done ) && " + + "(./gradlew -D" + ListTests.DISTRIBUTION_PROPERTY + "=" + distribution.name() + " -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + getLoggingLevel() + " 2>&1) ; " + "let rs=$? ; sleep 10 ; exit ${rs}"; return new String[]{"bash", "-c", shellScript}; } @@ -421,13 +431,13 @@ public class KubesTest extends DefaultTask { } if (fileToInspect.isDirectory()) { - filesToInspect.addAll(Arrays.stream(fileToInspect.listFiles()).collect(Collectors.toList())); + filesToInspect.addAll(Arrays.stream(Optional.ofNullable(fileToInspect.listFiles()).orElse(new File[]{})).collect(Collectors.toList())); } } return folders; } - private ExecListener buildExecListenerForPod(String podName, ByteArrayOutputStream errChannelStream, CompletableFuture waitingFuture) { + private ExecListener buildExecListenerForPod(String podName, ByteArrayOutputStream errChannelStream, CompletableFuture waitingFuture, KubernetesClient client) { return new ExecListener() { final Long start = System.currentTimeMillis(); @@ -457,6 +467,8 @@ public class KubesTest extends DefaultTask { waitingFuture.complete(resultCode); } catch (Exception e) { waitingFuture.completeExceptionally(e); + } finally { + client.close(); } } };