Re enabling persistent volume claims (#5628)

* TM-68 reenabling persistent volume claims using azure files

* TM-68 jenkins stackstracee

* TM-68 removing duplicate volume

* TM-68 pushing storage class yaml file

* TM-68 writing all results to the new persistent volume

* TM-68 fix wrong directory

* TM-68 fix wrong directory

* reapply lost merge commit

* investigate missing POD from test results

* more investigations around pods not executing their tests

* make Pod command line more strict with regards to sub command failure

* make logs an artifact within jenkins

* tidy up command line

.ci/dev/azureStorageClass.yml

@@ -0,0 +1,8 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: testing-storage
+provisioner: kubernetes.io/azure-file
+parameters:
+  storageAccount: testrestart
+  location: westeurope

Jenkinsfile

@@ -57,6 +57,7 @@ pipeline {
 
     post {
         always {
+            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
             junit '**/build/test-results-xml/**/*.xml'
         }
         cleanup {

buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy

@@ -176,55 +176,47 @@ class DistributedTesting implements Plugin<Project> {
 
     private Test modifyTestTaskForParallelExecution(Project subProject, Test task, BucketingAllocatorTask globalAllocator) {
         subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${globalAllocator.getPath()}")
-        def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name)
+        def reportsDir = new File(new File(KubesTest.TEST_RUN_DIR, "test-reports"), subProject.name + "-" + task.name)
+        reportsDir.mkdirs()
+        File executedTestsFile = new File(KubesTest.TEST_RUN_DIR + "/executedTests.txt")
         task.configure {
             dependsOn globalAllocator
             binResultsDir new File(reportsDir, "binary")
             reports.junitXml.destination new File(reportsDir, "xml")
-            maxHeapSize = "6g"
+            maxHeapSize = "10g"
+
             doFirst {
+                executedTestsFile.createNewFile()
                 filter {
-                    List<String> executedTests = []
+                    List<String> executedTests = executedTestsFile.readLines()
-                    File executedTestsFile = new File(KubesTest.TEST_RUN_DIR + "/executedTests.txt")
-                    try {
-                        executedTests = executedTestsFile.readLines()
-                    } catch (FileNotFoundException e) {
-                        executedTestsFile.createNewFile()
-                    }
-
-                    task.afterTest { desc, result ->
-                        executedTestsFile.withWriterAppend { writer ->
-                            writer.writeLine(desc.getClassName() + "." + desc.getName())
-                        }
-                    }
-
                     def fork = getPropertyAsInt(subProject, "dockerFork", 0)
                     subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (idx: ${fork})")
                     List<String> includes = globalAllocator.getTestIncludesForForkAndTestTask(
                             fork,
                             task)
                     subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}"
-
                     if (includes.size() == 0) {
                         subProject.logger.info "Disabling test execution for testing task ${task.getPath()}"
                         excludeTestsMatching "*"
                     }
-
                     includes.removeAll(executedTests)
-
                     executedTests.forEach { exclude ->
                         subProject.logger.info "excluding: $exclude for testing task ${task.getPath()}"
                         excludeTestsMatching exclude
                     }
-
                     includes.forEach { include ->
                         subProject.logger.info "including: $include for testing task ${task.getPath()}"
                         includeTestsMatching include
                     }
-
                     failOnNoMatchingTests false
                 }
             }
+
+            afterTest { desc, result ->
+                executedTestsFile.withWriterAppend { writer ->
+                    writer.writeLine(desc.getClassName() + "." + desc.getName())
+                }
+            }
         }
 
         return task

buildSrc/src/main/groovy/net/corda/testing/KubesTest.java

@@ -18,6 +18,7 @@ import io.fabric8.kubernetes.client.dsl.PodResource;
 import io.fabric8.kubernetes.client.utils.Serialization;
 import net.corda.testing.retry.Retry;
 import okhttp3.Response;
+import org.apache.commons.compress.utils.IOUtils;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.tasks.TaskAction;
 import org.jetbrains.annotations.NotNull;
@@ -26,6 +27,8 @@ import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStream;
@@ -150,7 +153,8 @@ public class KubesTest extends DefaultTask {
             int numberOfRetries
     ) {
         return CompletableFuture.supplyAsync(() -> {
-            return buildRunPodWithRetriesOrThrow(namespace, numberOfPods, podIdx, podName, printOutput, numberOfRetries);
+            PersistentVolumeClaim pvc = createPvc(podName);
+            return buildRunPodWithRetriesOrThrow(namespace, numberOfPods, podIdx, podName, printOutput, numberOfRetries, pvc);
         }, executorService);
     }
 
@@ -158,20 +162,26 @@ public class KubesTest extends DefaultTask {
         Runtime.getRuntime().addShutdownHook(new Thread(hook));
     }
 
-    private PersistentVolumeClaim createPvc(KubernetesClient client, String name) {
+    private PersistentVolumeClaim createPvc(String name) {
-        PersistentVolumeClaim pvc = client.persistentVolumeClaims()
+        PersistentVolumeClaim pvc;
-                .inNamespace(NAMESPACE)
+        try (KubernetesClient client = getKubernetesClient()) {
-                .createNew()
+            pvc = client.persistentVolumeClaims()
-                .editOrNewMetadata().withName(name).endMetadata()
+                    .inNamespace(NAMESPACE)
-                .editOrNewSpec()
+                    .createNew()
-                .withAccessModes("ReadWriteOnce")
+                    .editOrNewMetadata().withName(name).endMetadata()
-                .editOrNewResources().addToRequests("storage", new Quantity("100Mi")).endResources()
+                    .editOrNewSpec()
-                .endSpec()
+                    .withAccessModes("ReadWriteOnce")
-                .done();
+                    .editOrNewResources().addToRequests("storage", new Quantity("100Mi")).endResources()
+                    .withStorageClassName("testing-storage")
+                    .endSpec()
+                    .done();
+        }
 
         addShutdownHook(() -> {
-            System.out.println("Deleting PVC: " + pvc.getMetadata().getName());
+            try (KubernetesClient client = getKubernetesClient()) {
-            client.persistentVolumeClaims().delete(pvc);
+                System.out.println("Deleting PVC: " + pvc.getMetadata().getName());
+                client.persistentVolumeClaims().delete(pvc);
+            }
         });
         return pvc;
     }
@@ -182,8 +192,8 @@ public class KubesTest extends DefaultTask {
             int podIdx,
             String podName,
             boolean printOutput,
-            int numberOfRetries
+            int numberOfRetries,
-    ) {
+            PersistentVolumeClaim pvc) {
         addShutdownHook(() -> {
             System.out.println("deleting pod: " + podName);
             try (KubernetesClient client = getKubernetesClient()) {
@@ -207,7 +217,7 @@ public class KubesTest extends DefaultTask {
                         }
                     }
                     getProject().getLogger().lifecycle("creating pod: " + podName);
-                    createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName));
+                    createdPod = client.pods().inNamespace(namespace).create(buildPodRequest(podName, pvc));
                     getProject().getLogger().lifecycle("scheduled pod: " + podName);
                 }
 
@@ -221,12 +231,22 @@ public class KubesTest extends DefaultTask {
                 CompletableFuture<Integer> waiter = new CompletableFuture<>();
                 File podOutput = executeBuild(namespace, numberOfPods, podIdx, podName, printOutput, stdOutOs, stdOutIs, errChannelStream, waiter);
 
+
                 int resCode = waiter.join();
-                getProject().getLogger().lifecycle("build has ended on on pod " + podName + " (" + podIdx + "/" + numberOfPods + "), gathering results");
+                getProject().getLogger().lifecycle("build has ended on on pod " + podName + " (" + podIdx + "/" + numberOfPods + ") with result " + resCode + " , gathering results");
                 Collection<File> binaryResults = downloadTestXmlFromPod(namespace, createdPod);
                 getLogger().lifecycle("removing pod " + podName + " (" + podIdx + "/" + numberOfPods + ") after completed build");
+                File podLogsDirectory = new File(getProject().getBuildDir(), "pod-logs");
+                if (!podLogsDirectory.exists()) {
+                    podLogsDirectory.mkdirs();
+                }
+                File logFileToArchive = new File(podLogsDirectory, podName + ".log");
+                try (FileInputStream logIn = new FileInputStream(podOutput); FileOutputStream logOut = new FileOutputStream(logFileToArchive)) {
+                    IOUtils.copy(logIn, logOut);
+                }
                 try (KubernetesClient client = getKubernetesClient()) {
                     client.pods().delete(createdPod);
+                    client.persistentVolumeClaims().delete(pvc);
                 }
                 return new KubePodResult(resCode, podOutput, binaryResults);
             });
@@ -246,7 +266,7 @@ public class KubesTest extends DefaultTask {
                               ByteArrayOutputStream errChannelStream,
                               CompletableFuture<Integer> waiter) throws IOException {
         KubernetesClient client = getKubernetesClient();
-        ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter);
+        ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter, client);
         stdOutIs.connect(stdOutOs);
 
         String[] buildCommand = getBuildCommand(numberOfPods, podIdx);
@@ -260,7 +280,7 @@ public class KubesTest extends DefaultTask {
         return startLogPumping(stdOutIs, podIdx, printOutput);
     }
 
-    private Pod buildPodRequest(String podName) {
+    private Pod buildPodRequest(String podName, PersistentVolumeClaim pvc) {
         return new PodBuilder()
                 .withNewMetadata().withName(podName).endMetadata()
 
@@ -273,23 +293,12 @@ public class KubesTest extends DefaultTask {
                 .withPath("/tmp/gradle")
                 .endHostPath()
                 .endVolume()
-
                 .addNewVolume()
                 .withName("testruns")
-                .withNewHostPath()
+                .withNewPersistentVolumeClaim()
-                .withType("DirectoryOrCreate")
+                .withClaimName(pvc.getMetadata().getName())
-                .withPath("/tmp/testruns")
+                .endPersistentVolumeClaim()
-                .endHostPath()
                 .endVolume()
-
-
-//                .addNewVolume()
-//                .withName("testruns")
-//                .withNewPersistentVolumeClaim()
-//                .withClaimName(pvc.getMetadata().getName())
-//                .endPersistentVolumeClaim()
-//                .endVolume()
-
                 .addNewContainer()
                 .withImage(dockerTag)
                 .withCommand("bash")
@@ -367,7 +376,7 @@ public class KubesTest extends DefaultTask {
     }
 
     private Collection<File> downloadTestXmlFromPod(String namespace, Pod cp) {
-        String resultsInContainerPath = "/tmp/source/build/test-reports";
+        String resultsInContainerPath = TEST_RUN_DIR + "/test-reports";
         String binaryResultsFile = "results.bin";
         String podName = cp.getMetadata().getName();
         Path tempDir = new File(new File(getProject().getBuildDir(), "test-results-xml"), podName).toPath();
@@ -387,9 +396,10 @@ public class KubesTest extends DefaultTask {
     }
 
     private String[] getBuildCommand(int numberOfPods, int podIdx) {
-        String shellScript = "let x=1 ; while [ ${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=$? ; sleep 1 ; done ; " + "cd /tmp/source ; " +
+        String shellScript = "(let x=1 ; while [ ${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=$? ; sleep 1 ; done ) && "
-                "let y=1 ; while [ ${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=$? ; sleep 1 ; done ;" +
+                + " cd /tmp/source && " +
-                "./gradlew -D" + ListTests.DISTRIBUTION_PROPERTY + "=" + distribution.name() + " -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + getLoggingLevel() + " 2>&1 ;" +
+                "(let y=1 ; while [ ${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=$? ; sleep 1 ; done ) && " +
+                "(./gradlew -D" + ListTests.DISTRIBUTION_PROPERTY + "=" + distribution.name() + " -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " " + fullTaskToExecutePath + " " + getLoggingLevel() + " 2>&1) ; " +
                 "let rs=$? ; sleep 10 ; exit ${rs}";
         return new String[]{"bash", "-c", shellScript};
     }
@@ -421,13 +431,13 @@ public class KubesTest extends DefaultTask {
             }
 
             if (fileToInspect.isDirectory()) {
-                filesToInspect.addAll(Arrays.stream(fileToInspect.listFiles()).collect(Collectors.toList()));
+                filesToInspect.addAll(Arrays.stream(Optional.ofNullable(fileToInspect.listFiles()).orElse(new File[]{})).collect(Collectors.toList()));
             }
         }
         return folders;
     }
 
-    private ExecListener buildExecListenerForPod(String podName, ByteArrayOutputStream errChannelStream, CompletableFuture<Integer> waitingFuture) {
+    private ExecListener buildExecListenerForPod(String podName, ByteArrayOutputStream errChannelStream, CompletableFuture<Integer> waitingFuture, KubernetesClient client) {
 
         return new ExecListener() {
             final Long start = System.currentTimeMillis();
@@ -457,6 +467,8 @@ public class KubesTest extends DefaultTask {
                     waitingFuture.complete(resultCode);
                 } catch (Exception e) {
                     waitingFuture.completeExceptionally(e);
+                } finally {
+                    client.close();
                 }
             }
         };