add exception handling to handle situation where builds are tidying up same pods (#5440)

* add asterix to test name

* fix usages of System.hasProperty

* add exception handling to handle condition when Build2 is deleting pods that are being tidied up by Build1

* attempt to fix namespace issues

* add pull task to pull latest base image

* add logic to download dependencies in parallel before running tests

buildSrc/src/main/groovy/net/corda/testing/ImageBuilding.groovy

@@ -5,10 +5,7 @@ import com.bmuschko.gradle.docker.tasks.container.DockerCreateContainer
 import com.bmuschko.gradle.docker.tasks.container.DockerLogsContainer
 import com.bmuschko.gradle.docker.tasks.container.DockerStartContainer
 import com.bmuschko.gradle.docker.tasks.container.DockerWaitContainer
-import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage
+import com.bmuschko.gradle.docker.tasks.image.*
-import com.bmuschko.gradle.docker.tasks.image.DockerCommitImage
-import com.bmuschko.gradle.docker.tasks.image.DockerPushImage
-import com.bmuschko.gradle.docker.tasks.image.DockerTagImage
 import org.gradle.api.GradleException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
@@ -23,8 +20,21 @@ class ImageBuilding implements Plugin<Project> {
 
     @Override
     void apply(Project project) {
+
+        def registryCredentialsForPush = new DockerRegistryCredentials(project.getObjects())
+        registryCredentialsForPush.username.set("stefanotestingcr")
+        registryCredentialsForPush.password.set(System.getProperty("docker.push.password") ? System.getProperty("docker.push.password") : "")
+
+        DockerPullImage pullTask = project.tasks.create("pullBaseImage", DockerPullImage){
+            repository = "stefanotestingcr.azurecr.io/buildbase"
+            tag = "latest"
+            doFirst {
+                registryCredentials = registryCredentialsForPush
+            }
+        }
+
         DockerBuildImage buildDockerImageForSource = project.tasks.create('buildDockerImageForSource', DockerBuildImage) {
-            dependsOn project.rootProject.getTasksByName("clean", true)
+            dependsOn Arrays.asList(project.rootProject.getTasksByName("clean", true), pullTask)
             inputDir.set(new File("."))
             dockerFile.set(new File(new File("testing"), "Dockerfile"))
         }
@@ -79,9 +89,6 @@ class ImageBuilding implements Plugin<Project> {
             tag = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
             repository = "stefanotestingcr.azurecr.io/testing"
         }
-        def registryCredentialsForPush = new DockerRegistryCredentials(project.getObjects())
-        registryCredentialsForPush.username.set("stefanotestingcr")
-        registryCredentialsForPush.password.set(System.getProperty("docker.push.password") ? System.getProperty("docker.push.password") : "")
 
         if (System.getProperty("docker.tag")) {
             DockerPushImage pushBuildImage = project.tasks.create('pushBuildImage', DockerPushImage) {

buildSrc/src/main/groovy/net/corda/testing/KubesTest.groovy

@@ -66,15 +66,16 @@ class KubesTest extends DefaultTask {
 
         final KubernetesClient client = new DefaultKubernetesClient(config)
 
-        client.pods().inNamespace(namespace).list().getItems().forEach({ podToDelete ->
+        try {
-            if (podToDelete.getMetadata().name.contains(stableRunId)) {
+            client.pods().inNamespace(namespace).list().getItems().forEach({ podToDelete ->
-                project.logger.lifecycle("deleting: " + podToDelete.getMetadata().getName())
+                if (podToDelete.getMetadata().name.contains(stableRunId)) {
-                client.resource(podToDelete).delete()
+                    project.logger.lifecycle("deleting: " + podToDelete.getMetadata().getName())
-            }
+                    client.resource(podToDelete).delete()
-        })
+                }
-
+            })
-        Namespace ns = new NamespaceBuilder().withNewMetadata().withName(namespace).addToLabels("testing-env", "true").endMetadata().build()
+        } catch (Exception ignored) {
-        client.namespaces().createOrReplace(ns)
+            //it's possible that a pod is being deleted by the original build, this can lead to racey conditions
+        }
 
 
         List<CompletableFuture<KubePodResult>> podCreationFutures = IntStream.range(0, numberOfPods).mapToObj({ i ->
@@ -257,8 +258,12 @@ class KubesTest extends DefaultTask {
     }
 
     String[] getBuildCommand(int numberOfPods, int podIdx) {
-        return ["bash", "-c", "cd /tmp/source && ./gradlew -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 " +
+        return ["bash", "-c",
-                "; let rs=\$? ; sleep 10 ; exit \${rs}"]
+                "let x=1 ; while [ \${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=\$? ; sleep 1 ; done ; " +
+                        "cd /tmp/source ; " +
+                        "let y=1 ; while [ \${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=\$? ; sleep 1 ; done ;" +
+                        "./gradlew -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
+                        "let rs=\$? ; sleep 10 ; exit \${rs}"]
     }
 
     Collection<File> downloadTestXmlFromPod(KubernetesClient client, String namespace, Pod cp) {