From 35def14b1f2ddad89c726e337d1fe86e7e673fa7 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 15:31:36 +0000 Subject: [PATCH 1/7] add ability to tigger smoke tests on arbitrary PR --- .ci/dev/smoke/Jenkinsfile | 64 ++++++++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 603bc520dc..23e0e69749 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -4,22 +4,24 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { + agent { label 'k8s' } + options { timestamps() } + triggers { issueCommentTrigger('.*smoke tests.*') } - agent { label 'k8s' } - options { timestamps() } - environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}st" + DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" + ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { - stage('Smoke Tests') { + stage('Corda Smoke Tests - Generate Build Image') { steps { + script { pullRequest.createStatus(status: 'pending', context: 'continuous-integration/jenkins/pr-merge/smokeTest', @@ -29,24 +31,50 @@ pipeline { withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + "-Dkubenetize=true " + - "-DpreAllocatePods=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelSmokeTest" + " clean pushBuildImage --stacktrace" } + + + sh "kubectl auth can-i get pods" + } + } + + stage('Smoke Tests') { + + steps { + + script { + pullRequest.createStatus(status: 'pending', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Running', + targetUrl: "${env.JOB_URL}") + } + + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Dgit.branch=\"\${GIT_BRANCH}\" " + + "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + + " allParallelSmokeTest --stacktrace" } } } - post { + post { always { - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: false + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' } + success { script { pullRequest.createStatus(status: 'success', @@ -69,4 +97,18 @@ pipeline { deleteDir() /* clean up our workspace */ } } -} \ No newline at end of file +} + +//@NonCPS +//static def currentBuildTriggeredByComment(){ +// def triggerCause = currentBuild.rawBuild.getCause(org.jenkinsci.plugins.pipeline.github.trigger.IssueCommentCause) +// if (triggerCause) { +// echo("Build was started by ${triggerCause.userLogin}, who wrote: " + +// "\"${triggerCause.comment}\", which matches the " + +// "\"${triggerCause.triggerPattern}\" trigger pattern.") +// } else { +// echo('Build was not started by a trigger') +// } +// +// return triggerCause == null +//} \ No newline at end of file From bf792c63d5ef0c29f7afe421c02759d44cbd5771 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 16:18:31 +0000 Subject: [PATCH 2/7] redesign Jenkinsfile for smoketests to determine if current build is a comment triggered build --- .ci/dev/smoke/Jenkinsfile | 124 ++++++++++++++++---------------------- 1 file changed, 51 insertions(+), 73 deletions(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 23e0e69749..25bfae3c81 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -19,96 +19,74 @@ pipeline { } stages { - stage('Corda Smoke Tests - Generate Build Image') { + stage('Corda Smoke Tests') { steps { - script { - pullRequest.createStatus(status: 'pending', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Building', - targetUrl: "${env.JOB_URL}") + if (currentBuildTriggeredByComment()) { + stage('Run Smoke Tests') { + script { + pullRequest.createStatus(status: 'pending', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Building', + targetUrl: "${env.JOB_URL}") + } + + withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { + sh "./gradlew " + + "-Dkubenetize=true " + + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + " clean allParallelSmokeTest --stacktrace" + } + } + + } } - - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage --stacktrace" - } - - - sh "kubectl auth can-i get pods" } } - stage('Smoke Tests') { + post { + always { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' + } - steps { + success { script { - pullRequest.createStatus(status: 'pending', + pullRequest.createStatus(status: 'success', context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Running', - targetUrl: "${env.JOB_URL}") + description: 'Smoke Tests Passed', + targetUrl: "${env.JOB_URL}testResults") } - - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " allParallelSmokeTest --stacktrace" } - } - } - - post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' - } - - - success { - script { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Passed', - targetUrl: "${env.JOB_URL}testResults") + failure { + script { + pullRequest.createStatus(status: 'failure', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Failed', + targetUrl: "${env.JOB_URL}testResults") + } } - } - failure { - script { - pullRequest.createStatus(status: 'failure', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Failed', - targetUrl: "${env.JOB_URL}testResults") + cleanup { + deleteDir() /* clean up our workspace */ } } - - cleanup { - deleteDir() /* clean up our workspace */ - } } } -//@NonCPS -//static def currentBuildTriggeredByComment(){ -// def triggerCause = currentBuild.rawBuild.getCause(org.jenkinsci.plugins.pipeline.github.trigger.IssueCommentCause) -// if (triggerCause) { -// echo("Build was started by ${triggerCause.userLogin}, who wrote: " + -// "\"${triggerCause.comment}\", which matches the " + -// "\"${triggerCause.triggerPattern}\" trigger pattern.") -// } else { -// echo('Build was not started by a trigger') -// } -// -// return triggerCause == null -//} \ No newline at end of file +@NonCPS +def currentBuildTriggeredByComment() { + def triggerCause = currentBuild.rawBuild.getCause(org.jenkinsci.plugins.pipeline.github.trigger.IssueCommentCause) + if (triggerCause) { + echo("Build was started by ${triggerCause.userLogin}, who wrote: " + + "\"${triggerCause.comment}\", which matches the " + + "\"${triggerCause.triggerPattern}\" trigger pattern.") + } else { + echo('Build was not started by a trigger') + } + + return triggerCause == null +} \ No newline at end of file From 6ae082f67ff31e81f339e799991401036930ba10 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 16:19:55 +0000 Subject: [PATCH 3/7] fix pipeline syntax errors --- .ci/dev/smoke/Jenkinsfile | 51 ++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 25bfae3c81..bc2004608c 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -1,3 +1,4 @@ +import static com.r3.build.BuildControl.killAllExistingBuildsForJob @Library('existing-build-control') import static com.r3.build.BuildControl.killAllExistingBuildsForJob @@ -27,7 +28,7 @@ pipeline { script { pullRequest.createStatus(status: 'pending', context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Building', + description: 'Smoke Tests Running', targetUrl: "${env.JOB_URL}") } @@ -44,35 +45,35 @@ pipeline { } } } + } - post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + post { + always { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' + } + + + success { + script { + pullRequest.createStatus(status: 'success', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Passed', + targetUrl: "${env.JOB_URL}testResults") } + } - - success { - script { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Passed', - targetUrl: "${env.JOB_URL}testResults") - } + failure { + script { + pullRequest.createStatus(status: 'failure', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Failed', + targetUrl: "${env.JOB_URL}testResults") } + } - failure { - script { - pullRequest.createStatus(status: 'failure', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Failed', - targetUrl: "${env.JOB_URL}testResults") - } - } - - cleanup { - deleteDir() /* clean up our workspace */ - } + cleanup { + deleteDir() /* clean up our workspace */ } } } From f8588277571dd9eaa7d75eefef7dc5e80c05c996 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 16:24:14 +0000 Subject: [PATCH 4/7] trigger is not null when comment caused build --- .ci/dev/smoke/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index bc2004608c..7af32f482e 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -89,5 +89,5 @@ def currentBuildTriggeredByComment() { echo('Build was not started by a trigger') } - return triggerCause == null + return triggerCause != null } \ No newline at end of file From 62002e0b3cf20bc8bf950070a80906f4ebee2c16 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 16:28:52 +0000 Subject: [PATCH 5/7] surround post steps with check if comment build --- .ci/dev/smoke/Jenkinsfile | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 7af32f482e..1fb15a7582 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -49,26 +49,34 @@ pipeline { post { always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + script { + if (currentBuildTriggeredByComment()) { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' + } + } } success { script { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Passed', - targetUrl: "${env.JOB_URL}testResults") + if (currentBuildTriggeredByComment()) { + pullRequest.createStatus(status: 'success', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Passed', + targetUrl: "${env.JOB_URL}testResults") + } } } failure { script { - pullRequest.createStatus(status: 'failure', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Failed', - targetUrl: "${env.JOB_URL}testResults") + if (currentBuildTriggeredByComment()) { + pullRequest.createStatus(status: 'failure', + context: 'continuous-integration/jenkins/pr-merge/smokeTest', + description: 'Smoke Tests Failed', + targetUrl: "${env.JOB_URL}testResults") + } } } From 2dab35c362084312fe79e070e6b422f686fa4e54 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 17:00:38 +0000 Subject: [PATCH 6/7] tidy up environment within pipeline --- .ci/dev/smoke/Jenkinsfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 1fb15a7582..0c93fd2b79 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -1,4 +1,3 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob @Library('existing-build-control') import static com.r3.build.BuildControl.killAllExistingBuildsForJob @@ -13,10 +12,7 @@ pipeline { } environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { From 718b7abb2f8f4cc8604727c2a0b48f07cc6bebb5 Mon Sep 17 00:00:00 2001 From: stefano Date: Mon, 4 Nov 2019 17:12:50 +0000 Subject: [PATCH 7/7] add pod index to container output message to make it easier to find in artifacts --- buildSrc/src/main/groovy/net/corda/testing/KubesTest.java | 2 +- .../src/main/java/net/corda/testing/KubePodResult.java | 8 +++++++- .../src/main/java/net/corda/testing/KubesReporting.java | 6 +++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java index 09e713772f..736e3a80c1 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java +++ b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java @@ -257,7 +257,7 @@ public class KubesTest extends DefaultTask { client.pods().delete(createdPod); client.persistentVolumeClaims().delete(pvc); } - return new KubePodResult(resCode, podOutput, binaryResults); + return new KubePodResult(podIdx, resCode, podOutput, binaryResults); }); } catch (Retry.RetryException e) { throw new RuntimeException("Failed to build in pod " + podName + " (" + podIdx + "/" + numberOfPods + ") in " + numberOfRetries + " attempts", e); diff --git a/buildSrc/src/main/java/net/corda/testing/KubePodResult.java b/buildSrc/src/main/java/net/corda/testing/KubePodResult.java index 2f9a987f11..76ba668c39 100644 --- a/buildSrc/src/main/java/net/corda/testing/KubePodResult.java +++ b/buildSrc/src/main/java/net/corda/testing/KubePodResult.java @@ -5,11 +5,13 @@ import java.util.Collection; public class KubePodResult { + private final int podIndex; private final int resultCode; private final File output; private final Collection binaryResults; - public KubePodResult(int resultCode, File output, Collection binaryResults) { + public KubePodResult(int podIndex, int resultCode, File output, Collection binaryResults) { + this.podIndex = podIndex; this.resultCode = resultCode; this.output = output; this.binaryResults = binaryResults; @@ -26,4 +28,8 @@ public class KubePodResult { public Collection getBinaryResults() { return binaryResults; } + + public int getPodIndex() { + return podIndex; + } } diff --git a/buildSrc/src/main/java/net/corda/testing/KubesReporting.java b/buildSrc/src/main/java/net/corda/testing/KubesReporting.java index ac2306541e..4f8f8aee0c 100644 --- a/buildSrc/src/main/java/net/corda/testing/KubesReporting.java +++ b/buildSrc/src/main/java/net/corda/testing/KubesReporting.java @@ -151,12 +151,12 @@ public class KubesReporting extends DefaultTask { if (!containersWithNonZeroReturnCodes.isEmpty()) { String reportUrl = new ConsoleRenderer().asClickableFileUrl(new File(destinationDir, "index.html")); - if (shouldPrintOutput){ + if (shouldPrintOutput) { containersWithNonZeroReturnCodes.forEach(podResult -> { try { - System.out.println("\n##### CONTAINER OUTPUT START #####"); + System.out.println("\n##### CONTAINER " + podResult.getPodIndex() + " OUTPUT START #####"); IOUtils.copy(new FileInputStream(podResult.getOutput()), System.out); - System.out.println("##### CONTAINER OUTPUT END #####\n"); + System.out.println("##### CONTAINER " + podResult.getPodIndex() + " OUTPUT END #####\n"); } catch (IOException ignored) { } });