From 8d994283c6b92769d669f63e04305cd32b20ce21 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Tue, 1 Jun 2021 16:21:22 +0200 Subject: [PATCH] INFRA-1299: switching from AKS to linear builds on two Jenkins agents * removed `.ci/dev/unit/Jenkinsfile` and `.ci/dev/integration/Jenkinsfile` - they don't seem to be used anywhere * switched mandatory tests to linear build - modeled after Corda Enterprise * switched nightly regression tests to linear build - modeled after Corda Enterprise --- .ci/dev/integration/Jenkinsfile | 62 ---------- .ci/dev/nightly-regression/Jenkinsfile | 164 ++++++++++++++++++++----- .ci/dev/pr-code-checks/Jenkinsfile | 32 +---- .ci/dev/unit/Jenkinsfile | 60 --------- Jenkinsfile | 144 +++++++++++++++------- 5 files changed, 232 insertions(+), 230 deletions(-) delete mode 100644 .ci/dev/integration/Jenkinsfile delete mode 100644 .ci/dev/unit/Jenkinsfile diff --git a/.ci/dev/integration/Jenkinsfile b/.ci/dev/integration/Jenkinsfile deleted file mode 100644 index 1e5d032e8d..0000000000 --- a/.ci/dev/integration/Jenkinsfile +++ /dev/null @@ -1,62 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'local-k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Corda - Run Tests') { - stage('Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelIntegrationTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/integrationTest', - description: 'Integration Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 303bd722a9..5e34fcc408 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -1,66 +1,164 @@ +#!groovy +/** + * Jenkins pipeline to build Corda Opensource nightly regression for feature branches. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Common Gradle arguments for all Gradle executions + */ +String COMMON_GRADLE_PARAMS = [ + '--no-daemon', + '--stacktrace', + '--info', + '-Pcompilation.warningsAsErrors=false', + '-Ptests.failFast=true', +].join(' ') + pipeline { - agent { label 'k8s' } + agent { label 'standard' } + + /* + * List options in alphabetical order + */ options { - timestamps() + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) overrideIndexTriggers(false) - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) - timeout(time: 3, unit: 'HOURS') + parallelsAlwaysFailFast() + timeout(time: 6, unit: 'HOURS') + timestamps() } + triggers { pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight' } + /* + * List environment variables in alphabetical order + */ environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { - stage('Generate Build Image') { + stage('Compile') { steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage --stacktrace" - } - sh "kubectl auth can-i get pods" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'clean', + 'jar' + ].join(' ') } } - stage('Regression Test') { + stage('Stash') { steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" + stash name: 'compiled', useDefaultExcludes: false + } + } + + stage('All Tests') { + parallel { + stage('Another agent') { + agent { + label 'standard' + } + options { + skipDefaultCheckout true + } + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + stages { + stage('Unstash') { + steps { + unstash 'compiled' + } + } + stage('Recompile') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'jar' + ].join(' ') + } + } + stage('Unit Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } + stage('Proxy Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'proxyIntegrationTest' + ].join(' ') + } + } + stage('Smoke Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'smokeTest' + ].join(' ') + } + } + stage('Slow Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'slowIntegrationTest' + ].join(' ') + } + } + } + } + stage('Same agent') { + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + stages { + stage('Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } + } + } } } } - post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml' - } cleanup { deleteDir() /* clean up our workspace */ } } } - diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index a64813c92f..5cb1058811 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -4,27 +4,16 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } - environment { - PR_CONTEXT_STRING = "PR Code Checks" - } - stages { stage('Detekt check') { steps { - script { - pullRequest.createStatus( - status: 'pending', - context: "${PR_CONTEXT_STRING}", - description: "Running code checks", - targetUrl: "${env.BUILD_URL}") - } sh "./gradlew --no-daemon clean detekt" } } @@ -50,25 +39,6 @@ pipeline { } post { - success { - script { - pullRequest.createStatus( - status: 'success', - context: "${PR_CONTEXT_STRING}", - description: 'Code checks passed', - targetUrl: "${env.BUILD_URL}") - } - } - - failure { - script { - pullRequest.createStatus( - status: 'failure', - context: "${PR_CONTEXT_STRING}", - description: 'Code checks failed', - targetUrl: "${env.BUILD_URL}") - } - } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/.ci/dev/unit/Jenkinsfile b/.ci/dev/unit/Jenkinsfile deleted file mode 100644 index 65a6cc08ae..0000000000 --- a/.ci/dev/unit/Jenkinsfile +++ /dev/null @@ -1,60 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda Pull Request - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Unit Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelUnitTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/unitTest', - description: 'Unit Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 5f02f89de0..0b5a887f02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,64 +1,124 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob +#!groovy +/** + * Jenkins pipeline to build Corda Opensource Pull Requests. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Common Gradle arguments for all Gradle executions + */ +String COMMON_GRADLE_PARAMS = [ + '--no-daemon', + '--stacktrace', + '--info', + '-Pcompilation.warningsAsErrors=false', + '-Ptests.failFast=true', +].join(' ') + pipeline { - agent { label 'k8s' } + agent { label 'standard' } + + /* + * List options in alphabetical order + */ options { - timestamps() - timeout(time: 3, unit: 'HOURS') + ansiColor('xterm') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) + parallelsAlwaysFailFast() + timeout(time: 6, unit: 'HOURS') + timestamps() } + /* + * List environment variables in alphabetical order + */ environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { - stage('Corda Pull Request - Generate Build Image') { + stage('Compile') { steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelUnitTest --stacktrace" - } - sh "kubectl auth can-i get pods" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'clean', + 'jar' + ].join(' ') } } - stage('Corda Pull Request - Run Tests') { + stage('Stash') { + steps { + stash name: 'compiled', useDefaultExcludes: false + } + } + + stage('All Tests') { parallel { - stage('Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelIntegrationTest allParallelIntegrationTest --stacktrace" + stage('Another agent') { + agent { + label 'standard' + } + options { + skipDefaultCheckout true + } + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + stages { + stage('Unstash') { + steps { + unstash 'compiled' + } + } + stage('Recompile') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'jar' + ].join(' ') + } + } + stage('Unit Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } } } - stage('Unit Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelUnitTest allParallelUnitTest --stacktrace" + stage('Same agent') { + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + stages { + stage('Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } } } } @@ -66,10 +126,6 @@ pipeline { } post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' - } cleanup { deleteDir() /* clean up our workspace */ }