From 19274751d91e93ece6c4772f2c9cf871c3cde5e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Fri, 13 Nov 2020 18:03:33 +0100 Subject: [PATCH 1/5] INFRA-965: Jenkins/NexusIQ integration should target patches (#6802) as well as major/minor releases --- .ci/dev/regression/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index f773438b4b..fb6e3896ff 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -100,7 +100,7 @@ pipeline { script { sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ - def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim() + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\(\\.[0-9]\\+\\)\\+\\).*\$/\\1/'").trim() def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() def artifactId = 'corda' nexusAppId = "${groupId}-${artifactId}-${version}" From c6c5c065a5e41cc5cb0197debfdbbfe2870cdfc1 Mon Sep 17 00:00:00 2001 From: Nick Dunstone <49945179+nickdunstone13@users.noreply.github.com> Date: Tue, 11 May 2021 12:04:15 +0100 Subject: [PATCH 2/5] ENT-6295: Upgrade to BC 1.68 on O/S 4.5 (#6906) * Upgrade to BC 1.68 on O/S 4.5 * Added Ed25519 fix --- constants.properties | 2 +- .../main/kotlin/net/corda/core/crypto/internal/ProviderMap.kt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/constants.properties b/constants.properties index fdc1a61211..ca2248617b 100644 --- a/constants.properties +++ b/constants.properties @@ -20,7 +20,7 @@ quasarClassifier=jdk8 quasarVersion11=0.8.0_r3 jdkClassifier11=jdk11 proguardVersion=6.1.1 -bouncycastleVersion=1.66 +bouncycastleVersion=1.68 classgraphVersion=4.8.90 disruptorVersion=3.4.2 typesafeConfigVersion=1.3.4 diff --git a/core/src/main/kotlin/net/corda/core/crypto/internal/ProviderMap.kt b/core/src/main/kotlin/net/corda/core/crypto/internal/ProviderMap.kt index d065aa30e1..01e75a339b 100644 --- a/core/src/main/kotlin/net/corda/core/crypto/internal/ProviderMap.kt +++ b/core/src/main/kotlin/net/corda/core/crypto/internal/ProviderMap.kt @@ -35,6 +35,7 @@ val cordaBouncyCastleProvider = BouncyCastleProvider().apply { putAll(EdDSASecurityProvider()) // Override the normal EdDSA engine with one which can handle X509 keys. put("Signature.${EdDSAEngine.SIGNATURE_ALGORITHM}", X509EdDSAEngine::class.java.name) + put("Signature.Ed25519", X509EdDSAEngine::class.java.name) addKeyInfoConverter(`id-Curve25519ph`, object : AsymmetricKeyInfoConverter { override fun generatePublic(keyInfo: SubjectPublicKeyInfo) = decodePublicKey(EDDSA_ED25519_SHA512, keyInfo.encoded) override fun generatePrivate(keyInfo: PrivateKeyInfo) = decodePrivateKey(EDDSA_ED25519_SHA512, keyInfo.encoded) From 8d994283c6b92769d669f63e04305cd32b20ce21 Mon Sep 17 00:00:00 2001 From: Waldemar Zurowski Date: Tue, 1 Jun 2021 16:21:22 +0200 Subject: [PATCH 3/5] INFRA-1299: switching from AKS to linear builds on two Jenkins agents * removed `.ci/dev/unit/Jenkinsfile` and `.ci/dev/integration/Jenkinsfile` - they don't seem to be used anywhere * switched mandatory tests to linear build - modeled after Corda Enterprise * switched nightly regression tests to linear build - modeled after Corda Enterprise --- .ci/dev/integration/Jenkinsfile | 62 ---------- .ci/dev/nightly-regression/Jenkinsfile | 164 ++++++++++++++++++++----- .ci/dev/pr-code-checks/Jenkinsfile | 32 +---- .ci/dev/unit/Jenkinsfile | 60 --------- Jenkinsfile | 144 +++++++++++++++------- 5 files changed, 232 insertions(+), 230 deletions(-) delete mode 100644 .ci/dev/integration/Jenkinsfile delete mode 100644 .ci/dev/unit/Jenkinsfile diff --git a/.ci/dev/integration/Jenkinsfile b/.ci/dev/integration/Jenkinsfile deleted file mode 100644 index 1e5d032e8d..0000000000 --- a/.ci/dev/integration/Jenkinsfile +++ /dev/null @@ -1,62 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'local-k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Corda - Run Tests') { - stage('Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelIntegrationTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/integrationTest', - description: 'Integration Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 303bd722a9..5e34fcc408 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -1,66 +1,164 @@ +#!groovy +/** + * Jenkins pipeline to build Corda Opensource nightly regression for feature branches. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Common Gradle arguments for all Gradle executions + */ +String COMMON_GRADLE_PARAMS = [ + '--no-daemon', + '--stacktrace', + '--info', + '-Pcompilation.warningsAsErrors=false', + '-Ptests.failFast=true', +].join(' ') + pipeline { - agent { label 'k8s' } + agent { label 'standard' } + + /* + * List options in alphabetical order + */ options { - timestamps() + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) overrideIndexTriggers(false) - buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) - timeout(time: 3, unit: 'HOURS') + parallelsAlwaysFailFast() + timeout(time: 6, unit: 'HOURS') + timestamps() } + triggers { pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight' } + /* + * List environment variables in alphabetical order + */ environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { - stage('Generate Build Image') { + stage('Compile') { steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage --stacktrace" - } - sh "kubectl auth can-i get pods" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'clean', + 'jar' + ].join(' ') } } - stage('Regression Test') { + stage('Stash') { steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" + stash name: 'compiled', useDefaultExcludes: false + } + } + + stage('All Tests') { + parallel { + stage('Another agent') { + agent { + label 'standard' + } + options { + skipDefaultCheckout true + } + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + stages { + stage('Unstash') { + steps { + unstash 'compiled' + } + } + stage('Recompile') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'jar' + ].join(' ') + } + } + stage('Unit Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } + stage('Proxy Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'proxyIntegrationTest' + ].join(' ') + } + } + stage('Smoke Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'smokeTest' + ].join(' ') + } + } + stage('Slow Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'slowIntegrationTest' + ].join(' ') + } + } + } + } + stage('Same agent') { + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + stages { + stage('Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } + } + } } } } - post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml' - } cleanup { deleteDir() /* clean up our workspace */ } } } - diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index a64813c92f..5cb1058811 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -4,27 +4,16 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) } - environment { - PR_CONTEXT_STRING = "PR Code Checks" - } - stages { stage('Detekt check') { steps { - script { - pullRequest.createStatus( - status: 'pending', - context: "${PR_CONTEXT_STRING}", - description: "Running code checks", - targetUrl: "${env.BUILD_URL}") - } sh "./gradlew --no-daemon clean detekt" } } @@ -50,25 +39,6 @@ pipeline { } post { - success { - script { - pullRequest.createStatus( - status: 'success', - context: "${PR_CONTEXT_STRING}", - description: 'Code checks passed', - targetUrl: "${env.BUILD_URL}") - } - } - - failure { - script { - pullRequest.createStatus( - status: 'failure', - context: "${PR_CONTEXT_STRING}", - description: 'Code checks failed', - targetUrl: "${env.BUILD_URL}") - } - } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/.ci/dev/unit/Jenkinsfile b/.ci/dev/unit/Jenkinsfile deleted file mode 100644 index 65a6cc08ae..0000000000 --- a/.ci/dev/unit/Jenkinsfile +++ /dev/null @@ -1,60 +0,0 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'k8s' } - options { - timestamps() - timeout(time: 3, unit: 'HOURS') - } - - environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" - } - - stages { - stage('Corda Pull Request - Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Unit Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelUnitTest" - if (env.CHANGE_ID) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/unitTest', - description: 'Unit Tests Passed', - targetUrl: "${env.JOB_URL}/testResults") - } - } - } - } - - post { - always { - junit '**/build/test-results-xml/**/*.xml' - } - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 5f02f89de0..0b5a887f02 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,64 +1,124 @@ -import static com.r3.build.BuildControl.killAllExistingBuildsForJob +#!groovy +/** + * Jenkins pipeline to build Corda Opensource Pull Requests. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Common Gradle arguments for all Gradle executions + */ +String COMMON_GRADLE_PARAMS = [ + '--no-daemon', + '--stacktrace', + '--info', + '-Pcompilation.warningsAsErrors=false', + '-Ptests.failFast=true', +].join(' ') + pipeline { - agent { label 'k8s' } + agent { label 'standard' } + + /* + * List options in alphabetical order + */ options { - timestamps() - timeout(time: 3, unit: 'HOURS') + ansiColor('xterm') + buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) + parallelsAlwaysFailFast() + timeout(time: 6, unit: 'HOURS') + timestamps() } + /* + * List environment variables in alphabetical order + */ environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') } stages { - stage('Corda Pull Request - Generate Build Image') { + stage('Compile') { steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelUnitTest --stacktrace" - } - sh "kubectl auth can-i get pods" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'clean', + 'jar' + ].join(' ') } } - stage('Corda Pull Request - Run Tests') { + stage('Stash') { + steps { + stash name: 'compiled', useDefaultExcludes: false + } + } + + stage('All Tests') { parallel { - stage('Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelIntegrationTest allParallelIntegrationTest --stacktrace" + stage('Another agent') { + agent { + label 'standard' + } + options { + skipDefaultCheckout true + } + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + stages { + stage('Unstash') { + steps { + unstash 'compiled' + } + } + stage('Recompile') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'jar' + ].join(' ') + } + } + stage('Unit Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } } } - stage('Unit Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " + - " deAllocateForAllParallelUnitTest allParallelUnitTest --stacktrace" + stage('Same agent') { + post { + always { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + stages { + stage('Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } } } } @@ -66,10 +126,6 @@ pipeline { } post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' - } cleanup { deleteDir() /* clean up our workspace */ } From 286873a4e44b4576ae899377888f101a0cbaec01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Wed, 2 Jun 2021 14:23:30 +0200 Subject: [PATCH 4/5] INFRA-1299: switch from Azure Jenkins agents to AWS (#6911) * JDK11 compatibility - simply switch to AWS Jenkins agent * Corda preview - simply switch from Azure agent to AWS agent * part of retiring `k8s` agent used for parallel builds --- .ci/dev/compatibility/JenkinsfileJDK11Compile | 9 +++++++-- .ci/dev/publish-branch/Jenkinsfile.preview | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index d84a91db54..b65f0532c9 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -1,3 +1,8 @@ +#!groovy +/** + * Jenkins pipeline to build Corda Opensource Pull Requests with JDK11. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob @@ -6,8 +11,8 @@ killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { agent { dockerfile { - label 'k8s' - additionalBuildArgs "--build-arg USER=stresstester" + label 'standard' + additionalBuildArgs '--build-arg USER="${USER}"' // DON'T change quotation - USER variable is substituted by SHELL!!!! filename '.ci/dev/compatibility/DockerfileJDK11' } } diff --git a/.ci/dev/publish-branch/Jenkinsfile.preview b/.ci/dev/publish-branch/Jenkinsfile.preview index e66deeabab..c1cc300089 100644 --- a/.ci/dev/publish-branch/Jenkinsfile.preview +++ b/.ci/dev/publish-branch/Jenkinsfile.preview @@ -1,11 +1,15 @@ #!groovy +/** + * Jenkins pipeline to build Corda Opensource Preview. + */ + @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) pipeline { - agent { label 'k8s' } + agent { label 'standard' } options { timestamps() From 3cc8d93e3f9a0c066de5dc56cba84fe51d593bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Mon, 7 Jun 2021 10:05:18 +0200 Subject: [PATCH 5/5] INFRA-1299: retire parallel builds for JDK11 releases (#6913) --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 270 +++++++++++++-------- 1 file changed, 173 insertions(+), 97 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index ff799241c8..c2216d8a51 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -23,7 +23,7 @@ boolean isReleaseTag = (env.TAG_NAME =~ /^release.*JDK11$/) ** calculate the stage for NexusIQ evaluation ** * build for snapshots ** * stage-release: for release candidates and for health checks -** * operate: for final release +** * release: for GA release */ def nexusDefaultIqStage = "build" if (isReleaseTag) { @@ -47,141 +47,217 @@ def nexusIqStageChoices = [nexusDefaultIqStage].plus( 'operate' ].minus([nexusDefaultIqStage])) +/** + * Common Gradle arguments for all Gradle executions + */ +String COMMON_GRADLE_PARAMS = [ + '--no-daemon', + '--stacktrace', + '--info', + '-Pcompilation.warningsAsErrors=false', + '-Ptests.failFast=true', +].join(' ') + +/** + * The name of subfolders to run tests previously on Another Agent and Same Agent + */ +String sameAgentFolder = 'sameAgent' +String anotherAgentFolder = 'anotherAgent' + pipeline { - agent { label 'k8s' } + agent { + dockerfile { + label 'standard' + additionalBuildArgs '--build-arg USER="${USER}"' // DON'T change quotation - USER variable is substituted by SHELL!!!! + filename "${sameAgentFolder}/.ci/dev/compatibility/DockerfileJDK11" + } + } + + /* + * List options in alphabetical order + */ options { - timestamps() - timeout(time: 3, unit: 'HOURS') buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14')) + checkoutToSubdirectory "${sameAgentFolder}" + parallelsAlwaysFailFast() + timeout(time: 6, unit: 'HOURS') + timestamps() } parameters { choice choices: nexusIqStageChoices, description: 'NexusIQ stage for code evaluation', name: 'nexusIqStage' } + /* + * List environment variables in alphabetical order + */ environment { - DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}JDK11" - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" + ARTIFACTORY_BUILD_NAME = "Corda :: Publish :: Publish JDK 11 Release to Artifactory :: ${env.BRANCH_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') - ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") - CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" } stages { + stage('Compile') { + steps { + dir(sameAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'clean', + 'jar' + ].join(' ') + } + } + } + + stage('Copy') { + steps { + sh "rm -rf ${anotherAgentFolder} && mkdir -p ${anotherAgentFolder} && cd ${sameAgentFolder} && cp -aR . ../${anotherAgentFolder}" + } + } + stage('Sonatype Check') { steps { - sh "./gradlew --no-daemon clean jar" - script { - sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" - /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ - def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\(\\.[0-9]\\+\\)\\+\\).*\$/\\1/'").trim() - def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() - def artifactId = 'corda' - nexusAppId = "${groupId}-${artifactId}-${version}" + dir(sameAgentFolder) { + script { + sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties" + /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ + def version = sh(returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\(\\.[0-9]\\+\\)\\+\\).*\$/\\1/'").trim() + def groupId = sh(returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "${groupId}-${artifactId}-${version}" + } + nexusPolicyEvaluation( + failBuildOnNetworkError: false, + iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: params.nexusIqStage + ) } - nexusPolicyEvaluation ( - failBuildOnNetworkError: false, - iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! - iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], - iqStage: params.nexusIqStage - ) } } - stage('Generate Build Image') { - steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Ddocker.buildbase.tag=11latest " + - "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Ddocker.dockerfile=DockerfileJDK11Azul" + - " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace" - } - sh "kubectl auth can-i get pods" - } - } - - stage('Testing phase') { + stage('All Tests') { parallel { - stage('Regression Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" + stage('Another agent') { + post { + always { + dir(anotherAgentFolder) { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + } + stages { + stage('Unit Test') { + steps { + dir(anotherAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } + } + stage('Smoke Test') { + steps { + dir(anotherAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'smokeTest' + ].join(' ') + } + } + } + stage('Slow Integration Test') { + steps { + dir(anotherAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'slowIntegrationTest' + ].join(' ') + } + } + } } } - stage('Slow Integration Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " allParallelSlowIntegrationTest --stacktrace" + stage('Same agent') { + post { + always { + dir(sameAgentFolder) { + archiveArtifacts artifacts: '**/*.log', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + } + } + } + stages { + stage('Integration Test') { + steps { + dir(sameAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } + } + + stage('Deploy Node') { + steps { + dir(sameAgentFolder) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'deployNode' + ].join(' ') + } + } + } } } } } stage('Publish to Artifactory') { - agent { - dockerfile { - reuseNode true - additionalBuildArgs "--build-arg USER=stresstester" - filename '.ci/dev/compatibility/DockerfileJDK11' - } - } when { expression { isReleaseTag } } steps { - rtServer( - id: 'R3-Artifactory', - url: 'https://software.r3.com/artifactory', - credentialsId: 'artifactory-credentials' - ) - rtGradleDeployer( - id: 'deployer', - serverId: 'R3-Artifactory', - repo: 'corda-releases' - ) - rtGradleRun( - usesPlugin: true, - useWrapper: true, - switches: '-s --info', - tasks: 'artifactoryPublish', - deployerId: 'deployer', - buildName: env.ARTIFACTORY_BUILD_NAME - ) - rtPublishBuildInfo( - serverId: 'R3-Artifactory', - buildName: env.ARTIFACTORY_BUILD_NAME - ) + dir(sameAgentFolder) { + rtServer( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'corda-releases' + ) + rtGradleRun( + usesPlugin: true, + useWrapper: true, + switches: '-s --info', + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + rtPublishBuildInfo( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } } } } post { - always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true - } cleanup { deleteDir() /* clean up our workspace */ }