From 813e9dd242919d34a66aa5ce7c23352bf5954ae0 Mon Sep 17 00:00:00 2001 From: Razvan Codreanu <52859362+Schife@users.noreply.github.com> Date: Thu, 18 Jun 2020 12:52:33 +0100 Subject: [PATCH 1/6] INFRA-394 enable full junit log to avoid truncation (#6368) --- .ci/dev/integration/Jenkinsfile | 2 +- .ci/dev/nightly-regression/Jenkinsfile | 2 +- .ci/dev/regression/Jenkinsfile | 2 +- .ci/dev/smoke/Jenkinsfile | 2 +- .ci/dev/unit/Jenkinsfile | 2 +- Jenkinsfile | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/dev/integration/Jenkinsfile b/.ci/dev/integration/Jenkinsfile index eba467e5a7..e8610ee716 100644 --- a/.ci/dev/integration/Jenkinsfile +++ b/.ci/dev/integration/Jenkinsfile @@ -53,7 +53,7 @@ pipeline { post { always { - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index a7f5d5ba17..62b2fcd820 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -79,7 +79,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true + junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true, keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 899f44684e..354f693f59 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -69,7 +69,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true script { try { diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile index 3ddc3cdce8..aeb6638e95 100644 --- a/.ci/dev/smoke/Jenkinsfile +++ b/.ci/dev/smoke/Jenkinsfile @@ -52,7 +52,7 @@ pipeline { script { if (currentBuildTriggeredByComment()) { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } } } diff --git a/.ci/dev/unit/Jenkinsfile b/.ci/dev/unit/Jenkinsfile index b2d2d54393..14b93f7425 100644 --- a/.ci/dev/unit/Jenkinsfile +++ b/.ci/dev/unit/Jenkinsfile @@ -51,7 +51,7 @@ pipeline { post { always { - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/Jenkinsfile b/Jenkinsfile index 8048bed1ae..02365f00a4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,7 +68,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true } cleanup { deleteDir() /* clean up our workspace */ From 494073590b411014c096770a63a20c64b27a854b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Thu, 18 Jun 2020 23:19:06 +0100 Subject: [PATCH 2/6] INFRA-347: Corda builds for Windows Servers (#6356) --- .ci/dev/mswin/Jenkinsfile | 93 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .ci/dev/mswin/Jenkinsfile diff --git a/.ci/dev/mswin/Jenkinsfile b/.ci/dev/mswin/Jenkinsfile new file mode 100644 index 0000000000..8a7d19c5a3 --- /dev/null +++ b/.ci/dev/mswin/Jenkinsfile @@ -0,0 +1,93 @@ +#!groovy +/** + * Jenkins pipeline to build Corda on MS Windows server. + * Because it takes a long time to run tests sequentially, unit tests and + * integration tests are started in parallel on separate agents. + * + * Additionally, pull requests by default run only unit tests. + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +/** + * Sense environment + */ +boolean isReleaseBranch = (env.BRANCH_NAME =~ /^release\/os\/.*/) + +pipeline { + agent none + options { + ansiColor('xterm') + timestamps() + timeout(time: 3, unit: 'HOURS') + } + + parameters { + booleanParam defaultValue: (isReleaseBranch), description: 'Run integration tests?', name: 'DO_INTEGRATION_TESTS' + } + + /* + * Do no receive Github's push events for release branches -> suitable for nightly builds + * but projects for pull requests will receive them as normal, and PR builds are started ASAP + */ + triggers { + pollSCM ignorePostCommitHooks: isReleaseBranch, scmpoll_spec: '@midnight' + } + + stages { + stage('Tests') { + parallel { + stage('Unit Tests') { + agent { label 'mswin' } + steps { + sh "./gradlew --no-daemon " + + "--stacktrace " + + "-Pcompilation.warningsAsErrors=false " + + "-Ptests.failFast=true " + + "clean test" + } + post { + always { + archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + bat '.ci/kill_corda_procs.cmd' + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + + } + stage('Integration Tests') { + when { + expression { params.DO_INTEGRATION_TESTS } + beforeAgent true + } + agent { label 'mswin' } + steps { + sh "./gradlew --no-daemon " + + "clean integrationTest" + } + post { + always { + archiveArtifacts allowEmptyArchive: true, artifacts: '**/logs/**/*.log' + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true + bat '.ci/kill_corda_procs.cmd' + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } + } + } + } + } +} From 4bfb595e57f450d6cc158e98466b351830935d65 Mon Sep 17 00:00:00 2001 From: Euan Cairncross <35581327+cairncross@users.noreply.github.com> Date: Thu, 18 Jun 2020 23:46:29 +0100 Subject: [PATCH 3/6] INFRA-357 - Add release tag Jenkins build for OS 4.4 (#6362) --- .ci/dev/regression/Jenkinsfile | 82 ++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 354f693f59..be45291160 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,8 +1,24 @@ +#!groovy +/** + * Jenkins pipeline to build Corda OS release branches and tags + */ + +/** + * Kill already started job. + * Assume new commit takes precendence and results from previous + * unfinished builds are not required. + * This feature doesn't play well with disableConcurrentBuilds() option + */ @Library('corda-shared-build-pipeline-steps') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Sense environment + */ +boolean isTag = (env.TAG_NAME =~ /^release-V(\d+\.\d+)?(\.\d+)?(-.+)?$/) + pipeline { agent { label 'k8s' } options { @@ -19,6 +35,12 @@ pipeline { } stages { + stage('Deploy Nodes') { + steps { + sh "./gradlew --no-daemon jar deployNodes" + } + } + stage('Generate Build Image') { steps { withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { @@ -27,7 +49,7 @@ pipeline { "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean preAllocateForParallelRegressionTest preAllocateForAllSlowIntegrationTest pushBuildImage --stacktrace" + " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace" } sh "kubectl auth can-i get pods" } @@ -101,38 +123,40 @@ pipeline { } } } - + script { - // We want to send a summary email, but want to limit to once per day. - // Comparing the dates of the previous and current builds achieves this, - // i.e. we will only send an email for the first build on a given day. - def prevBuildDate = new Date( - currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() - def currentBuildDate = new Date( - currentBuild.timeInMillis).clearTime() + if (!isTag) { + // We want to send a summary email, but want to limit to once per day. + // Comparing the dates of the previous and current builds achieves this, + // i.e. we will only send an email for the first build on a given day. + def prevBuildDate = new Date( + currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() + def currentBuildDate = new Date( + currentBuild.timeInMillis).clearTime() - if (prevBuildDate != currentBuildDate) { - def statusSymbol = '\u2753' - switch(currentBuild.result) { - case 'SUCCESS': - statusSymbol = '\u2705' - break; - case 'UNSTABLE': - case 'FAILURE': - statusSymbol = '\u274c' - break; - default: - break; + if (prevBuildDate != currentBuildDate) { + def statusSymbol = '\u2753' + switch(currentBuild.result) { + case 'SUCCESS': + statusSymbol = '\u2705' + break; + case 'UNSTABLE': + case 'FAILURE': + statusSymbol = '\u274c' + break; + default: + break; + } + + echo('First build for this date, sending summary email') + emailext to: '$DEFAULT_RECIPIENTS', + subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', + mimeType: 'text/html', + body: '${SCRIPT, template="groovy-html.template"}' + } else { + echo('Already sent summary email today, suppressing') } - - echo('First build for this date, sending summary email') - emailext to: '$DEFAULT_RECIPIENTS', - subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', - mimeType: 'text/html', - body: '${SCRIPT, template="groovy-html.template"}' - } else { - echo('Already sent summary email today, suppressing') } } } From e77f7a7546b68188b8a9d4e732846ffc425a5462 Mon Sep 17 00:00:00 2001 From: alicer3 <50432430+alicer3@users.noreply.github.com> Date: Fri, 19 Jun 2020 16:49:07 +0800 Subject: [PATCH 4/6] center console message for registration (#6191) --- .../internal/subcommands/InitialRegistrationCli.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt b/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt index d0ea54969e..881de2c7df 100644 --- a/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt +++ b/node/src/main/kotlin/net/corda/node/internal/subcommands/InitialRegistrationCli.kt @@ -62,11 +62,11 @@ class InitialRegistration(val baseDirectory: Path, private val networkRootTrustS val versionInfo = startup.getVersionInfo() println("\n" + - "******************************************************************\n" + - "* *\n" + - "* Registering as a new participant with a Corda network *\n" + - "* *\n" + - "******************************************************************\n") + "*******************************************************************\n" + + "* *\n" + + "* Registering as a new participant with a Corda network *\n" + + "* *\n" + + "*******************************************************************\n") NodeRegistrationHelper(NodeRegistrationConfiguration(conf), HTTPNetworkRegistrationService( From ad21e381dc16af7c80c0ffbe3993edfb6e2666c1 Mon Sep 17 00:00:00 2001 From: Razvan Codreanu <52859362+Schife@users.noreply.github.com> Date: Fri, 19 Jun 2020 14:10:31 +0100 Subject: [PATCH 5/6] INFRA-357 Adding JDK 11 Regression and Build tag jenkinsfile (#6374) --- ...DockerfileJDK11Compile => DockerfileJDK11} | 0 .ci/dev/compatibility/JenkinsfileJDK11Azul | 121 ++++++++++++++++++ .ci/dev/compatibility/JenkinsfileJDK11Compile | 2 +- testing/DockerfileJDK11Azul | 3 + 4 files changed, 125 insertions(+), 1 deletion(-) rename .ci/dev/compatibility/{DockerfileJDK11Compile => DockerfileJDK11} (100%) create mode 100644 .ci/dev/compatibility/JenkinsfileJDK11Azul create mode 100644 testing/DockerfileJDK11Azul diff --git a/.ci/dev/compatibility/DockerfileJDK11Compile b/.ci/dev/compatibility/DockerfileJDK11 similarity index 100% rename from .ci/dev/compatibility/DockerfileJDK11Compile rename to .ci/dev/compatibility/DockerfileJDK11 diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul new file mode 100644 index 0000000000..a61b5dc228 --- /dev/null +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -0,0 +1,121 @@ +@Library('corda-shared-build-pipeline-steps') +import static com.r3.build.BuildControl.killAllExistingBuildsForJob + +killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) + +/** + * Sense environment + */ +boolean isReleaseTag = (env.TAG_NAME =~ /^release.*JDK11$/) + +pipeline { + agent { + dockerfile { + label 'k8s' + additionalBuildArgs "--build-arg USER=stresstester" + filename '.ci/dev/compatibility/DockerfileJDK11' + } + } + options { + timestamps() + buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7')) + timeout(time: 3, unit: 'HOURS') + } + + environment { + DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}JDK11" + EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" + BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" + ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") + } + + stages { + stage('Generate Build Image') { + steps { + withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { + sh "./gradlew " + + "-Dkubenetize=true " + + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Ddocker.buildbase.tag=11latest " + + "-Ddocker.dockerfile=DockerfileJDK11Azul" + + " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace" + } + sh "kubectl auth can-i get pods" + } + } + + stage('Testing phase') { + parallel { + stage('Regression Test') { + steps { + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Dgit.branch=\"\${GIT_BRANCH}\" " + + "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + + " parallelRegressionTest --stacktrace" + } + } + stage('Slow Integration Test') { + steps { + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + + "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + + "-Dgit.branch=\"\${GIT_BRANCH}\" " + + "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + + " allParallelSlowIntegrationTest --stacktrace" + } + } + } + } + + stage('Publish to Artifactory') { + when { + expression { isReleaseTag } + } + steps { + rtServer( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'r3-corda-releases' + ) + rtGradleRun( + usesPlugin: true, + useWrapper: true, + switches: '-s --info', + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + rtPublishBuildInfo( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } + } + + post { + always { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false + junit '**/build/test-results-xml/**/*.xml' + } + cleanup { + deleteDir() /* clean up our workspace */ + } + } +} diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index 670717da68..d2251fad15 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -8,7 +8,7 @@ pipeline { dockerfile { label 'k8s' additionalBuildArgs "--build-arg USER=stresstester" - filename '.ci/dev/compatibility/DockerfileJDK11Compile' + filename '.ci/dev/compatibility/DockerfileJDK11' } } options { diff --git a/testing/DockerfileJDK11Azul b/testing/DockerfileJDK11Azul new file mode 100644 index 0000000000..83f3c324de --- /dev/null +++ b/testing/DockerfileJDK11Azul @@ -0,0 +1,3 @@ +FROM stefanotestingcr.azurecr.io/buildbase:11latest +COPY . /tmp/source +CMD cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --inf \ No newline at end of file From 23f81e73a6299679fcd0a916a94a40d8490d7ed0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Fri, 19 Jun 2020 19:30:11 +0100 Subject: [PATCH 6/6] Remove obsolete Jenkins configration for smoke tests (#6379) --- .ci/dev/smoke/Jenkinsfile | 101 -------------------------------------- 1 file changed, 101 deletions(-) delete mode 100644 .ci/dev/smoke/Jenkinsfile diff --git a/.ci/dev/smoke/Jenkinsfile b/.ci/dev/smoke/Jenkinsfile deleted file mode 100644 index 3ddc3cdce8..0000000000 --- a/.ci/dev/smoke/Jenkinsfile +++ /dev/null @@ -1,101 +0,0 @@ -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - -pipeline { - agent { label 'k8s' } - options { - timestamps() - overrideIndexTriggers(false) - timeout(time: 3, unit: 'HOURS') - } - - triggers { - issueCommentTrigger('.*smoke tests.*') - } - - environment { - EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" - } - - stages { - stage('Corda Smoke Tests') { - steps { - script { - if (currentBuildTriggeredByComment()) { - stage('Run Smoke Tests') { - script { - pullRequest.createStatus(status: 'pending', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Running', - targetUrl: "${env.JOB_URL}") - } - - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - " clean allParallelSmokeTest --stacktrace" - } - } - - } - } - } - } - } - - post { - always { - script { - if (currentBuildTriggeredByComment()) { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' - } - } - } - - - success { - script { - if (currentBuildTriggeredByComment()) { - pullRequest.createStatus(status: 'success', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Passed', - targetUrl: "${env.BUILD_URL}testResults") - } - } - } - - failure { - script { - if (currentBuildTriggeredByComment()) { - pullRequest.createStatus(status: 'failure', - context: 'continuous-integration/jenkins/pr-merge/smokeTest', - description: 'Smoke Tests Failed', - targetUrl: "${env.BUILD_URL}testResults") - } - } - } - - cleanup { - deleteDir() /* clean up our workspace */ - } - } -} - -@NonCPS -def currentBuildTriggeredByComment() { - def triggerCause = currentBuild.rawBuild.getCause(org.jenkinsci.plugins.pipeline.github.trigger.IssueCommentCause) - if (triggerCause) { - echo("Build was started by ${triggerCause.userLogin}, who wrote: " + - "\"${triggerCause.comment}\", which matches the " + - "\"${triggerCause.triggerPattern}\" trigger pattern.") - } else { - echo('Build was not started by a trigger') - } - - return triggerCause != null -}