From b3d9d1291f74e0b3cc4be99425f87ac8fe18f789 Mon Sep 17 00:00:00 2001 From: Ross Nicoll Date: Mon, 14 Sep 2020 11:29:42 +0100 Subject: [PATCH] INFRA-683 Move Corda OS release branch builds to serial (#6703) Co-authored-by: Waldemar Zurowski --- .ci/dev/regression/Jenkinsfile | 273 ++++++++++++++---- build.gradle | 2 +- .../services/network/NetworkMapUpdaterTest.kt | 15 +- .../persistence/NodeAttachmentServiceTest.kt | 2 +- 4 files changed, 227 insertions(+), 65 deletions(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 41cc2ad218..3b131da9d2 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,58 +1,209 @@ -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob +#!groovy +/** + * Jenkins pipeline to build Corda OS release branches and tags. + * PLEASE NOTE: we DO want to run a build for each commit!!! + */ -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) +/** + * Sense environment + */ +boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?version-properties" + /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ + def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\.[0-9]\\+\\).*\$/\\1/'").trim() + def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim() + def artifactId = 'corda' + nexusAppId = "${groupId}-${artifactId}-${version}" } - sh "kubectl auth can-i get pods" + nexusPolicyEvaluation ( + failBuildOnNetworkError: false, + iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts! + iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']], + iqStage: params.nexusIqStage + ) } } - stage('Regression Test') { + stage('Unit Test') { steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') + } + } + + stage('Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') + } + } + + stage('Smoke Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'smokeTest' + ].join(' ') + } + } + + stage('Slow Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'slowIntegrationTest' + ].join(' ') + } + } + + stage('Deploy Node') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'deployNode' + ].join(' ') + } + } + + stage('Publish to Artifactory') { + when { + expression { isReleaseTag } + } + steps { + rtServer( + id: 'R3-Artifactory', + url: 'https://software.r3.com/artifactory', + credentialsId: 'artifactory-credentials' + ) + rtGradleDeployer( + id: 'deployer', + serverId: 'R3-Artifactory', + repo: 'corda-releases' + ) + rtGradleRun( + usesPlugin: true, + useWrapper: true, + switches: '-s --info', + tasks: 'artifactoryPublish', + deployerId: 'deployer', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + rtPublishBuildInfo( + serverId: 'R3-Artifactory', + buildName: env.ARTIFACTORY_BUILD_NAME + ) + } + } + + stage('Publish Release to Docker Hub') { + when { + expression { !isInternalRelease && isReleaseTag } + } + steps { + withCredentials([ + usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials', + usernameVariable: 'DOCKER_USERNAME', + passwordVariable: 'DOCKER_PASSWORD')]) { + sh "./gradlew pushOfficialImages" + } } } } - post { always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + archiveArtifacts artifacts: '**/*.log', fingerprint: false + archiveArtifacts artifacts: '**/build/reports/tests/**', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true script { try { @@ -84,38 +235,42 @@ pipeline { } } } - + script { - // We want to send a summary email, but want to limit to once per day. - // Comparing the dates of the previous and current builds achieves this, - // i.e. we will only send an email for the first build on a given day. - def prevBuildDate = new Date( - currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() - def currentBuildDate = new Date( - currentBuild.timeInMillis).clearTime() + if (!isReleaseTag) { + // We want to send a summary email, but want to limit to once per day. + // Comparing the dates of the previous and current builds achieves this, + // i.e. we will only send an email for the first build on a given day. + def prevBuildDate = new Date( + currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() + def currentBuildDate = new Date( + currentBuild.timeInMillis).clearTime() - if (prevBuildDate != currentBuildDate) { - def statusSymbol = '\u2753' - switch(currentBuild.result) { - case 'SUCCESS': - statusSymbol = '\u2705' - break; - case 'UNSTABLE': - case 'FAILURE': - statusSymbol = '\u274c' - break; - default: - break; + if (prevBuildDate != currentBuildDate) { + def statusSymbol = '\u2753' + switch(currentBuild.result) { + case 'SUCCESS': + statusSymbol = '\u2705' + break; + case 'UNSTABLE': + statusSymbol = '\u26A0' + break; + case 'FAILURE': + statusSymbol = '\u274c' + break; + default: + break; + } + + echo('First build for this date, sending summary email') + emailext to: '$DEFAULT_RECIPIENTS', + subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', + mimeType: 'text/html', + body: '${SCRIPT, template="groovy-html.template"}' + } else { + echo('Already sent summary email today, suppressing') } - - echo('First build for this date, sending summary email') - emailext to: '$DEFAULT_RECIPIENTS', - subject: "$statusSymbol" + '$BRANCH_NAME regression tests - $BUILD_STATUS', - mimeType: 'text/html', - body: '${SCRIPT, template="groovy-html.template"}' - } else { - echo('Already sent summary email today, suppressing') } } } diff --git a/build.gradle b/build.gradle index e1b3594f59..2a33ecb311 100644 --- a/build.gradle +++ b/build.gradle @@ -309,7 +309,7 @@ allprojects { } tasks.withType(Test) { - forkEvery = 10 + forkEvery = 20 ignoreFailures = project.hasProperty('tests.ignoreFailures') ? project.property('tests.ignoreFailures').toBoolean() : false failFast = project.hasProperty('tests.failFast') ? project.property('tests.failFast').toBoolean() : false diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt index 83ec4a6369..adec0f92d7 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt @@ -43,6 +43,8 @@ import org.junit.* import rx.schedulers.TestScheduler import java.io.IOException import java.net.URL +import java.nio.file.FileSystem +import java.nio.file.Path import java.security.KeyPair import java.time.Instant import java.time.temporal.ChronoUnit @@ -60,11 +62,12 @@ class NetworkMapUpdaterTest { private val cacheExpiryMs = 1000 private val privateNetUUID = UUID.randomUUID() - private val fs = Jimfs.newFileSystem(unix()) - private val baseDir = fs.getPath("/node") - private val nodeInfoDir = baseDir / NODE_INFO_DIRECTORY + private lateinit var fs: FileSystem + private lateinit var baseDir: Path + private val nodeInfoDir + get() = baseDir / NODE_INFO_DIRECTORY private val scheduler = TestScheduler() - private val fileWatcher = NodeInfoWatcher(baseDir, scheduler) + private lateinit var fileWatcher: NodeInfoWatcher private val nodeReadyFuture = openFuture() private val networkMapCache = createMockNetworkMapCache() private lateinit var ourKeyPair: KeyPair @@ -80,6 +83,10 @@ class NetworkMapUpdaterTest { // register BouncyCastle and EdDSA provider separately, which wrecks havoc. Crypto.registerProviders() + fs = Jimfs.newFileSystem(unix()) + baseDir = fs.getPath("/node") + fileWatcher = NodeInfoWatcher(baseDir, scheduler) + ourKeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) ourNodeInfo = createNodeInfoAndSigned("Our info", ourKeyPair).signed server = NetworkMapServer(cacheExpiryMs.millis) diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt b/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt index c79f4a76cd..e0f447ff62 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt @@ -25,7 +25,6 @@ import net.corda.nodeapi.exceptions.DuplicateAttachmentException import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.testing.common.internal.testNetworkParameters -import net.corda.testing.core.internal.ContractJarTestUtils import net.corda.testing.core.internal.ContractJarTestUtils.makeTestContractJar import net.corda.testing.core.internal.ContractJarTestUtils.makeTestJar import net.corda.testing.core.internal.ContractJarTestUtils.makeTestSignedContractJar @@ -96,6 +95,7 @@ class NodeAttachmentServiceTest { @After fun tearDown() { database.close() + fs.close() } @Test