diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 642eb8ade9..b805ff0fd8 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,29 +1,21 @@ #!groovy /** - * Jenkins pipeline to build Corda OS release branches and tags + * Jenkins pipeline to build Corda OS release branches and tags. + * PLEASE NOTE: we DO want to run a build for each commit!!! */ -/** - * Kill already started job. - * Assume new commit takes precendence and results from previous - * unfinished builds are not required. - * This feature doesn't play well with disableConcurrentBuilds() option - */ -@Library('corda-shared-build-pipeline-steps') -import static com.r3.build.BuildControl.killAllExistingBuildsForJob - -killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) - /** * Sense environment */ boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?version-properties" /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */ @@ -91,56 +104,53 @@ pipeline { } } - stage('Deploy Nodes') { + stage('Unit Test') { steps { - sh "./gradlew --no-daemon jar deployNodes" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'test' + ].join(' ') } } - stage('Generate Build Image') { + stage('Integration Test') { steps { - withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) { - sh "./gradlew " + - "-Dkubenetize=true " + - "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + - "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace" - } - sh "kubectl auth can-i get pods" + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'integrationTest' + ].join(' ') } } - stage('Testing phase') { - parallel { - stage('Regression Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " parallelRegressionTest --stacktrace" - } - } - stage('Slow Integration Test') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " + - "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + - "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + - "-Dgit.branch=\"\${GIT_BRANCH}\" " + - "-Dgit.target.branch=\"\${GIT_BRANCH}\" " + - " allParallelSlowIntegrationTest --stacktrace" - } - } + stage('Smoke Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'smokeTest' + ].join(' ') + } + } + + stage('Slow Integration Test') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'slowIntegrationTest' + ].join(' ') + } + } + + stage('Deploy Node') { + steps { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'deployNode' + ].join(' ') } } @@ -176,14 +186,19 @@ pipeline { stage('Publish Release to Docker Hub') { when { - expression { !isInternalRelease && isReleaseTag } + expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate} } steps { withCredentials([ usernamePassword(credentialsId: 'corda-publisher-docker-hub-credentials', usernameVariable: 'DOCKER_USERNAME', - passwordVariable: 'DOCKER_PASSWORD')]) { - sh "./gradlew pushOfficialImages" + passwordVariable: 'DOCKER_PASSWORD') + ]) { + sh script: [ + './gradlew', + COMMON_GRADLE_PARAMS, + 'pushOfficialImages' + ].join(' ') } } } @@ -191,8 +206,9 @@ pipeline { post { always { - archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true + archiveArtifacts artifacts: '**/*.log', fingerprint: false + archiveArtifacts artifacts: '**/build/reports/tests/**', fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true script { try { @@ -214,8 +230,8 @@ pipeline { sourceCaptureExpression: '.*test-results-xml/.*-([\\d]+)/.*/([^/]+)$', targetNameExpression: '$1-$2')]) allure includeProperties: false, - jdk: '', - results: [[path: '**/allure-input']] + jdk: '', + results: [[path: '**/allure-input']] } catch (err) { echo("Allure report generation failed: $err") @@ -232,7 +248,7 @@ pipeline { // Comparing the dates of the previous and current builds achieves this, // i.e. we will only send an email for the first build on a given day. def prevBuildDate = new Date( - currentBuild?.previousBuild.timeInMillis ?: 0).clearTime() + currentBuild.previousBuild?.timeInMillis ?: 0).clearTime() def currentBuildDate = new Date( currentBuild.timeInMillis).clearTime() @@ -243,6 +259,8 @@ pipeline { statusSymbol = '\u2705' break; case 'UNSTABLE': + statusSymbol = '\u26A0' + break; case 'FAILURE': statusSymbol = '\u274c' break; diff --git a/build.gradle b/build.gradle index 77e6857f3d..185056b20a 100644 --- a/build.gradle +++ b/build.gradle @@ -333,7 +333,7 @@ allprojects { } tasks.withType(Test) { - forkEvery = 10 + forkEvery = 20 ignoreFailures = project.hasProperty('tests.ignoreFailures') ? project.property('tests.ignoreFailures').toBoolean() : false failFast = project.hasProperty('tests.failFast') ? project.property('tests.failFast').toBoolean() : false diff --git a/core-tests/src/test/kotlin/net/corda/coretests/indentity/PartyAndCertificateTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/indentity/PartyAndCertificateTest.kt index 54898eeaeb..9908e01521 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/indentity/PartyAndCertificateTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/indentity/PartyAndCertificateTest.kt @@ -2,6 +2,7 @@ package net.corda.coretests.indentity import com.google.common.jimfs.Configuration.unix import com.google.common.jimfs.Jimfs +import net.corda.core.crypto.Crypto import net.corda.core.crypto.entropyToKeyPair import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party @@ -14,6 +15,7 @@ import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.core.getTestPartyAndCertificate import net.corda.coretesting.internal.DEV_ROOT_CA import org.assertj.core.api.Assertions.assertThat +import org.junit.Before import org.junit.Rule import org.junit.Test import java.math.BigInteger @@ -24,6 +26,13 @@ class PartyAndCertificateTest { @JvmField val testSerialization = SerializationEnvironmentRule() + @Before + fun setUp() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + } + @Test(timeout=300_000) fun `reject a path with no roles`() { val path = X509Utilities.buildCertPath(DEV_ROOT_CA.certificate) diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt index d2689ce039..dffba81aaf 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkMapUpdaterTest.kt @@ -62,6 +62,8 @@ import org.junit.Test import rx.schedulers.TestScheduler import java.io.IOException import java.net.URL +import java.nio.file.FileSystem +import java.nio.file.Path import java.security.KeyPair import java.time.Instant import java.time.temporal.ChronoUnit @@ -79,11 +81,12 @@ class NetworkMapUpdaterTest { private val cacheExpiryMs = 1000 private val privateNetUUID = UUID.randomUUID() - private val fs = Jimfs.newFileSystem(unix()) - private val baseDir = fs.getPath("/node") - private val nodeInfoDir = baseDir / NODE_INFO_DIRECTORY + private lateinit var fs: FileSystem + private lateinit var baseDir: Path + private val nodeInfoDir + get() = baseDir / NODE_INFO_DIRECTORY private val scheduler = TestScheduler() - private val fileWatcher = NodeInfoWatcher(baseDir, scheduler) + private lateinit var fileWatcher: NodeInfoWatcher private val nodeReadyFuture = openFuture() private val networkMapCache = createMockNetworkMapCache() private lateinit var ourKeyPair: KeyPair @@ -95,6 +98,14 @@ class NetworkMapUpdaterTest { @Before fun setUp() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + + fs = Jimfs.newFileSystem(unix()) + baseDir = fs.getPath("/node") + fileWatcher = NodeInfoWatcher(baseDir, scheduler) + ourKeyPair = Crypto.generateKeyPair(X509Utilities.DEFAULT_TLS_SIGNATURE_SCHEME) ourNodeInfo = createNodeInfoAndSigned("Our info", ourKeyPair).signed server = NetworkMapServer(cacheExpiryMs.millis) diff --git a/node/src/test/kotlin/net/corda/node/services/network/NetworkParametersReaderTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NetworkParametersReaderTest.kt index f44331d296..523710e74c 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NetworkParametersReaderTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NetworkParametersReaderTest.kt @@ -2,6 +2,7 @@ package net.corda.node.services.network import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs +import net.corda.core.crypto.Crypto import net.corda.core.internal.* import net.corda.core.serialization.deserialize import net.corda.core.utilities.days @@ -29,7 +30,7 @@ class NetworkParametersReaderTest { @JvmField val testSerialization = SerializationEnvironmentRule(true) - private val fs: FileSystem = Jimfs.newFileSystem(Configuration.unix()) + private lateinit var fs: FileSystem private val cacheTimeout = 100000.seconds private lateinit var server: NetworkMapServer @@ -37,6 +38,11 @@ class NetworkParametersReaderTest { @Before fun setUp() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + + fs = Jimfs.newFileSystem(Configuration.unix()) server = NetworkMapServer(cacheTimeout) val address = server.start() networkMapClient = NetworkMapClient(URL("http://$address"), VersionInfo(1, "TEST", "TEST", "TEST")) @@ -84,4 +90,4 @@ class NetworkParametersReaderTest { val parameters = inByteArray.deserialize() assertThat(parameters.verified().eventHorizon).isEqualTo(Int.MAX_VALUE.days) } -} \ No newline at end of file +} diff --git a/node/src/test/kotlin/net/corda/node/services/network/NodeInfoWatcherTest.kt b/node/src/test/kotlin/net/corda/node/services/network/NodeInfoWatcherTest.kt index 6ec93664a3..3872dd0fcb 100644 --- a/node/src/test/kotlin/net/corda/node/services/network/NodeInfoWatcherTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/network/NodeInfoWatcherTest.kt @@ -2,6 +2,7 @@ package net.corda.node.services.network import com.google.common.jimfs.Configuration import com.google.common.jimfs.Jimfs +import net.corda.core.crypto.Crypto import net.corda.core.internal.NODE_INFO_DIRECTORY import net.corda.core.internal.createDirectories import net.corda.core.internal.div @@ -47,6 +48,10 @@ class NodeInfoWatcherTest { @Before fun start() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + nodeInfoAndSigned = createNodeInfoAndSigned(ALICE_NAME) val identityService = makeTestIdentityService() keyManagementService = MockKeyManagementService(identityService) diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt b/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt index 801c917c35..4c1e8d2bac 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt @@ -7,6 +7,7 @@ import com.google.common.jimfs.Jimfs import com.nhaarman.mockito_kotlin.doReturn import com.nhaarman.mockito_kotlin.whenever import net.corda.core.contracts.ContractAttachment +import net.corda.core.crypto.Crypto import net.corda.core.crypto.SecureHash import net.corda.core.crypto.sha256 import net.corda.core.flows.FlowLogic @@ -67,6 +68,10 @@ class NodeAttachmentServiceTest { @Before fun setUp() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + LogHelper.setLevel(PersistentUniquenessProvider::class) val dataSourceProperties = makeTestDataSourceProperties() @@ -90,6 +95,7 @@ class NodeAttachmentServiceTest { @After fun tearDown() { database.close() + fs.close() } @Test(timeout=300_000) diff --git a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt index dde5082f6b..9a21f20942 100644 --- a/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt +++ b/node/src/test/kotlin/net/corda/node/utilities/registration/NetworkRegistrationHelperTest.kt @@ -38,6 +38,7 @@ import org.junit.After import org.junit.Before import org.junit.Test import java.nio.file.Files +import java.nio.file.FileSystem import java.security.PublicKey import java.security.cert.CertPathValidatorException import java.security.cert.X509Certificate @@ -47,7 +48,7 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue class NetworkRegistrationHelperTest { - private val fs = Jimfs.newFileSystem(unix()) + private lateinit var fs: FileSystem private val nodeLegalName = ALICE_NAME private lateinit var config: NodeConfiguration @@ -56,6 +57,11 @@ class NetworkRegistrationHelperTest { @Before fun init() { + // Register providers before creating Jimfs filesystem. JimFs creates an SSHD instance which + // register BouncyCastle and EdDSA provider separately, which wrecks havoc. + Crypto.registerProviders() + + fs = Jimfs.newFileSystem(unix()) val baseDirectory = fs.getPath("/baseDir").createDirectories() abstract class AbstractNodeConfiguration : NodeConfiguration