INFRA-1955: Merging forward from 4.10 to 4.11

.ci/dev/nightly-regression/JenkinsfileSnykScan

@@ -0,0 +1,7 @@
+@Library('corda-shared-build-pipeline-steps') _
+
+cordaSnykScanPipeline (
+    snykTokenId: 'c4-os-snyk-api-token-secret',
+    // specify the Gradle submodules to scan and monitor on snyk Server
+    modulesToScan: ['node', 'capsule', 'bridge', 'bridgecapsule']
+)

.ci/dev/regression/Jenkinsfile

@@ -5,6 +5,10 @@
  */
 @Library('corda-shared-build-pipeline-steps')
 
+import com.r3.build.utils.GitUtils
+
+GitUtils gitUtils = new GitUtils(this)
+
 /**
  * Sense environment
  */
@@ -14,34 +18,6 @@ boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
 boolean isReleaseCandidate = (env.TAG_NAME =~ /^(release-.*(RC|HC).*(?<!_JDK11))$/)
 def buildEdition = (isReleaseTag || isReleaseCandidate) ? "Corda Community Edition" : "Corda Open Source"
 
-/*
-** calculate the stage for NexusIQ evaluation
-**  * build for snapshots
-**  * stage-release:  for release candidates and for health checks
-**  * release: for GA release
-*/
-def nexusDefaultIqStage = "build"
-if (isReleaseTag) {
-    switch (env.TAG_NAME) {
-        case ~/.*-RC\d+(-.*)?/: nexusDefaultIqStage = "stage-release"; break;
-        case ~/.*-HC\d+(-.*)?/: nexusDefaultIqStage = "stage-release"; break;
-        default: nexusDefaultIqStage = "release"
-    }
-}
-
-/**
- * make sure calculated default value of NexusIQ stage is first in the list
- * thus making it default for the `choice` parameter
- */
-def nexusIqStageChoices = [nexusDefaultIqStage].plus(
-                [
-                        'develop',
-                        'build',
-                        'stage-release',
-                        'release',
-                        'operate'
-                ].minus([nexusDefaultIqStage]))
-
 /**
  * Common Gradle arguments for all Gradle executions
  */
@@ -55,7 +31,7 @@ String COMMON_GRADLE_PARAMS = [
 ].join(' ')
 
 pipeline {
-    agent { label 'standard' }
+    agent { label 'standard-latest-ami' }
 
     /*
      * List options in alphabetical order
@@ -68,7 +44,6 @@ pipeline {
     }
 
     parameters {
-        choice choices: nexusIqStageChoices, description: 'NexusIQ stage for code evaluation', name: 'nexusIqStage'
         booleanParam defaultValue: true, description: 'Run tests during this build?', name: 'DO_TEST'
     }
 
@@ -83,8 +58,9 @@ pipeline {
         CORDA_BUILD_EDITION = "${buildEdition}"
         DOCKER_URL = "https://index.docker.io/v1/"
         EMAIL_RECIPIENTS = credentials('corda4-email-recipient')
-        SNYK_API_KEY = "c4-os-snyk"
+        SNYK_API_KEY = "c4-os-snyk" //Jenkins credential type: Snyk Api token
-        CORDA_USE_CACHE = "corda-remotes"
+        SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret') //Jenkins credential type: Secret text
+        C4_OS_SNYK_ORG_ID = credentials('corda4-os-snyk-org-id')
     }
 
     stages {
@@ -108,93 +84,8 @@ pipeline {
                 stash name: 'compiled', useDefaultExcludes: false
             }
         }
-        stage('Sonatype Check') {
-            steps {
-                script {
-                    sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
-                    /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
-                    def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\(\\.[0-9]\\+\\)\\+\\).*\$/\\1/'").trim()
-                    def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim()
-                    def artifactId = 'corda'
-                    nexusAppId = "${groupId}-${artifactId}-${version}"
-                }
-                nexusPolicyEvaluation (
-                        failBuildOnNetworkError: false,
-                        iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts!
-                        iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']],
-                        iqStage: params.nexusIqStage
-                )
-            }
-        }
-        stage('Generate Wiki Report') {
-            when {
-                expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate }
-                beforeAgent true
-            }
-            agent {
-                docker {
-                    image 'nexusiq-sonatype-cli:latest'
-                    reuseNode true
-                    registryUrl 'https://engineering-docker.software.r3.com/'
-                    registryCredentialsId 'artifactory-credentials'
-                }
-            }
-            options {
-                retry(3)
-            }
-            environment {
-                NEXUS_APP_ID="${nexusAppId}"
-                NEXUS_APP_STAGE="${params.nexusIqStage}"
-                NEXUSIQ_CREDENTIALS = credentials('jenkins-nexusiq-credentials')
-            }
-            steps {
-                sh '''\
-                    rm -f wiki-report.md
-                    env NEXUSIQ_USERNAME="${NEXUSIQ_CREDENTIALS_USR}" \
-                        NEXUSIQ_PASSWORD="${NEXUSIQ_CREDENTIALS_PSW}" \
-                            /opt/app/wrapper wiki-report \
-                                --app "${NEXUS_APP_ID}" \
-                                --stage "${NEXUS_APP_STAGE}" >wiki-report.md
-                '''.stripIndent()
-                archiveArtifacts 'wiki-report.md'
-            }
-        }
-        stage('Generate Licence Report') {
-            when {
-                expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate }
-                beforeAgent true
-            }
-            agent {
-                docker {
-                    image 'nexusiq-licence-report:latest'
-                    reuseNode true
-                    registryUrl 'https://engineering-docker.software.r3.com/'
-                    registryCredentialsId 'artifactory-credentials'
-                }
-            }
-            options {
-                retry(3)
-            }
-            environment {
-                NEXUS_APP_ID="${nexusAppId}"
-                NEXUS_APP_STAGE="${params.nexusIqStage}"
-                NEXUSIQ_CREDENTIALS = credentials('jenkins-nexusiq-credentials')
-            }
-            steps {
-                sh '''\
-                    rm -rf report
-                    env NEXUSIQ_USERNAME="${NEXUSIQ_CREDENTIALS_USR}" \
-                        NEXUSIQ_PASSWORD="${NEXUSIQ_CREDENTIALS_PSW}" \
-                            /opt/app/wrapper --write --outdir report \
-                                --force \
-                                --app "${NEXUS_APP_ID}" \
-                                --stage "${NEXUS_APP_STAGE}"
-                '''.stripIndent()
-                archiveArtifacts 'report/*.md'
-            }
-        }
 
-       stage('Snyk Security') {
+        stage('Snyk Security') {
             when {
                 expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
             }
@@ -209,6 +100,22 @@ pipeline {
             }
         }
 
+        stage('Generate Snyk License Report') {
+            when {
+                expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
+            }
+            steps {
+                snykLicenseGeneration(env.SNYK_API_TOKEN, env.C4_OS_SNYK_ORG_ID)
+            }
+            post {
+                always {
+                    script {
+                        archiveArtifacts artifacts: 'snyk-license-report/*-snyk-license-report.html', allowEmptyArchive: true, fingerprint: true
+                    }
+                }
+            }
+        }
+
         stage('All Tests') {
             when {
                 expression { params.DO_TEST }
@@ -420,6 +327,9 @@ pipeline {
     post {
         always {
             script {
+                if (gitUtils.isReleaseTag()) {
+                    gitUtils.getGitLog(env.TAG_NAME, env.GIT_URL.replace('https://github.com/corda/', ''), scm.userRemoteConfigs[0].credentialsId)
+                }
                 try {
                     if (params.DO_TEST) {
                         unstash 'allure-input'
@@ -475,20 +385,23 @@ pipeline {
             }
         }
         success {
-        	script {
+            script {
-        		sendSlackNotifications("good", "BUILD PASSED", false, "#corda-corda4-open-source-build-notifications")
+                sendSlackNotifications("good", "BUILD PASSED", false, "#corda-corda4-open-source-build-notifications")
                 if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
                     snykSecurityScan.generateHtmlElements()
                 }
-        	}
+            }
         }
         unstable {
-        	script {
+            script {
-        		sendSlackNotifications("warning", "BUILD UNSTABLE - Unstable Builds are likely a result of Nexus Sonar Scanner violations", false, "#corda-corda4-open-source-build-notifications")
+                sendSlackNotifications("warning", "BUILD UNSTABLE", false, "#corda-corda4-open-source-build-notifications")
                 if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
                     snykSecurityScan.generateHtmlElements()
                 }
-        	}
+                if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
+                    snykSecurityScan.generateHtmlElements()
+                }
+            }
         }
         failure {
             script {

core/src/main/kotlin/net/corda/core/node/services/ServiceLifecycleObserver.kt

@@ -17,6 +17,18 @@ interface ServiceLifecycleObserver {
 }
 
 enum class ServiceLifecycleEvent {
+
+    /**
+     * This event is dispatched when the node is about to start the State Machine. The State Machine will not be started until
+     * all handlers return from processing this event.
+     * Handlers can delay the node start-up, and the processing of flows, by not returning from handling this event until their
+     * CorDapp is ready for flows to be started/resumed.
+     *
+     * If a handler for this event throws [CordaServiceCriticalFailureException] - this is the way to flag that it will not make
+     * sense for Corda node to continue its operation. The lifecycle events dispatcher will endeavor to terminate node's JVM as soon
+     * as practically possible.
+     */
+    BEFORE_STATE_MACHINE_START,
     /**
      * This event is dispatched when State Machine is fully started such that [net.corda.core.node.AppServiceHub] available
      * for [CordaService] to be use.
@@ -25,7 +37,6 @@ enum class ServiceLifecycleEvent {
      * sense for Corda node to continue its operation. The lifecycle events dispatcher will endeavor to terminate node's JVM as soon
      * as practically possible.
      */
-    BEFORE_STATE_MACHINE_START,
     STATE_MACHINE_STARTED,
 }
 

testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt

@@ -29,6 +29,7 @@ import rx.Observable
 import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.time.Duration
 import java.util.concurrent.atomic.AtomicInteger
 
 /**
@@ -216,7 +217,8 @@ fun <A> driver(defaultParameters: DriverParameters = DriverParameters(), dsl: Dr
                     djvmCordaSource = defaultParameters.djvmCordaSource,
                     environmentVariables = defaultParameters.environmentVariables,
                     allowHibernateToManageAppSchema = defaultParameters.allowHibernateToManageAppSchema,
-                    premigrateH2Database = defaultParameters.premigrateH2Database
+                    premigrateH2Database = defaultParameters.premigrateH2Database,
+                    notaryHandleTimeout = defaultParameters.notaryHandleTimeout
             ),
             coerce = { it },
             dsl = dsl
@@ -256,6 +258,8 @@ fun <A> driver(defaultParameters: DriverParameters = DriverParameters(), dsl: Dr
  * @property djvmBootstrapSource Location of a JAR containing the Java APIs for the DJVM to use.
  * @property djvmCordaSource Locations of JARs of user-supplied classes to execute within the DJVM sandbox.
  * @property premigrateH2Database Whether to use a prebuilt H2 database schema or start from an empty schema.
+ * @property notaryHandleTimeout Specifies how long to wait to receive a notary handle. This waiting includes waiting for
+ * the notary to start.
  * This can save time for tests which do not need to migrate from a blank schema.
  */
 @Suppress("unused")
@@ -281,7 +285,8 @@ data class DriverParameters(
         val djvmCordaSource: List<Path> = emptyList(),
         val environmentVariables: Map<String, String> = emptyMap(),
         val allowHibernateToManageAppSchema: Boolean = true,
-        val premigrateH2Database: Boolean = true
+        val premigrateH2Database: Boolean = true,
+        val notaryHandleTimeout: Duration = Duration.ofMinutes(1)
 ) {
     constructor(cordappsForAllNodes: Collection<TestCordapp>) : this(isDebug = false, cordappsForAllNodes = cordappsForAllNodes)
 
@@ -464,6 +469,51 @@ data class DriverParameters(
             cordappsForAllNodes = null
     )
 
+    constructor(
+            isDebug: Boolean,
+            driverDirectory: Path,
+            portAllocation: PortAllocation,
+            debugPortAllocation: PortAllocation,
+            systemProperties: Map<String, String>,
+            useTestClock: Boolean,
+            startNodesInProcess: Boolean,
+            waitForAllNodesToFinish: Boolean,
+            notarySpecs: List<NotarySpec>,
+            extraCordappPackagesToScan: List<String>,
+            jmxPolicy: JmxPolicy,
+            networkParameters: NetworkParameters,
+            notaryCustomOverrides: Map<String, Any?>,
+            inMemoryDB: Boolean,
+            cordappsForAllNodes: Collection<TestCordapp>?,
+            djvmBootstrapSource: Path?,
+            djvmCordaSource: List<Path>,
+            environmentVariables: Map<String, String>,
+            allowHibernateToManageAppSchema: Boolean,
+            premigrateH2Database: Boolean = true
+    ) : this(
+            isDebug,
+            driverDirectory,
+            portAllocation,
+            debugPortAllocation,
+            systemProperties,
+            useTestClock,
+            startNodesInProcess,
+            waitForAllNodesToFinish,
+            notarySpecs,
+            extraCordappPackagesToScan,
+            jmxPolicy,
+            networkParameters,
+            notaryCustomOverrides,
+            inMemoryDB,
+            cordappsForAllNodes,
+            djvmBootstrapSource,
+            djvmCordaSource,
+            environmentVariables,
+            allowHibernateToManageAppSchema,
+            premigrateH2Database,
+            notaryHandleTimeout = Duration.ofMinutes(1)
+    )
+
     fun withIsDebug(isDebug: Boolean): DriverParameters = copy(isDebug = isDebug)
     fun withDriverDirectory(driverDirectory: Path): DriverParameters = copy(driverDirectory = driverDirectory)
     fun withPortAllocation(portAllocation: PortAllocation): DriverParameters = copy(portAllocation = portAllocation)
@@ -487,6 +537,7 @@ data class DriverParameters(
     fun withDjvmCordaSource(djvmCordaSource: List<Path>): DriverParameters = copy(djvmCordaSource = djvmCordaSource)
     fun withEnvironmentVariables(variables: Map<String, String>): DriverParameters = copy(environmentVariables = variables)
     fun withAllowHibernateToManageAppSchema(value: Boolean): DriverParameters = copy(allowHibernateToManageAppSchema = value)
+    fun withNotaryHandleTimeout(value: Duration): DriverParameters = copy(notaryHandleTimeout = value)
 
     fun copy(
             isDebug: Boolean,
@@ -631,4 +682,50 @@ data class DriverParameters(
             allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
             premigrateH2Database = true
     )
+
+    @Suppress("LongParameterList")
+    fun copy(
+            isDebug: Boolean,
+            driverDirectory: Path,
+            portAllocation: PortAllocation,
+            debugPortAllocation: PortAllocation,
+            systemProperties: Map<String, String>,
+            useTestClock: Boolean,
+            startNodesInProcess: Boolean,
+            waitForAllNodesToFinish: Boolean,
+            notarySpecs: List<NotarySpec>,
+            extraCordappPackagesToScan: List<String>,
+            jmxPolicy: JmxPolicy,
+            networkParameters: NetworkParameters,
+            notaryCustomOverrides: Map<String, Any?>,
+            inMemoryDB: Boolean,
+            cordappsForAllNodes: Collection<TestCordapp>?,
+            djvmBootstrapSource: Path?,
+            djvmCordaSource: List<Path>,
+            environmentVariables: Map<String, String>,
+            allowHibernateToManageAppSchema: Boolean,
+            premigrateH2Database: Boolean
+    ) = this.copy(
+            isDebug = isDebug,
+            driverDirectory = driverDirectory,
+            portAllocation = portAllocation,
+            debugPortAllocation = debugPortAllocation,
+            systemProperties = systemProperties,
+            useTestClock = useTestClock,
+            startNodesInProcess = startNodesInProcess,
+            waitForAllNodesToFinish = waitForAllNodesToFinish,
+            notarySpecs = notarySpecs,
+            extraCordappPackagesToScan = extraCordappPackagesToScan,
+            jmxPolicy = jmxPolicy,
+            networkParameters = networkParameters,
+            notaryCustomOverrides = notaryCustomOverrides,
+            inMemoryDB = inMemoryDB,
+            cordappsForAllNodes = cordappsForAllNodes,
+            djvmBootstrapSource = djvmBootstrapSource,
+            djvmCordaSource = djvmCordaSource,
+            environmentVariables = environmentVariables,
+            allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
+            premigrateH2Database = premigrateH2Database,
+            notaryHandleTimeout = Duration.ofMinutes(1)
+    )
 }

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt

@@ -154,7 +154,8 @@ class DriverDSLImpl(
         val djvmCordaSource: List<Path>,
         val environmentVariables: Map<String, String>,
         val allowHibernateToManageAppSchema: Boolean = true,
-        val premigrateH2Database: Boolean = true
+        val premigrateH2Database: Boolean = true,
+        val notaryHandleTimeout: Duration = Duration.ofMinutes(1)
 ) : InternalDriverDSL {
 
     private var _executorService: ScheduledExecutorService? = null
@@ -854,7 +855,6 @@ class DriverDSLImpl(
         // While starting with inProcess mode, we need to have different names to avoid clashes
         private val inMemoryCounter = AtomicInteger()
 
-        private val notaryHandleTimeout = Duration.ofMinutes(1)
         private val defaultRpcUserList = listOf(InternalUser("default", "default", setOf("ALL")).toConfig().root().unwrapped())
         private val names = arrayOf(ALICE_NAME, BOB_NAME, DUMMY_BANK_A_NAME)
 
@@ -1332,7 +1332,8 @@ fun <DI : DriverDSL, D : InternalDriverDSL, A> genericDriver(
                         djvmCordaSource = defaultParameters.djvmCordaSource,
                         environmentVariables = defaultParameters.environmentVariables,
                         allowHibernateToManageAppSchema = defaultParameters.allowHibernateToManageAppSchema,
-                        premigrateH2Database = defaultParameters.premigrateH2Database
+                        premigrateH2Database = defaultParameters.premigrateH2Database,
+                        notaryHandleTimeout = defaultParameters.notaryHandleTimeout
                 )
         )
         val shutdownHook = addShutdownHook(driverDsl::shutdown)
@@ -1432,6 +1433,7 @@ fun <A> internalDriver(
         environmentVariables: Map<String, String> = emptyMap(),
         allowHibernateToManageAppSchema: Boolean = true,
         premigrateH2Database: Boolean = true,
+        notaryHandleTimeout: Duration = Duration.ofMinutes(1),
         dsl: DriverDSLImpl.() -> A
 ): A {
     return genericDriver(
@@ -1456,7 +1458,8 @@ fun <A> internalDriver(
                     djvmCordaSource = djvmCordaSource,
                     environmentVariables = environmentVariables,
                     allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
-                    premigrateH2Database = premigrateH2Database
+                    premigrateH2Database = premigrateH2Database,
+                    notaryHandleTimeout = notaryHandleTimeout
             ),
             coerce = { it },
             dsl = dsl