INFRA-1955: Merging forward from 4.10 to 4.11

2024-12-19 04:57:58 +00:00 · 2023-03-22 11:24:31 +00:00 · 2023-03-22 11:24:31 +00:00 · fda5cd07d2
commit fda5cd07d2
parent d2900d54ab 96eb683d89
5 changed files with 162 additions and 131 deletions
--- a/.ci/dev/nightly-regression/JenkinsfileSnykScan
+++ b/.ci/dev/nightly-regression/JenkinsfileSnykScan
@ -0,0 +1,7 @@
+@Library('corda-shared-build-pipeline-steps') _
+
+cordaSnykScanPipeline (
+    snykTokenId: 'c4-os-snyk-api-token-secret',
+    // specify the Gradle submodules to scan and monitor on snyk Server
+    modulesToScan: ['node', 'capsule', 'bridge', 'bridgecapsule']
+)
--- a/.ci/dev/regression/Jenkinsfile
+++ b/.ci/dev/regression/Jenkinsfile
@ -5,6 +5,10 @@
 */
@Library('corda-shared-build-pipeline-steps')

+import com.r3.build.utils.GitUtils
+
+GitUtils gitUtils = new GitUtils(this)
+
 /**
 * Sense environment
 */
@ -14,34 +18,6 @@ boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
 boolean isReleaseCandidate = (env.TAG_NAME =~ /^(release-.*(RC|HC).*(?<!_JDK11))$/)
 def buildEdition = (isReleaseTag || isReleaseCandidate) ? "Corda Community Edition" : "Corda Open Source"

-/*
-** calculate the stage for NexusIQ evaluation
-**  * build for snapshots
-**  * stage-release:  for release candidates and for health checks
-**  * release: for GA release
-*/
-def nexusDefaultIqStage = "build"
-if (isReleaseTag) {
-    switch (env.TAG_NAME) {
-        case ~/.*-RC\d+(-.*)?/: nexusDefaultIqStage = "stage-release"; break;
-        case ~/.*-HC\d+(-.*)?/: nexusDefaultIqStage = "stage-release"; break;
-        default: nexusDefaultIqStage = "release"
-    }
-}
-
-/**
- * make sure calculated default value of NexusIQ stage is first in the list
- * thus making it default for the `choice` parameter
- */
-def nexusIqStageChoices = [nexusDefaultIqStage].plus(
-                [
-                        'develop',
-                        'build',
-                        'stage-release',
-                        'release',
-                        'operate'
-                ].minus([nexusDefaultIqStage]))
-
 /**
 * Common Gradle arguments for all Gradle executions
 */
@ -55,7 +31,7 @@ String COMMON_GRADLE_PARAMS = [
 ].join(' ')

 pipeline {
-    agent { label 'standard' }
+    agent { label 'standard-latest-ami' }

    /*
     * List options in alphabetical order
@ -68,7 +44,6 @@ pipeline {
    }

    parameters {
-        choice choices: nexusIqStageChoices, description: 'NexusIQ stage for code evaluation', name: 'nexusIqStage'
        booleanParam defaultValue: true, description: 'Run tests during this build?', name: 'DO_TEST'
    }

@ -83,8 +58,9 @@ pipeline {
        CORDA_BUILD_EDITION = "${buildEdition}"
        DOCKER_URL = "https://index.docker.io/v1/"
        EMAIL_RECIPIENTS = credentials('corda4-email-recipient')
-        SNYK_API_KEY = "c4-os-snyk"
-        CORDA_USE_CACHE = "corda-remotes"
+        SNYK_API_KEY = "c4-os-snyk" //Jenkins credential type: Snyk Api token
+        SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret') //Jenkins credential type: Secret text
+        C4_OS_SNYK_ORG_ID = credentials('corda4-os-snyk-org-id')
    }

    stages {
@ -108,93 +84,8 @@ pipeline {
                stash name: 'compiled', useDefaultExcludes: false
            }
        }
-        stage('Sonatype Check') {
-            steps {
-                script {
-                    sh "./gradlew --no-daemon properties | grep -E '^(version|group):' >version-properties"
-                    /* every build related to Corda X.Y (GA, RC, HC, patch or snapshot) uses the same NexusIQ application */
-                    def version = sh (returnStdout: true, script: "grep ^version: version-properties | sed -e 's/^version: \\([0-9]\\+\\(\\.[0-9]\\+\\)\\+\\).*\$/\\1/'").trim()
-                    def groupId = sh (returnStdout: true, script: "grep ^group: version-properties | sed -e 's/^group: //'").trim()
-                    def artifactId = 'corda'
-                    nexusAppId = "${groupId}-${artifactId}-${version}"
-                }
-                nexusPolicyEvaluation (
-                        failBuildOnNetworkError: false,
-                        iqApplication: selectedApplication(nexusAppId), // application *has* to exist before a build starts!
-                        iqScanPatterns: [[scanPattern: 'node/capsule/build/libs/corda*.jar']],
-                        iqStage: params.nexusIqStage
-                )
-            }
-        }
-        stage('Generate Wiki Report') {
-            when {
-                expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate }
-                beforeAgent true
-            }
-            agent {
-                docker {
-                    image 'nexusiq-sonatype-cli:latest'
-                    reuseNode true
-                    registryUrl 'https://engineering-docker.software.r3.com/'
-                    registryCredentialsId 'artifactory-credentials'
-                }
-            }
-            options {
-                retry(3)
-            }
-            environment {
-                NEXUS_APP_ID="${nexusAppId}"
-                NEXUS_APP_STAGE="${params.nexusIqStage}"
-                NEXUSIQ_CREDENTIALS = credentials('jenkins-nexusiq-credentials')
-            }
-            steps {
-                sh '''\
-                    rm -f wiki-report.md
-                    env NEXUSIQ_USERNAME="${NEXUSIQ_CREDENTIALS_USR}" \
-                        NEXUSIQ_PASSWORD="${NEXUSIQ_CREDENTIALS_PSW}" \
-                            /opt/app/wrapper wiki-report \
-                                --app "${NEXUS_APP_ID}" \
-                                --stage "${NEXUS_APP_STAGE}" >wiki-report.md
-                '''.stripIndent()
-                archiveArtifacts 'wiki-report.md'
-            }
-        }
-        stage('Generate Licence Report') {
-            when {
-                expression { isReleaseTag && !isInternalRelease && !isReleaseCandidate }
-                beforeAgent true
-            }
-            agent {
-                docker {
-                    image 'nexusiq-licence-report:latest'
-                    reuseNode true
-                    registryUrl 'https://engineering-docker.software.r3.com/'
-                    registryCredentialsId 'artifactory-credentials'
-                }
-            }
-            options {
-                retry(3)
-            }
-            environment {
-                NEXUS_APP_ID="${nexusAppId}"
-                NEXUS_APP_STAGE="${params.nexusIqStage}"
-                NEXUSIQ_CREDENTIALS = credentials('jenkins-nexusiq-credentials')
-            }
-            steps {
-                sh '''\
-                    rm -rf report
-                    env NEXUSIQ_USERNAME="${NEXUSIQ_CREDENTIALS_USR}" \
-                        NEXUSIQ_PASSWORD="${NEXUSIQ_CREDENTIALS_PSW}" \
-                            /opt/app/wrapper --write --outdir report \
-                                --force \
-                                --app "${NEXUS_APP_ID}" \
-                                --stage "${NEXUS_APP_STAGE}"
-                '''.stripIndent()
-                archiveArtifacts 'report/*.md'
-            }
-        }

-       stage('Snyk Security') {
+        stage('Snyk Security') {
            when {
                expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
            }
@ -209,6 +100,22 @@ pipeline {
            }
        }

+        stage('Generate Snyk License Report') {
+            when {
+                expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
+            }
+            steps {
+                snykLicenseGeneration(env.SNYK_API_TOKEN, env.C4_OS_SNYK_ORG_ID)
+            }
+            post {
+                always {
+                    script {
+                        archiveArtifacts artifacts: 'snyk-license-report/*-snyk-license-report.html', allowEmptyArchive: true, fingerprint: true
+                    }
+                }
+            }
+        }
+
        stage('All Tests') {
            when {
                expression { params.DO_TEST }
@ -420,6 +327,9 @@ pipeline {
    post {
        always {
            script {
+                if (gitUtils.isReleaseTag()) {
+                    gitUtils.getGitLog(env.TAG_NAME, env.GIT_URL.replace('https://github.com/corda/', ''), scm.userRemoteConfigs[0].credentialsId)
+                }
                try {
                    if (params.DO_TEST) {
                        unstash 'allure-input'
@ -475,20 +385,23 @@ pipeline {
            }
        }
        success {
-        	script {
-        		sendSlackNotifications("good", "BUILD PASSED", false, "#corda-corda4-open-source-build-notifications")
+            script {
+                sendSlackNotifications("good", "BUILD PASSED", false, "#corda-corda4-open-source-build-notifications")
                if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
                    snykSecurityScan.generateHtmlElements()
                }
-        	}
+            }
        }
        unstable {
-        	script {
-        		sendSlackNotifications("warning", "BUILD UNSTABLE - Unstable Builds are likely a result of Nexus Sonar Scanner violations", false, "#corda-corda4-open-source-build-notifications")
+            script {
+                sendSlackNotifications("warning", "BUILD UNSTABLE", false, "#corda-corda4-open-source-build-notifications")
                if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
                    snykSecurityScan.generateHtmlElements()
                }
-        	}
+                if (isReleaseTag || isReleaseCandidate || isReleaseBranch) {
+                    snykSecurityScan.generateHtmlElements()
+                }
+            }
        }
        failure {
            script {
--- a/core/src/main/kotlin/net/corda/core/node/services/ServiceLifecycleObserver.kt
+++ b/core/src/main/kotlin/net/corda/core/node/services/ServiceLifecycleObserver.kt
@ -17,6 +17,18 @@ interface ServiceLifecycleObserver {
 }

 enum class ServiceLifecycleEvent {
+
+    /**
+     * This event is dispatched when the node is about to start the State Machine. The State Machine will not be started until
+     * all handlers return from processing this event.
+     * Handlers can delay the node start-up, and the processing of flows, by not returning from handling this event until their
+     * CorDapp is ready for flows to be started/resumed.
+     *
+     * If a handler for this event throws [CordaServiceCriticalFailureException] - this is the way to flag that it will not make
+     * sense for Corda node to continue its operation. The lifecycle events dispatcher will endeavor to terminate node's JVM as soon
+     * as practically possible.
+     */
+    BEFORE_STATE_MACHINE_START,
    /**
     * This event is dispatched when State Machine is fully started such that [net.corda.core.node.AppServiceHub] available
     * for [CordaService] to be use.
@ -25,7 +37,6 @@ enum class ServiceLifecycleEvent {
     * sense for Corda node to continue its operation. The lifecycle events dispatcher will endeavor to terminate node's JVM as soon
     * as practically possible.
     */
-    BEFORE_STATE_MACHINE_START,
    STATE_MACHINE_STARTED,
 }

--- a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt
+++ b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt
@ -29,6 +29,7 @@ import rx.Observable
 import java.io.File
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.time.Duration
 import java.util.concurrent.atomic.AtomicInteger

 /**
@ -216,7 +217,8 @@ fun <A> driver(defaultParameters: DriverParameters = DriverParameters(), dsl: Dr
                    djvmCordaSource = defaultParameters.djvmCordaSource,
                    environmentVariables = defaultParameters.environmentVariables,
                    allowHibernateToManageAppSchema = defaultParameters.allowHibernateToManageAppSchema,
-                    premigrateH2Database = defaultParameters.premigrateH2Database
+                    premigrateH2Database = defaultParameters.premigrateH2Database,
+                    notaryHandleTimeout = defaultParameters.notaryHandleTimeout
            ),
            coerce = { it },
            dsl = dsl
@ -256,6 +258,8 @@ fun <A> driver(defaultParameters: DriverParameters = DriverParameters(), dsl: Dr
 * @property djvmBootstrapSource Location of a JAR containing the Java APIs for the DJVM to use.
 * @property djvmCordaSource Locations of JARs of user-supplied classes to execute within the DJVM sandbox.
 * @property premigrateH2Database Whether to use a prebuilt H2 database schema or start from an empty schema.
+ * @property notaryHandleTimeout Specifies how long to wait to receive a notary handle. This waiting includes waiting for
+ * the notary to start.
 * This can save time for tests which do not need to migrate from a blank schema.
 */
@Suppress("unused")
@ -281,7 +285,8 @@ data class DriverParameters(
        val djvmCordaSource: List<Path> = emptyList(),
        val environmentVariables: Map<String, String> = emptyMap(),
        val allowHibernateToManageAppSchema: Boolean = true,
-        val premigrateH2Database: Boolean = true
+        val premigrateH2Database: Boolean = true,
+        val notaryHandleTimeout: Duration = Duration.ofMinutes(1)
 ) {
    constructor(cordappsForAllNodes: Collection<TestCordapp>) : this(isDebug = false, cordappsForAllNodes = cordappsForAllNodes)

@ -464,6 +469,51 @@ data class DriverParameters(
            cordappsForAllNodes = null
    )

+    constructor(
+            isDebug: Boolean,
+            driverDirectory: Path,
+            portAllocation: PortAllocation,
+            debugPortAllocation: PortAllocation,
+            systemProperties: Map<String, String>,
+            useTestClock: Boolean,
+            startNodesInProcess: Boolean,
+            waitForAllNodesToFinish: Boolean,
+            notarySpecs: List<NotarySpec>,
+            extraCordappPackagesToScan: List<String>,
+            jmxPolicy: JmxPolicy,
+            networkParameters: NetworkParameters,
+            notaryCustomOverrides: Map<String, Any?>,
+            inMemoryDB: Boolean,
+            cordappsForAllNodes: Collection<TestCordapp>?,
+            djvmBootstrapSource: Path?,
+            djvmCordaSource: List<Path>,
+            environmentVariables: Map<String, String>,
+            allowHibernateToManageAppSchema: Boolean,
+            premigrateH2Database: Boolean = true
+    ) : this(
+            isDebug,
+            driverDirectory,
+            portAllocation,
+            debugPortAllocation,
+            systemProperties,
+            useTestClock,
+            startNodesInProcess,
+            waitForAllNodesToFinish,
+            notarySpecs,
+            extraCordappPackagesToScan,
+            jmxPolicy,
+            networkParameters,
+            notaryCustomOverrides,
+            inMemoryDB,
+            cordappsForAllNodes,
+            djvmBootstrapSource,
+            djvmCordaSource,
+            environmentVariables,
+            allowHibernateToManageAppSchema,
+            premigrateH2Database,
+            notaryHandleTimeout = Duration.ofMinutes(1)
+    )
+
    fun withIsDebug(isDebug: Boolean): DriverParameters = copy(isDebug = isDebug)
    fun withDriverDirectory(driverDirectory: Path): DriverParameters = copy(driverDirectory = driverDirectory)
    fun withPortAllocation(portAllocation: PortAllocation): DriverParameters = copy(portAllocation = portAllocation)
@ -487,6 +537,7 @@ data class DriverParameters(
    fun withDjvmCordaSource(djvmCordaSource: List<Path>): DriverParameters = copy(djvmCordaSource = djvmCordaSource)
    fun withEnvironmentVariables(variables: Map<String, String>): DriverParameters = copy(environmentVariables = variables)
    fun withAllowHibernateToManageAppSchema(value: Boolean): DriverParameters = copy(allowHibernateToManageAppSchema = value)
+    fun withNotaryHandleTimeout(value: Duration): DriverParameters = copy(notaryHandleTimeout = value)

    fun copy(
            isDebug: Boolean,
@ -631,4 +682,50 @@ data class DriverParameters(
            allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
            premigrateH2Database = true
    )
+
+    @Suppress("LongParameterList")
+    fun copy(
+            isDebug: Boolean,
+            driverDirectory: Path,
+            portAllocation: PortAllocation,
+            debugPortAllocation: PortAllocation,
+            systemProperties: Map<String, String>,
+            useTestClock: Boolean,
+            startNodesInProcess: Boolean,
+            waitForAllNodesToFinish: Boolean,
+            notarySpecs: List<NotarySpec>,
+            extraCordappPackagesToScan: List<String>,
+            jmxPolicy: JmxPolicy,
+            networkParameters: NetworkParameters,
+            notaryCustomOverrides: Map<String, Any?>,
+            inMemoryDB: Boolean,
+            cordappsForAllNodes: Collection<TestCordapp>?,
+            djvmBootstrapSource: Path?,
+            djvmCordaSource: List<Path>,
+            environmentVariables: Map<String, String>,
+            allowHibernateToManageAppSchema: Boolean,
+            premigrateH2Database: Boolean
+    ) = this.copy(
+            isDebug = isDebug,
+            driverDirectory = driverDirectory,
+            portAllocation = portAllocation,
+            debugPortAllocation = debugPortAllocation,
+            systemProperties = systemProperties,
+            useTestClock = useTestClock,
+            startNodesInProcess = startNodesInProcess,
+            waitForAllNodesToFinish = waitForAllNodesToFinish,
+            notarySpecs = notarySpecs,
+            extraCordappPackagesToScan = extraCordappPackagesToScan,
+            jmxPolicy = jmxPolicy,
+            networkParameters = networkParameters,
+            notaryCustomOverrides = notaryCustomOverrides,
+            inMemoryDB = inMemoryDB,
+            cordappsForAllNodes = cordappsForAllNodes,
+            djvmBootstrapSource = djvmBootstrapSource,
+            djvmCordaSource = djvmCordaSource,
+            environmentVariables = environmentVariables,
+            allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
+            premigrateH2Database = premigrateH2Database,
+            notaryHandleTimeout = Duration.ofMinutes(1)
+    )
 }
--- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt
+++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt
@ -154,7 +154,8 @@ class DriverDSLImpl(
        val djvmCordaSource: List<Path>,
        val environmentVariables: Map<String, String>,
        val allowHibernateToManageAppSchema: Boolean = true,
-        val premigrateH2Database: Boolean = true
+        val premigrateH2Database: Boolean = true,
+        val notaryHandleTimeout: Duration = Duration.ofMinutes(1)
 ) : InternalDriverDSL {

    private var _executorService: ScheduledExecutorService? = null
@ -854,7 +855,6 @@ class DriverDSLImpl(
        // While starting with inProcess mode, we need to have different names to avoid clashes
        private val inMemoryCounter = AtomicInteger()

-        private val notaryHandleTimeout = Duration.ofMinutes(1)
        private val defaultRpcUserList = listOf(InternalUser("default", "default", setOf("ALL")).toConfig().root().unwrapped())
        private val names = arrayOf(ALICE_NAME, BOB_NAME, DUMMY_BANK_A_NAME)

@ -1332,7 +1332,8 @@ fun <DI : DriverDSL, D : InternalDriverDSL, A> genericDriver(
                        djvmCordaSource = defaultParameters.djvmCordaSource,
                        environmentVariables = defaultParameters.environmentVariables,
                        allowHibernateToManageAppSchema = defaultParameters.allowHibernateToManageAppSchema,
-                        premigrateH2Database = defaultParameters.premigrateH2Database
+                        premigrateH2Database = defaultParameters.premigrateH2Database,
+                        notaryHandleTimeout = defaultParameters.notaryHandleTimeout
                )
        )
        val shutdownHook = addShutdownHook(driverDsl::shutdown)
@ -1432,6 +1433,7 @@ fun <A> internalDriver(
        environmentVariables: Map<String, String> = emptyMap(),
        allowHibernateToManageAppSchema: Boolean = true,
        premigrateH2Database: Boolean = true,
+        notaryHandleTimeout: Duration = Duration.ofMinutes(1),
        dsl: DriverDSLImpl.() -> A
 ): A {
    return genericDriver(
@ -1456,7 +1458,8 @@ fun <A> internalDriver(
                    djvmCordaSource = djvmCordaSource,
                    environmentVariables = environmentVariables,
                    allowHibernateToManageAppSchema = allowHibernateToManageAppSchema,
-                    premigrateH2Database = premigrateH2Database
+                    premigrateH2Database = premigrateH2Database,
+                    notaryHandleTimeout = notaryHandleTimeout
            ),
            coerce = { it },
            dsl = dsl