INFRA-1299: switching from AKS to linear builds on two Jenkins agents

* removed `.ci/dev/unit/Jenkinsfile` and
`.ci/dev/integration/Jenkinsfile` - they don't seem to be used anywhere
* switched mandatory tests to linear build - modeled after Corda
  Enterprise
* switched nightly regression tests to linear build - modeled after
  Corda Enterprise

.ci/dev/integration/Jenkinsfile

@@ -1,62 +0,0 @@
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-@Library('corda-shared-build-pipeline-steps')
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-
-killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
-
-pipeline {
-    agent { label 'local-k8s' }
-    options {
-            timestamps()
-            timeout(time: 3, unit: 'HOURS')
-    }
-
-    environment {
-        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
-    }
-
-    stages {
-        stage('Corda - Generate Build Image') {
-            steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
-                    sh "./gradlew " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
-                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean pushBuildImage"
-                }
-                sh "kubectl auth can-i get pods"
-            }
-        }
-
-        stage('Corda - Run Tests') {
-            stage('Integration Tests') {
-                steps {
-                    sh "./gradlew " +
-                            "-DbuildId=\"\${BUILD_ID}\" " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " allParallelIntegrationTest"
-                    if (env.CHANGE_ID) {
-                        pullRequest.createStatus(status: 'success',
-                                context: 'continuous-integration/jenkins/pr-merge/integrationTest',
-                                description: 'Integration Tests Passed',
-                                targetUrl: "${env.JOB_URL}/testResults")
-                    }
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            junit '**/build/test-results-xml/**/*.xml'
-        }
-        cleanup {
-            deleteDir() /* clean up our workspace */
-        }
-    }
-}

.ci/dev/nightly-regression/Jenkinsfile

@@ -1,66 +1,164 @@
+#!groovy
+/**
+ * Jenkins pipeline to build Corda Opensource nightly regression for feature branches.
+ */
+
 @Library('corda-shared-build-pipeline-steps')
 import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
+/**
+ * Common Gradle arguments for all Gradle executions
+ */
+String COMMON_GRADLE_PARAMS = [
+        '--no-daemon',
+        '--stacktrace',
+        '--info',
+        '-Pcompilation.warningsAsErrors=false',
+        '-Ptests.failFast=true',
+].join(' ')
+
 pipeline {
-    agent { label 'k8s' }
+    agent { label 'standard' }
+
+    /*
+     * List options in alphabetical order
+     */
     options {
-        timestamps()
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
         overrideIndexTriggers(false)
-        buildDiscarder(logRotator(daysToKeepStr: '7', artifactDaysToKeepStr: '7'))
+        parallelsAlwaysFailFast()
-        timeout(time: 3, unit: 'HOURS')
+        timeout(time: 6, unit: 'HOURS')
+        timestamps()
     }
+
     triggers {
         pollSCM ignorePostCommitHooks: true, scmpoll_spec: '@midnight'
     }
 
+    /*
+     * List environment variables in alphabetical order
+     */
     environment {
-        DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
     }
 
     stages {
-        stage('Generate Build Image') {
+        stage('Compile') {
             steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                sh script: [
-                    sh "./gradlew " +
+                        './gradlew',
-                            "-Dkubenetize=true " +
+                        COMMON_GRADLE_PARAMS,
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                        'clean',
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                        'jar'
-                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                ].join(' ')
-                            " clean pushBuildImage --stacktrace"
-                }
-                sh "kubectl auth can-i get pods"
             }
         }
 
-        stage('Regression Test') {
+        stage('Stash') {
             steps {
-                sh "./gradlew " +
+                stash name: 'compiled', useDefaultExcludes: false
-                        "-DbuildId=\"\${BUILD_ID}\" " +
-                        "-Dkubenetize=true " +
-                        "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
-                        "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
-                        "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
-                        "-Dgit.branch=\"\${GIT_BRANCH}\" " +
-                        "-Dgit.target.branch=\"\${GIT_BRANCH}\" " +
-                        " parallelRegressionTest --stacktrace"
-            }
             }
         }
 
-
+        stage('All Tests') {
+            parallel {
+                stage('Another agent') {
+                    agent {
+                        label 'standard'
+                    }
+                    options {
+                        skipDefaultCheckout true
+                    }
                     post {
                         always {
-            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
+                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml'
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
                         }
                         cleanup {
                             deleteDir() /* clean up our workspace */
                         }
                     }
-}
+                    stages {
+                        stage('Unstash') {
+                            steps {
+                                unstash 'compiled'
+                            }
+                        }
+                        stage('Recompile') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'jar'
+                                ].join(' ')
+                            }
+                        }
+                        stage('Unit Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'test'
+                                ].join(' ')
+                            }
+                        }
+                        stage('Proxy Integration Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'proxyIntegrationTest'
+                                ].join(' ')
+                            }
+                        }
+                        stage('Smoke Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'smokeTest'
+                                ].join(' ')
+                            }
+                        }
+                        stage('Slow Integration Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'slowIntegrationTest'
+                                ].join(' ')
+                            }
+                        }
+                    }
+                }
+                stage('Same agent') {
+                    post {
+                        always {
+                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                        }
+                    }
+                    stages {
+                        stage('Integration Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'integrationTest'
+                                ].join(' ')
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
 
+    post {
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

.ci/dev/pr-code-checks/Jenkinsfile

@@ -4,27 +4,16 @@ import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
 pipeline {
-    agent { label 'k8s' }
+    agent { label 'standard' }
     options {
         timestamps()
         timeout(time: 3, unit: 'HOURS')
         buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
     }
 
-    environment {
-        PR_CONTEXT_STRING = "PR Code Checks"
-    }
-
     stages {
         stage('Detekt check') {
             steps {
-                script {
-                    pullRequest.createStatus(
-                            status: 'pending',
-                            context: "${PR_CONTEXT_STRING}",
-                            description: "Running code checks",
-                            targetUrl: "${env.BUILD_URL}")
-                }
                 sh "./gradlew --no-daemon clean detekt"
             }
         }
@@ -50,25 +39,6 @@ pipeline {
     }
 
     post {
-        success {
-            script {
-                pullRequest.createStatus(
-                        status: 'success',
-                        context: "${PR_CONTEXT_STRING}",
-                        description: 'Code checks passed',
-                        targetUrl: "${env.BUILD_URL}")
-            }
-        }
-
-        failure {
-            script {
-                pullRequest.createStatus(
-                        status: 'failure',
-                        context: "${PR_CONTEXT_STRING}",
-                        description: 'Code checks failed',
-                        targetUrl: "${env.BUILD_URL}")
-            }
-        }
         cleanup {
             deleteDir() /* clean up our workspace */
         }

.ci/dev/unit/Jenkinsfile

@@ -1,60 +0,0 @@
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-@Library('corda-shared-build-pipeline-steps')
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
-
-killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
-
-pipeline {
-    agent { label 'k8s' }
-    options {
-            timestamps()
-            timeout(time: 3, unit: 'HOURS')
-        }
-
-    environment {
-        DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
-    }
-
-    stages {
-        stage('Corda Pull Request - Generate Build Image') {
-            steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
-                    sh "./gradlew " +
-                            "-Dkubenetize=true " +
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
-                            "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                            " clean pushBuildImage"
-                }
-                sh "kubectl auth can-i get pods"
-            }
-        }
-
-        stage('Unit Tests') {
-            steps {
-                sh "./gradlew " +
-                        "-DbuildId=\"\${BUILD_ID}\" " +
-                        "-Dkubenetize=true " +
-                        "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
-                        " allParallelUnitTest"
-                if (env.CHANGE_ID) {
-                    pullRequest.createStatus(status: 'success',
-                            context: 'continuous-integration/jenkins/pr-merge/unitTest',
-                            description: 'Unit Tests Passed',
-                            targetUrl: "${env.JOB_URL}/testResults")
-                }
-            }
-        }
-    }
-
-    post {
-        always {
-            junit '**/build/test-results-xml/**/*.xml'
-        }
-        cleanup {
-            deleteDir() /* clean up our workspace */
-        }
-    }
-}

Jenkinsfile

@@ -1,64 +1,124 @@
-import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+#!groovy
+/**
+ * Jenkins pipeline to build Corda Opensource Pull Requests.
+ */
+
 @Library('corda-shared-build-pipeline-steps')
 import static com.r3.build.BuildControl.killAllExistingBuildsForJob
 
 killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 
+/**
+ * Common Gradle arguments for all Gradle executions
+ */
+String COMMON_GRADLE_PARAMS = [
+        '--no-daemon',
+        '--stacktrace',
+        '--info',
+        '-Pcompilation.warningsAsErrors=false',
+        '-Ptests.failFast=true',
+].join(' ')
+
 pipeline {
-    agent { label 'k8s' }
+    agent { label 'standard' }
+
+    /*
+     * List options in alphabetical order
+     */
     options {
+        ansiColor('xterm')
+        buildDiscarder(logRotator(daysToKeepStr: '14', artifactDaysToKeepStr: '14'))
+        parallelsAlwaysFailFast()
+        timeout(time: 6, unit: 'HOURS')
         timestamps()
-            timeout(time: 3, unit: 'HOURS')
     }
 
+    /*
+     * List environment variables in alphabetical order
+     */
     environment {
-        DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}"
-        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
-        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
     }
 
     stages {
-        stage('Corda Pull Request - Generate Build Image') {
+        stage('Compile') {
             steps {
-                withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
+                sh script: [
-                    sh "./gradlew " +
+                        './gradlew',
-                            "-Dkubenetize=true " +
+                        COMMON_GRADLE_PARAMS,
-                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
+                        'clean',
-                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                        'jar'
-                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
+                ].join(' ')
-                            " clean pushBuildImage preAllocateForAllParallelIntegrationTest preAllocateForAllParallelUnitTest --stacktrace"
-                }
-                sh "kubectl auth can-i get pods"
             }
         }
 
-        stage('Corda Pull Request - Run Tests') {
+        stage('Stash') {
+            steps {
+                stash name: 'compiled', useDefaultExcludes: false
+            }
+        }
+
+        stage('All Tests') {
             parallel {
-                stage('Integration Tests') {
+                stage('Another agent') {
-                    steps {
+                    agent {
-                        sh "./gradlew " +
+                        label 'standard'
-                                "-DbuildId=\"\${BUILD_ID}\" " +
+                    }
-                                "-Dkubenetize=true " +
+                    options {
-                                "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
+                        skipDefaultCheckout true
-                                "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                    }
-                                "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
+                    post {
-                                "-Dgit.branch=\"\${GIT_BRANCH}\" " +
+                        always {
-                                "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " +
+                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
-                                " deAllocateForAllParallelIntegrationTest allParallelIntegrationTest  --stacktrace"
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                        }
+                        cleanup {
+                            deleteDir() /* clean up our workspace */
                         }
                     }
-                stage('Unit Tests') {
+                    stages {
+                        stage('Unstash') {
                             steps {
-                        sh "./gradlew " +
+                                unstash 'compiled'
-                                "-DbuildId=\"\${BUILD_ID}\" " +
+                            }
-                                "-Dkubenetize=true " +
+                        }
-                                "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\" " +
+                        stage('Recompile') {
-                                "-Dartifactory.username=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            steps {
-                                "-Dartifactory.password=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
+                                sh script: [
-                                "-Dgit.branch=\"\${GIT_BRANCH}\" " +
+                                        './gradlew',
-                                "-Dgit.target.branch=\"\${CHANGE_TARGET}\" " +
+                                        COMMON_GRADLE_PARAMS,
-                                " deAllocateForAllParallelUnitTest allParallelUnitTest --stacktrace"
+                                        'jar'
+                                ].join(' ')
+                            }
+                        }
+                        stage('Unit Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'test'
+                                ].join(' ')
+                            }
+                        }
+                    }
+                }
+                stage('Same agent') {
+                    post {
+                        always {
+                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                        }
+                    }
+                    stages {
+                        stage('Integration Test') {
+                            steps {
+                                sh script: [
+                                        './gradlew',
+                                        COMMON_GRADLE_PARAMS,
+                                        'integrationTest'
+                                ].join(' ')
+                            }
+                        }
                     }
                 }
             }
@@ -66,10 +126,6 @@ pipeline {
     }
 
     post {
-        always {
-            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
-        }
         cleanup {
             deleteDir() /* clean up our workspace */
         }