INFRA-330 Use Artifactory as cache for all dependencies (#6253)

* Use a virtual repo (corda-remotes) containing all Corda repositories with dependencies
* activated when CORDA_USE_CACHE environment variable is set
* Update Jenkins configuration to use new functionality
* it does *not* affect local builds as long as environment variable is not set!

.ci/dev/compatibility/JenkinsfileJDK11Azul

@@ -37,6 +37,9 @@ pipeline {
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
         ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -68,6 +71,9 @@ pipeline {
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " +
                             "-Ddocker.buildbase.tag=11latest " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.dockerfile=DockerfileJDK11Azul" +
                             " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace"
                 }
@@ -147,7 +153,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

.ci/dev/nightly-regression/Jenkinsfile

@@ -20,6 +20,9 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -36,6 +39,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean pushBuildImage --stacktrace"
                 }
@@ -75,7 +81,6 @@ pipeline {
         }
     }
 
-
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
@@ -86,4 +91,3 @@ pipeline {
         }
     }
 }
-

.ci/dev/regression/Jenkinsfile

@@ -48,6 +48,9 @@ pipeline {
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
         ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -83,6 +86,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace"
                 }
@@ -170,7 +176,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true
 
             script {
                 try {
@@ -243,4 +249,4 @@ pipeline {
             deleteDir() /* clean up our workspace */
         }
     }
-}
+}

Jenkinsfile

@@ -17,6 +17,9 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -27,6 +30,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace"
                 }
@@ -73,7 +79,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

build.gradle

@@ -155,16 +155,34 @@ buildscript {
     ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion"
     repositories {
         mavenLocal()
-        mavenCentral()
+        // Use system environment to activate caching with Artifactory,
-        jcenter()
+        // because it is actually easier to pass that during parallel build.
-        maven {
+        // NOTE: it has to be a name of a virtual repository with all
-            url 'https://kotlin.bintray.com/kotlinx'
+        // required remote or local repositories!
-        }
+        if (System.getenv("CORDA_USE_CACHE")) {
-        maven {
+            maven {
-            url "$artifactory_contextUrl/corda-dependencies-dev"
+                name "R3 Maven remote repositories"
-        }
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
-        maven {
+                authentication {
-            url "$artifactory_contextUrl/corda-releases"
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven {
+                url 'https://kotlin.bintray.com/kotlinx'
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-dependencies-dev"
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-releases"
+            }
         }
     }
     dependencies {
@@ -357,11 +375,29 @@ allprojects {
 
     repositories {
         mavenLocal()
-        mavenCentral()
+        // Use system environment to activate caching with Artifactory,
-        jcenter()
+        // because it is actually easier to pass that during parallel build.
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
+        // NOTE: it has to be a name of a virtual repository with all
-        maven { url 'https://repo.gradle.org/gradle/libs-releases' }
+        // required remote or local repositories!
-        maven { url "$artifactory_contextUrl/corda-dev" }
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+            maven { url 'https://repo.gradle.org/gradle/libs-releases' }
+            maven { url "${artifactory_contextUrl}/corda-dev" }
+        }
     }
 
     configurations {

settings.gradle

@@ -2,9 +2,27 @@ pluginManagement {
     ext.artifactory_contextUrl = 'https://software.r3.com/artifactory'
 
     repositories {
-        mavenLocal()
+        // Use system environment to activate caching with Artifactory,
-        gradlePluginPortal()
+        // because it is actually easier to pass that during parallel build.
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenLocal()
+            gradlePluginPortal()
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+        }
     }
 }
 // The project is named 'corda-project' and not 'corda' because if this is named the same as the