From 309d2ec69f9e1884b0280ca7fb46743cce2188e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Waldemar=20=C5=BBurowski?= <45210402+wzur-r3@users.noreply.github.com> Date: Fri, 17 Jul 2020 09:39:45 +0100 Subject: [PATCH] INFRA-330 Use Artifactory as cache for all dependencies (#6253) * Use a virtual repo (corda-remotes) containing all Corda repositories with dependencies * activated when CORDA_USE_CACHE environment variable is set * Update Jenkins configuration to use new functionality * it does *not* affect local builds as long as environment variable is not set! --- .ci/dev/compatibility/JenkinsfileJDK11Azul | 8 ++- .ci/dev/nightly-regression/Jenkinsfile | 8 ++- .ci/dev/regression/Jenkinsfile | 10 +++- Jenkinsfile | 8 ++- build.gradle | 66 +++++++++++++++++----- settings.gradle | 24 +++++++- 6 files changed, 100 insertions(+), 24 deletions(-) diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Azul b/.ci/dev/compatibility/JenkinsfileJDK11Azul index 4f04153ded..d24a3f7ac4 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Azul +++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul @@ -37,6 +37,9 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::") + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -68,6 +71,9 @@ pipeline { "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " + "-Ddocker.buildbase.tag=11latest " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.dockerfile=DockerfileJDK11Azul" + " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace" } @@ -147,7 +153,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit '**/build/test-results-xml/**/*.xml' + junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/.ci/dev/nightly-regression/Jenkinsfile b/.ci/dev/nightly-regression/Jenkinsfile index 5c2af45b99..c8d5b0e73a 100644 --- a/.ci/dev/nightly-regression/Jenkinsfile +++ b/.ci/dev/nightly-regression/Jenkinsfile @@ -20,6 +20,9 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -36,6 +39,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean pushBuildImage --stacktrace" } @@ -75,7 +81,6 @@ pipeline { } } - post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false @@ -86,4 +91,3 @@ pipeline { } } } - diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 66af41226a..77aad30eeb 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -48,6 +48,9 @@ pipeline { BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::") + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -83,6 +86,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace" } @@ -170,7 +176,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true script { try { @@ -243,4 +249,4 @@ pipeline { deleteDir() /* clean up our workspace */ } } -} \ No newline at end of file +} diff --git a/Jenkinsfile b/Jenkinsfile index fc2c701c29..c27f461148 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,6 +17,9 @@ pipeline { EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + CORDA_USE_CACHE = "corda-remotes" + CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" } stages { @@ -27,6 +30,9 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + + "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " + + "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace" } @@ -73,7 +79,7 @@ pipeline { post { always { archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false - junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true + junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ diff --git a/build.gradle b/build.gradle index e7d5dd3c84..4ecd0a9f8f 100644 --- a/build.gradle +++ b/build.gradle @@ -155,16 +155,34 @@ buildscript { ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion" repositories { mavenLocal() - mavenCentral() - jcenter() - maven { - url 'https://kotlin.bintray.com/kotlinx' - } - maven { - url "$artifactory_contextUrl/corda-dependencies-dev" - } - maven { - url "$artifactory_contextUrl/corda-releases" + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { + url 'https://kotlin.bintray.com/kotlinx' + } + maven { + url "${artifactory_contextUrl}/corda-dependencies-dev" + } + maven { + url "${artifactory_contextUrl}/corda-releases" + } } } dependencies { @@ -357,11 +375,29 @@ allprojects { repositories { mavenLocal() - mavenCentral() - jcenter() - maven { url "$artifactory_contextUrl/corda-dependencies" } - maven { url 'https://repo.gradle.org/gradle/libs-releases' } - maven { url "$artifactory_contextUrl/corda-dev" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenCentral() + jcenter() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + maven { url 'https://repo.gradle.org/gradle/libs-releases' } + maven { url "${artifactory_contextUrl}/corda-dev" } + } } configurations { diff --git a/settings.gradle b/settings.gradle index ae6dad0838..05bb5040fc 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,9 +2,27 @@ pluginManagement { ext.artifactory_contextUrl = 'https://software.r3.com/artifactory' repositories { - mavenLocal() - gradlePluginPortal() - maven { url "$artifactory_contextUrl/corda-dependencies" } + // Use system environment to activate caching with Artifactory, + // because it is actually easier to pass that during parallel build. + // NOTE: it has to be a name of a virtual repository with all + // required remote or local repositories! + if (System.getenv("CORDA_USE_CACHE")) { + maven { + name "R3 Maven remote repositories" + url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}" + authentication { + basic(BasicAuthentication) + } + credentials { + username = System.getenv('CORDA_ARTIFACTORY_USERNAME') + password = System.getenv('CORDA_ARTIFACTORY_PASSWORD') + } + } + } else { + mavenLocal() + gradlePluginPortal() + maven { url "${artifactory_contextUrl}/corda-dependencies" } + } } } // The project is named 'corda-project' and not 'corda' because if this is named the same as the