ES-1331: Enable Gradle Enterprise build scans & remote caching (#7574)

* ES-1131: Enable Gradle Enterprise build scans & remote caching

.ci/dev/compatibility/JenkinsfileJDK11Compile

@@ -24,26 +24,32 @@ pipeline {
 
     environment {
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials')
+        BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}"
+        BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}"
         CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
         CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key')
         CORDA_USE_CACHE = "corda-remotes"
     }
 
     stages {
         stage('JDK 11 Compile') {
             steps {
-                sh "./gradlew --no-daemon -Pcompilation.allWarningsAsErrors=true -Ptests.failFast=false " +
+                sh "./gradlew --no-daemon --parallel --build-cache -Pcompilation.allWarningsAsErrors=true -Ptests.failFast=false " +
                 "-Ptests.ignoreFailures=true clean compileAll --stacktrace"
             }
         }
         stage('Deploy nodes') {
             steps {
-                sh "./gradlew --no-daemon deployNodes"
+                sh "./gradlew --no-daemon --build-cache deployNodes"
             }
         }
     }
-
     post {
+        always {
+            findBuildScans()
+        }
         cleanup {
             deleteDir() /* clean up our workspace */
         }

.ci/dev/pr-code-checks/Jenkinsfile

@@ -15,24 +15,28 @@ pipeline {
      * List environment variables in alphabetical order
      */
     environment {
-        SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret')
-        C4_OS_SNYK_ORG_ID = credentials('c4-os-snyk-org-id')
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials')
+        BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}"
+        BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}"
         CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
         CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key')
         CORDA_USE_CACHE = "corda-remotes"
+        C4_OS_SNYK_ORG_ID = credentials('c4-os-snyk-org-id')
+        SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret')
     }
 
     stages {
         stage('Detekt check') {
             steps {
-                sh "./gradlew --no-daemon clean detekt"
+                sh "./gradlew --no-daemon --parallel --build-cache clean detekt"
             }
         }
 
         stage('Compilation warnings check') {
             steps {
-                sh "./gradlew --no-daemon -Pcompilation.warningsAsErrors=true compileAll"
+                sh "./gradlew --no-daemon --parallel --build-cache -Pcompilation.warningsAsErrors=true compileAll"
             }
         }
 
@@ -57,19 +61,21 @@ pipeline {
 
         stage('No API change check') {
             steps {
-                sh "./gradlew --no-daemon generateApi"
+                sh "./gradlew --no-daemon --parallel --build-cache generateApi"
                 sh ".ci/check-api-changes.sh"
             }
         }
 
         stage('Deploy Nodes') {
             steps {
-                sh "./gradlew --no-daemon jar deployNodes"
+                sh "./gradlew --no-daemon --build-cache jar deployNodes"
             }
         }
     }
-
     post {
+        always {
+            findBuildScans()
+        }
         cleanup {
             deleteDir() /* clean up our workspace */
         }

.ci/dev/regression/Jenkinsfile

@@ -27,6 +27,7 @@ String COMMON_GRADLE_PARAMS = [
         '--info',
         '-Pcompilation.warningsAsErrors=false',
         '-Ptests.failFast=true',
+        '--build-cache'
 ].join(' ')
 
 pipeline {
@@ -52,8 +53,12 @@ pipeline {
     environment {
         ARTIFACTORY_BUILD_NAME = "Corda :: Publish :: Publish Release to Artifactory :: ${env.BRANCH_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials')
+        BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}"
+        BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}"
         CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
         CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key')
         CORDA_USE_CACHE = "corda-remotes"
         DOCKER_URL = "https://index.docker.io/v1/"
         EMAIL_RECIPIENTS = credentials('corda4-email-recipient')
@@ -69,7 +74,8 @@ pipeline {
                         './gradlew',
                         COMMON_GRADLE_PARAMS,
                         'clean',
-                        'jar'
+                        'jar',
+                        '--parallel'
                 ].join(' ')
             }
         }
@@ -129,8 +135,8 @@ pipeline {
                     }
                     post {
                         always {
-                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: false
-                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true
                             /*
                              * Copy all JUnit results files into a single top level directory.
                              * This is necessary to stop the allure plugin from hitting out
@@ -165,7 +171,8 @@ pipeline {
                                 sh script: [
                                         './gradlew',
                                         COMMON_GRADLE_PARAMS,
-                                        'jar'
+                                        'jar',
+                                        '--parallel'
                                 ].join(' ')
                             }
                         }
@@ -201,8 +208,8 @@ pipeline {
                 stage('Same agent') {
                     post {
                         always {
-                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: false
-                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true
                             /*
                              * Copy all JUnit results files into a single top level directory.
                              * This is necessary to stop the allure plugin from hitting out
@@ -324,6 +331,7 @@ pipeline {
     post {
         always {
             script {
+                findBuildScans()
                 if (gitUtils.isReleaseTag()) {
                     gitUtils.getGitLog(env.TAG_NAME, env.GIT_URL.replace('https://github.com/corda/', ''))
                 }

Jenkinsfile

@@ -24,6 +24,7 @@ String COMMON_GRADLE_PARAMS = [
         '-Ptests.failFast=true',
         '-Ddependx.branch.origin="${GIT_COMMIT}"',    // DON'T change quotation - GIT_COMMIT variable is substituted by SHELL!!!!
         '-Ddependx.branch.target="${CHANGE_TARGET}"', // DON'T change quotation - CHANGE_TARGET variable is substituted by SHELL!!!!
+        '--build-cache',
 ].join(' ')
 
 pipeline {
@@ -45,8 +46,12 @@ pipeline {
      */
     environment {
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials')
+        BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}"
+        BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}"
         CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
         CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key')
         CORDA_USE_CACHE = "corda-remotes"
     }
 
@@ -57,7 +62,8 @@ pipeline {
                         './gradlew',
                         COMMON_GRADLE_PARAMS,
                         'clean',
-                        'jar'
+                        'jar',
+                        '--parallel'
                 ].join(' ')
             }
         }
@@ -79,8 +85,8 @@ pipeline {
                     }
                     post {
                         always {
-                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: true
-                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true,allowEmptyResults: true
                         }
                         cleanup {
                             deleteDir() /* clean up our workspace */
@@ -97,7 +103,8 @@ pipeline {
                                 sh script: [
                                         './gradlew',
                                         COMMON_GRADLE_PARAMS,
-                                        'jar'
+                                        'jar',
+                                        '--parallel'
                                 ].join(' ')
                             }
                         }
@@ -115,8 +122,8 @@ pipeline {
                 stage('Same agent') {
                     post {
                         always {
-                            archiveArtifacts artifacts: '**/*.log', fingerprint: false
+                            archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: true
-                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true
+                            junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true
                         }
                     }
                     stages {
@@ -134,8 +141,10 @@ pipeline {
             }
         }
     }
-
     post {
+        always {
+            findBuildScans()
+        }
         cleanup {
             deleteDir() /* clean up our workspace */
         }

build.gradle

@@ -188,6 +188,7 @@ buildscript {
                     includeGroupByRegex 'com\\.r3(\\..*)?'
                 }
             }
+            gradlePluginPortal()
             mavenCentral()
             jcenter()
         }
@@ -213,6 +214,8 @@ buildscript {
         classpath "us.kirchmeier:gradle-capsule-plugin:1.0.4_r3"
         classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: '1.3.0'
         classpath "org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.8"
+        classpath "com.gradle:gradle-enterprise-gradle-plugin:$gradleEnterprisePlugin"
+        classpath "com.gradle:common-custom-user-data-gradle-plugin:$customUserDataGradlePlugin"
     }
 
     configurations.all {
@@ -223,7 +226,6 @@ buildscript {
 plugins {
     // Add the shadow plugin to the plugins classpath for the entire project.
     id 'com.github.johnrengelman.shadow' version '2.0.4' apply false
-    id "com.gradle.build-scan" version "2.2.1"
     id "org.ajoberstar.grgit" version "4.0.0"
 }
 
@@ -232,7 +234,22 @@ apply plugin: 'com.github.ben-manes.versions'
 apply plugin: 'net.corda.plugins.publish-utils'
 apply plugin: 'com.jfrog.artifactory'
 apply plugin: 'com.r3.testing.distributed-testing'
+apply plugin: "com.gradle.build-scan"
+apply plugin: "com.gradle.common-custom-user-data-gradle-plugin"
 
+buildScan {
+    server = gradleEnterpriseUrl
+    allowUntrustedServer = false
+    def apiKey = project.findProperty('CORDA_GRADLE_SCAN_KEY') ?: System.getenv('CORDA_GRADLE_SCAN_KEY')
+    if (apiKey?.trim()) {
+        publishAlways()
+        capture {
+            taskInputFiles = true
+        }
+        uploadInBackground = false
+        accessKey = apiKey
+    }
+}
 
 // If the command line project option -PversionFromGit is added to the gradle invocation, we'll resolve 
 // the latest git commit hash and timestamp and create a version postfix from that
@@ -277,6 +294,19 @@ allprojects {
         )
     }
 
+    // we do this to allow for Gradle task caching.
+    // below block tells Gradle to ignore specifically the dymaically generated files in the manifest when checking if a file is up to date
+    // this has no impact on publishing or production of jar, This only reates to Grades mechamish for verifying the Cache key
+    normalization {
+        runtimeClasspath {
+            ignore("**/*.EC")  //signing related
+            ignore("**/*.SF")  //signing related
+            ignore("**/*.MF")
+            ignore("**/*.kotlin_module")
+            ignore("**/Cordapp-Dependencies")
+        }
+    }
+
     dependencyCheck {
         suppressionFile = '.ci/dependency-checker/suppressedLibraries.xml'
         cveValidForHours = 1
@@ -729,11 +759,6 @@ wrapper {
     distributionType = Wrapper.DistributionType.ALL
 }
 
-buildScan {
-    termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-    termsOfServiceAgree = 'yes'
-}
-
 distributedTesting {
     profilesURL = 'https://raw.githubusercontent.com/corda/infrastructure-profiles/master'
 

buildCacheSettings.gradle

@@ -1,16 +0,0 @@
-// Gradle Build Cache configuration recommendation: https://docs.gradle.org/current/userguide/build_cache.html
-ext {
-    isCiServer = System.getenv().containsKey("CORDA_CI")
-    gradleBuildCacheURL = System.getenv().containsKey("GRADLE_BUILD_CACHE_URL") ? System.getenv().get("GRADLE_BUILD_CACHE_URL") : 'http://localhost:5071/cache/'
-}
-
-buildCache {
-    local {
-        enabled = !isCiServer
-    }
-    remote(HttpBuildCache) {
-        enabled = isCiServer
-        url = gradleBuildCacheURL
-        push = isCiServer
-    }
-}

gradle.properties

@@ -1,7 +1,12 @@
 kotlin.incremental=true
 org.gradle.jvmargs=-XX:+UseG1GC -Xmx4g -Dfile.encoding=UTF-8
-org.gradle.caching=false
+org.gradle.caching=true
 owasp.failOnError=false
 owasp.failBuildOnCVSS=11.0
 compilation.allWarningsAsErrors=false
 test.parallel=false
+
+# Gradle Enterprise
+gradleEnterpriseUrl = https://gradle.dev.r3.com
+gradleEnterprisePlugin = 3.8.1
+customUserDataGradlePlugin = 1.6.3

settings.gradle

@@ -117,8 +117,6 @@ include 'common-logging'
 project(":common-logging").projectDir = new File("$settingsDir/common/logging")
 // Common libraries - end
 
-apply from: 'buildCacheSettings.gradle'
-
 include 'core-deterministic'
 include 'core-deterministic:testing'
 include 'core-deterministic:testing:data'
@@ -128,3 +126,20 @@ include 'serialization-deterministic'
 include 'detekt-plugins'
 include 'tools:error-tool'
 
+buildCache {
+    local { enabled = false }
+    remote(HttpBuildCache) {
+        url = "${gradleEnterpriseUrl}/cache/"
+        credentials {
+            username = settings.ext.find('BUILD_CACHE_CREDENTIALS_USR') ?: System.getenv('BUILD_CACHE_CREDENTIALS_USR')
+            password = settings.ext.find('BUILD_CACHE_CREDENTIALS_PSW') ?: System.getenv('BUILD_CACHE_CREDENTIALS_PSW')
+        }
+        if (System.getenv().containsKey("JENKINS_URL")) {
+            push = true
+            enabled = true
+        } else {
+            push = false
+            enabled = false
+        }
+    }
+}