diff --git a/.ci/dev/compatibility/JenkinsfileJDK11Compile b/.ci/dev/compatibility/JenkinsfileJDK11Compile index 549845dc5d..83d4923df3 100644 --- a/.ci/dev/compatibility/JenkinsfileJDK11Compile +++ b/.ci/dev/compatibility/JenkinsfileJDK11Compile @@ -24,8 +24,12 @@ pipeline { environment { ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials') + BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}" + BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key') CORDA_USE_CACHE = "corda-remotes" } @@ -33,18 +37,20 @@ pipeline { stage('JDK 11 Compile') { steps { authenticateGradleWrapper() - sh "./gradlew --no-daemon -Pcompilation.allWarningsAsErrors=true -Ptests.failFast=false " + + sh "./gradlew --no-daemon --parallel --build-cache -Pcompilation.allWarningsAsErrors=true -Ptests.failFast=false " + "-Ptests.ignoreFailures=true clean compileAll --stacktrace" } } stage('Deploy nodes') { steps { - sh "./gradlew --no-daemon deployNodes" + sh "./gradlew --no-daemon --build-cache deployNodes" } } } - post { + always { + findBuildScans() + } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/.ci/dev/pr-code-checks/Jenkinsfile b/.ci/dev/pr-code-checks/Jenkinsfile index ba89d0b85d..456dc80994 100644 --- a/.ci/dev/pr-code-checks/Jenkinsfile +++ b/.ci/dev/pr-code-checks/Jenkinsfile @@ -15,25 +15,29 @@ pipeline { * List environment variables in alphabetical order */ environment { - SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret') - C4_OS_SNYK_ORG_ID = credentials('c4-os-snyk-org-id') ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials') + BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}" + BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key') CORDA_USE_CACHE = "corda-remotes" + C4_OS_SNYK_ORG_ID = credentials('c4-os-snyk-org-id') + SNYK_API_TOKEN = credentials('c4-os-snyk-api-token-secret') } stages { stage('Detekt check') { steps { authenticateGradleWrapper() - sh "./gradlew --no-daemon clean detekt" + sh "./gradlew --no-daemon --parallel --build-cache clean detekt" } } stage('Compilation warnings check') { steps { - sh "./gradlew --no-daemon -Pcompilation.warningsAsErrors=true compileAll" + sh "./gradlew --no-daemon --parallel --build-cache -Pcompilation.warningsAsErrors=true compileAll" } } @@ -59,19 +63,21 @@ pipeline { stage('No API change check') { steps { - sh "./gradlew --no-daemon generateApi" + sh "./gradlew --no-daemon --parallel --build-cache generateApi" sh ".ci/check-api-changes.sh" } } stage('Deploy Nodes') { steps { - sh "./gradlew --no-daemon jar deployNodes" + sh "./gradlew --no-daemon --build-cache jar deployNodes" } } } - post { + always { + findBuildScans() + } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index a8ab2dcece..d4126f51e5 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -29,7 +29,8 @@ String COMMON_GRADLE_PARAMS = [ '--info', '-Pcompilation.warningsAsErrors=false', '-Ptests.failFast=true', - '-DexcludeShell', + '--build-cache', + '-DexcludeShell' ].join(' ') pipeline { @@ -55,8 +56,12 @@ pipeline { environment { ARTIFACTORY_BUILD_NAME = "Corda :: Publish :: Publish Release to Artifactory :: ${env.BRANCH_NAME}" ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials') + BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}" + BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key') CORDA_BUILD_EDITION = "${buildEdition}" CORDA_USE_CACHE = "corda-remotes" DOCKER_URL = "https://index.docker.io/v1/" @@ -75,7 +80,8 @@ pipeline { './gradlew', COMMON_GRADLE_PARAMS, 'clean', - 'jar' + 'jar', + '--parallel' ].join(' ') } } @@ -135,8 +141,8 @@ pipeline { } post { always { - archiveArtifacts artifacts: '**/*.log', fingerprint: false - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true /* * Copy all JUnit results files into a single top level directory. * This is necessary to stop the allure plugin from hitting out @@ -172,7 +178,8 @@ pipeline { sh script: [ './gradlew', COMMON_GRADLE_PARAMS, - 'jar' + 'jar', + '--parallel' ].join(' ') } } @@ -208,8 +215,8 @@ pipeline { stage('Same agent') { post { always { - archiveArtifacts artifacts: '**/*.log', fingerprint: false - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: false + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true /* * Copy all JUnit results files into a single top level directory. * This is necessary to stop the allure plugin from hitting out @@ -331,6 +338,7 @@ pipeline { post { always { script { + findBuildScans() if (gitUtils.isReleaseTag()) { gitUtils.getGitLog(env.TAG_NAME, env.GIT_URL.replace('https://github.com/corda/', '')) } diff --git a/Jenkinsfile b/Jenkinsfile index 0e8c951218..f2b5686859 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,6 +24,7 @@ String COMMON_GRADLE_PARAMS = [ '-Ptests.failFast=true', '-Ddependx.branch.origin="${GIT_COMMIT}"', // DON'T change quotation - GIT_COMMIT variable is substituted by SHELL!!!! '-Ddependx.branch.target="${CHANGE_TARGET}"', // DON'T change quotation - CHANGE_TARGET variable is substituted by SHELL!!!! + '--build-cache', ].join(' ') pipeline { @@ -45,8 +46,12 @@ pipeline { */ environment { ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials') + BUILD_CACHE_CREDENTIALS = credentials('gradle-ent-cache-credentials') + BUILD_CACHE_PASSWORD = "${env.BUILD_CACHE_CREDENTIALS_PSW}" + BUILD_CACHE_USERNAME = "${env.BUILD_CACHE_CREDENTIALS_USR}" CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}" CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}" + CORDA_GRADLE_SCAN_KEY = credentials('gradle-build-scans-key') CORDA_USE_CACHE = "corda-remotes" } @@ -58,7 +63,8 @@ pipeline { './gradlew', COMMON_GRADLE_PARAMS, 'clean', - 'jar' + 'jar', + '--parallel' ].join(' ') } } @@ -80,8 +86,8 @@ pipeline { } post { always { - archiveArtifacts artifacts: '**/*.log', fingerprint: false - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: true + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true,allowEmptyResults: true } cleanup { deleteDir() /* clean up our workspace */ @@ -99,7 +105,8 @@ pipeline { sh script: [ './gradlew', COMMON_GRADLE_PARAMS, - 'jar' + 'jar', + '--parallel' ].join(' ') } } @@ -117,8 +124,8 @@ pipeline { stage('Same agent') { post { always { - archiveArtifacts artifacts: '**/*.log', fingerprint: false - junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true + archiveArtifacts artifacts: '**/*.log', allowEmptyArchive: true, fingerprint: true + junit testResults: '**/build/test-results/**/*.xml', keepLongStdio: true, allowEmptyResults: true } } stages { @@ -136,8 +143,10 @@ pipeline { } } } - post { + always { + findBuildScans() + } cleanup { deleteDir() /* clean up our workspace */ } diff --git a/build.gradle b/build.gradle index f54442500c..e175387848 100644 --- a/build.gradle +++ b/build.gradle @@ -183,6 +183,7 @@ buildscript { includeGroupByRegex 'com\\.r3(\\..*)?' } } + gradlePluginPortal() mavenCentral() jcenter() } @@ -208,6 +209,8 @@ buildscript { classpath "us.kirchmeier:gradle-capsule-plugin:1.0.4_r3" classpath group: "com.r3.testing", name: "gradle-distributed-testing-plugin", version: '1.3.0' classpath "org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:2.8" + classpath "com.gradle:gradle-enterprise-gradle-plugin:$gradleEnterprisePlugin" + classpath "com.gradle:common-custom-user-data-gradle-plugin:$customUserDataGradlePlugin" } configurations.classpath { @@ -219,7 +222,6 @@ buildscript { plugins { // Add the shadow plugin to the plugins classpath for the entire project. id 'com.github.johnrengelman.shadow' version '2.0.4' apply false - id "com.gradle.build-scan" version "2.2.1" id "org.ajoberstar.grgit" version "4.0.0" } @@ -228,7 +230,22 @@ apply plugin: 'com.github.ben-manes.versions' apply plugin: 'net.corda.plugins.publish-utils' apply plugin: 'com.jfrog.artifactory' apply plugin: 'com.r3.testing.distributed-testing' +apply plugin: "com.gradle.build-scan" +apply plugin: "com.gradle.common-custom-user-data-gradle-plugin" +buildScan { + server = gradleEnterpriseUrl + allowUntrustedServer = false + def apiKey = project.findProperty('CORDA_GRADLE_SCAN_KEY') ?: System.getenv('CORDA_GRADLE_SCAN_KEY') + if (apiKey?.trim()) { + publishAlways() + capture { + taskInputFiles = true + } + uploadInBackground = false + accessKey = apiKey + } +} // If the command line project option -PversionFromGit is added to the gradle invocation, we'll resolve // the latest git commit hash and timestamp and create a version postfix from that @@ -273,6 +290,19 @@ allprojects { ) } + // we do this to allow for Gradle task caching. + // below block tells Gradle to ignore specifically the dymaically generated files in the manifest when checking if a file is up to date + // this has no impact on publishing or production of jar, This only reates to Grades mechamish for verifying the Cache key + normalization { + runtimeClasspath { + ignore("**/*.EC") //signing related + ignore("**/*.SF") //signing related + ignore("**/*.MF") + ignore("**/*.kotlin_module") + ignore("**/Cordapp-Dependencies") + } + } + dependencyCheck { suppressionFile = '.ci/dependency-checker/suppressedLibraries.xml' cveValidForHours = 1 @@ -771,11 +801,6 @@ wrapper { distributionType = Wrapper.DistributionType.ALL } -buildScan { - termsOfServiceUrl = 'https://gradle.com/terms-of-service' - termsOfServiceAgree = 'yes' -} - distributedTesting { profilesURL = 'https://raw.githubusercontent.com/corda/infrastructure-profiles/master' diff --git a/buildCacheSettings.gradle b/buildCacheSettings.gradle deleted file mode 100644 index b4d0175f6e..0000000000 --- a/buildCacheSettings.gradle +++ /dev/null @@ -1,16 +0,0 @@ -// Gradle Build Cache configuration recommendation: https://docs.gradle.org/current/userguide/build_cache.html -ext { - isCiServer = System.getenv().containsKey("CORDA_CI") - gradleBuildCacheURL = System.getenv().containsKey("GRADLE_BUILD_CACHE_URL") ? System.getenv().get("GRADLE_BUILD_CACHE_URL") : 'http://localhost:5071/cache/' -} - -buildCache { - local { - enabled = !isCiServer - } - remote(HttpBuildCache) { - enabled = isCiServer - url = gradleBuildCacheURL - push = isCiServer - } -} diff --git a/gradle.properties b/gradle.properties index d70b133fa3..a54780ad19 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,7 +1,12 @@ kotlin.incremental=true org.gradle.jvmargs=-XX:+UseG1GC -Xmx4g -Dfile.encoding=UTF-8 -org.gradle.caching=false +org.gradle.caching=true owasp.failOnError=false owasp.failBuildOnCVSS=11.0 compilation.allWarningsAsErrors=false test.parallel=false + +# Gradle Enterprise +gradleEnterpriseUrl = https://gradle.dev.r3.com +gradleEnterprisePlugin = 3.8.1 +customUserDataGradlePlugin = 1.6.3 diff --git a/settings.gradle b/settings.gradle index 65fd739c94..bd4aef6bd8 100644 --- a/settings.gradle +++ b/settings.gradle @@ -116,8 +116,6 @@ include 'common-logging' project(":common-logging").projectDir = new File("$settingsDir/common/logging") // Common libraries - end -apply from: 'buildCacheSettings.gradle' - include 'core-deterministic' include 'core-deterministic:testing' include 'core-deterministic:testing:data' @@ -127,3 +125,20 @@ include 'serialization-deterministic' include 'detekt-plugins' include 'tools:error-tool' +buildCache { + local { enabled = false } + remote(HttpBuildCache) { + url = "${gradleEnterpriseUrl}/cache/" + credentials { + username = settings.ext.find('BUILD_CACHE_CREDENTIALS_USR') ?: System.getenv('BUILD_CACHE_CREDENTIALS_USR') + password = settings.ext.find('BUILD_CACHE_CREDENTIALS_PSW') ?: System.getenv('BUILD_CACHE_CREDENTIALS_PSW') + } + if (System.getenv().containsKey("JENKINS_URL")) { + push = true + enabled = true + } else { + push = false + enabled = false + } + } +}