Merge remote-tracking branch 'origin/release/os/4.5' into vkolomeyko/os-4.5-os-4.6-merge

2024-12-19 21:17:58 +00:00 · 2020-07-20 15:16:33 +01:00 · 2020-07-20 15:16:33 +01:00 · 3e9a8cdfcd
commit 3e9a8cdfcd
parent 5fef0726a2 2a84108e13
9 changed files with 259 additions and 25 deletions
--- a/.ci/dev/compatibility/JenkinsfileJDK11Azul
+++ b/.ci/dev/compatibility/JenkinsfileJDK11Azul
@ -37,6 +37,9 @@ pipeline {
        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
        ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
    }

    stages {
@ -68,6 +71,9 @@ pipeline {
                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " +
                            "-Ddocker.buildbase.tag=11latest " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                            "-Ddocker.dockerfile=DockerfileJDK11Azul" +
                            " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace"
                }
@ -147,7 +153,7 @@ pipeline {
    post {
        always {
            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true
        }
        cleanup {
            deleteDir() /* clean up our workspace */
--- a/.ci/dev/nightly-regression/Jenkinsfile
+++ b/.ci/dev/nightly-regression/Jenkinsfile
@ -20,6 +20,9 @@ pipeline {
        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
    }

    stages {
@ -36,6 +39,9 @@ pipeline {
                            "-Dkubenetize=true " +
                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                            " clean pushBuildImage --stacktrace"
                }
@ -75,7 +81,6 @@ pipeline {
        }
    }

-
    post {
        always {
            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
@ -86,4 +91,3 @@ pipeline {
        }
    }
 }
-
--- a/.ci/dev/publish-api-docs/Jenkinsfile
+++ b/.ci/dev/publish-api-docs/Jenkinsfile
@ -0,0 +1,35 @@
+@Library('corda-shared-build-pipeline-steps')
+
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'standard' }
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 3, unit: 'HOURS')
+    }
+
+    environment {
+        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
+    }
+
+    stages {
+        stage('Publish Archived API Docs to Artifactory') {
+            when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' }
+            steps {
+                sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs"
+            }
+        }
+    }
+
+    post {
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}
--- a/.ci/dev/regression/Jenkinsfile
+++ b/.ci/dev/regression/Jenkinsfile
@ -18,6 +18,7 @@ killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
 * Sense environment
 */
 boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?<!_JDK11)$/)
+boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
 /*
 ** calculate the stage for NexusIQ evaluation
 **  * build for snapshots
@ -49,6 +50,9 @@ pipeline {
        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
        ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
    }

    stages {
@ -84,6 +88,9 @@ pipeline {
                            "-Dkubenetize=true " +
                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                            " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace"
                }
@ -154,7 +161,7 @@ pipeline {

        stage('Publish Release to Docker Hub') {
            when {
-                expression { isReleaseTag }
+                expression { !isInternalRelease && isReleaseTag }
            }
            steps {
                withCredentials([
@ -171,7 +178,7 @@ pipeline {
    post {
        always {
            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true

            script {
                try {
--- a/8
+++ b/8
@ -17,6 +17,9 @@ pipeline {
        EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
        BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
    }

    stages {
@ -27,6 +30,9 @@ pipeline {
                            "-Dkubenetize=true " +
                            "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                            "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                            "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                            " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace"
                }
@ -73,7 +79,7 @@ pipeline {
    post {
        always {
            archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true
        }
        cleanup {
            deleteDir() /* clean up our workspace */
--- a/build.gradle
+++ b/build.gradle
@ -155,16 +155,34 @@ buildscript {
    ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion"
    repositories {
        mavenLocal()
-        mavenCentral()
-        jcenter()
-        maven {
-            url 'https://kotlin.bintray.com/kotlinx'
-        }
-        maven {
-            url "$artifactory_contextUrl/corda-dependencies-dev"
-        }
-        maven {
-            url "$artifactory_contextUrl/corda-releases"
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven {
+                url 'https://kotlin.bintray.com/kotlinx'
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-dependencies-dev"
+            }
+            maven {
+                url "${artifactory_contextUrl}/corda-releases"
+            }
        }
    }
    dependencies {
@ -355,11 +373,29 @@ allprojects {

    repositories {
        mavenLocal()
-        mavenCentral()
-        jcenter()
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
-        maven { url 'https://repo.gradle.org/gradle/libs-releases' }
-        maven { url "$artifactory_contextUrl/corda-dev" }
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenCentral()
+            jcenter()
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+            maven { url 'https://repo.gradle.org/gradle/libs-releases' }
+            maven { url "${artifactory_contextUrl}/corda-dev" }
+        }
    }

    configurations {
--- a/docs/build.gradle
+++ b/docs/build.gradle
@ -0,0 +1,122 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+apply plugin: 'org.jetbrains.dokka'
+apply plugin: 'net.corda.plugins.publish-utils'
+apply plugin: 'maven-publish'
+apply plugin: 'com.jfrog.artifactory'
+
+def internalPackagePrefixes(sourceDirs) {
+    def prefixes = []
+    // Kotlin allows packages to deviate from the directory structure, but let's assume they don't:
+    sourceDirs.collect { sourceDir ->
+        sourceDir.traverse(type: groovy.io.FileType.DIRECTORIES) {
+            if (it.name == 'internal') {
+                prefixes.add sourceDir.toPath().relativize(it.toPath()).toString().replace(File.separator, '.')
+            }
+        }
+    }
+    prefixes
+}
+
+ext {
+    // TODO: Add '../client/jfx/src/main/kotlin' and '../client/mock/src/main/kotlin' if we decide to make them into public API
+    dokkaSourceDirs = files('../core/src/main/kotlin', '../client/rpc/src/main/kotlin', '../finance/workflows/src/main/kotlin', '../finance/contracts/src/main/kotlin', '../client/jackson/src/main/kotlin',
+            '../testing/test-utils/src/main/kotlin', '../testing/node-driver/src/main/kotlin')
+    internalPackagePrefixes = internalPackagePrefixes(dokkaSourceDirs)
+    archivedApiDocsBaseFilename = 'api-docs'
+}
+
+dokka {
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/kotlin")
+}
+
+task dokkaJavadoc(type: org.jetbrains.dokka.gradle.DokkaTask) {
+    outputFormat = "javadoc"
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/javadoc")
+}
+
+[dokka, dokkaJavadoc].collect {
+    it.configure {
+        moduleName = 'corda'
+        processConfigurations = ['compile']
+        sourceDirs = dokkaSourceDirs
+        includes = ['packages.md']
+        jdkVersion = 8
+        externalDocumentationLink {
+            url = new URL("http://fasterxml.github.io/jackson-core/javadoc/2.9/")
+        }
+        externalDocumentationLink {
+            url = new URL("https://docs.oracle.com/javafx/2/api/")
+        }
+        externalDocumentationLink {
+            url = new URL("http://www.bouncycastle.org/docs/docs1.5on/")
+        }
+        internalPackagePrefixes.collect { packagePrefix ->
+            packageOptions {
+                prefix = packagePrefix
+                suppress = true
+            }
+        }
+    }
+}
+
+task apidocs(dependsOn: ['dokka', 'dokkaJavadoc']) {
+    group "Documentation"
+    description "Build API documentation"
+}
+
+task makeHTMLDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-html.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-html.sh"
+    }
+}
+
+task makePDFDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-pdf.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-pdf.sh"
+    }
+}
+
+task makeDocs(dependsOn: ['makeHTMLDocs', 'makePDFDocs'])
+apidocs.shouldRunAfter makeDocs
+
+task archiveApiDocs(type: Tar) {
+    dependsOn apidocs
+    from buildDir
+    include 'html/**'
+    extension 'tgz'
+    compression Compression.GZIP
+}
+
+publishing {
+    publications {
+        if (System.getProperty('publishApiDocs') != null) {
+            archivedApiDocs(MavenPublication) {
+                artifact archiveApiDocs {
+                    artifactId archivedApiDocsBaseFilename
+                }
+            }
+        }
+    }
+}
+
+artifactoryPublish {
+    publications('archivedApiDocs')
+    version = version.replaceAll('-SNAPSHOT', '')
+    publishPom = false
+}
+
+artifactory {
+    publish {
+        contextUrl = artifactory_contextUrl
+        repository {
+            repoKey = 'corda-dependencies-dev'
+            username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+            password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+        }
+    }
+}
--- a/settings.gradle
+++ b/settings.gradle
@ -2,9 +2,27 @@ pluginManagement {
    ext.artifactory_contextUrl = 'https://software.r3.com/artifactory'

    repositories {
-        mavenLocal()
-        gradlePluginPortal()
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
+            mavenLocal()
+            gradlePluginPortal()
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+        }
    }
 }
 // The project is named 'corda-project' and not 'corda' because if this is named the same as the
--- a/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java
+++ b/testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java
@ -17,7 +17,7 @@ import java.nio.channels.FileChannel;
 * import sun.misc.Unsafe;
 * import sun.nio.ch.DirectBuffer;
 */
-class SharedMemoryIncremental extends PortAllocation {
+public class SharedMemoryIncremental extends PortAllocation {

    static private final int DEFAULT_START_PORT = 10_000;
    static private final int FIRST_EPHEMERAL_PORT = 30_000;