Merge remote-tracking branch 'origin/release/os/4.5' into vkolomeyko/os-4.5-os-4.6-merge

.ci/dev/compatibility/JenkinsfileJDK11Azul

@@ -37,6 +37,9 @@ pipeline {
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
         ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish JDK 11 Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -68,6 +71,9 @@ pipeline {
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\" " +
                             "-Ddocker.buildbase.tag=11latest " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.dockerfile=DockerfileJDK11Azul" +
                             " clean pushBuildImage preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest --stacktrace"
                 }
@@ -147,7 +153,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit '**/build/test-results-xml/**/*.xml'
+            junit testResults: '**/build/test-results-xml/**/*.xml', allowEmptyResults: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

.ci/dev/nightly-regression/Jenkinsfile

@@ -20,6 +20,9 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -36,6 +39,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean pushBuildImage --stacktrace"
                 }
@@ -75,7 +81,6 @@ pipeline {
         }
     }
 
-
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
@@ -86,4 +91,3 @@ pipeline {
         }
     }
 }
-

.ci/dev/publish-api-docs/Jenkinsfile

@@ -0,0 +1,35 @@
+@Library('corda-shared-build-pipeline-steps')
+
+import static com.r3.build.BuildControl.killAllExistingBuildsForJob
+
+killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
+
+pipeline {
+    agent { label 'standard' }
+    options {
+        ansiColor('xterm')
+        timestamps()
+        timeout(time: 3, unit: 'HOURS')
+    }
+
+    environment {
+        ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
+    }
+
+    stages {
+        stage('Publish Archived API Docs to Artifactory') {
+            when { tag pattern: /^release-os-V(\d+\.\d+)(\.\d+){0,1}(-GA){0,1}(-\d{4}-\d\d-\d\d-\d{4}){0,1}$/, comparator: 'REGEXP' }
+            steps {
+                sh "./gradlew :clean :docs:artifactoryPublish -DpublishApiDocs"
+            }
+        }
+    }
+
+    post {
+        cleanup {
+            deleteDir() /* clean up our workspace */
+        }
+    }
+}

.ci/dev/regression/Jenkinsfile

@@ -18,6 +18,7 @@ killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
  * Sense environment
  */
 boolean isReleaseTag = (env.TAG_NAME =~ /^release-.*(?<!_JDK11)$/)
+boolean isInternalRelease = (env.TAG_NAME =~ /^internal-release-.*$/)
 /*
 ** calculate the stage for NexusIQ evaluation
 **  * build for snapshots
@@ -49,6 +50,9 @@ pipeline {
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
         ARTIFACTORY_BUILD_NAME = "Corda / Publish / Publish Release to Artifactory".replaceAll("/", "::")
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -84,6 +88,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean preAllocateForParallelRegressionTest preAllocateForAllParallelSlowIntegrationTest pushBuildImage --stacktrace"
                 }
@@ -154,7 +161,7 @@ pipeline {
 
         stage('Publish Release to Docker Hub') {
             when {
-                expression { isReleaseTag }
+                expression { !isInternalRelease && isReleaseTag }
             }
             steps {
                 withCredentials([
@@ -171,7 +178,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true
 
             script {
                 try {

Jenkinsfile

@@ -17,6 +17,9 @@ pipeline {
         EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
         BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
         ARTIFACTORY_CREDENTIALS = credentials('artifactory-credentials')
+        CORDA_USE_CACHE = "corda-remotes"
+        CORDA_ARTIFACTORY_USERNAME = "${env.ARTIFACTORY_CREDENTIALS_USR}"
+        CORDA_ARTIFACTORY_PASSWORD = "${env.ARTIFACTORY_CREDENTIALS_PSW}"
     }
 
     stages {
@@ -27,6 +30,9 @@ pipeline {
                             "-Dkubenetize=true " +
                             "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
                             "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_USE_CACHE=\"${CORDA_USE_CACHE}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_USERNAME=\"\${ARTIFACTORY_CREDENTIALS_USR}\" " +
+                            "-Ddocker.container.env.parameter.CORDA_ARTIFACTORY_PASSWORD=\"\${ARTIFACTORY_CREDENTIALS_PSW}\" " +
                             "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" +
                             " clean preAllocateForAllParallelUnitTest preAllocateForAllParallelIntegrationTest pushBuildImage --stacktrace"
                 }
@@ -73,7 +79,7 @@ pipeline {
     post {
         always {
             archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false
-            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true
+            junit testResults: '**/build/test-results-xml/**/*.xml', keepLongStdio: true, allowEmptyResults: true
         }
         cleanup {
             deleteDir() /* clean up our workspace */

build.gradle

@@ -155,16 +155,34 @@ buildscript {
     ext.corda_docs_link = "https://docs.corda.net/docs/corda-os/$baseVersion"
     repositories {
         mavenLocal()
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
             mavenCentral()
             jcenter()
             maven {
                 url 'https://kotlin.bintray.com/kotlinx'
             }
             maven {
-            url "$artifactory_contextUrl/corda-dependencies-dev"
+                url "${artifactory_contextUrl}/corda-dependencies-dev"
             }
             maven {
-            url "$artifactory_contextUrl/corda-releases"
+                url "${artifactory_contextUrl}/corda-releases"
+            }
         }
     }
     dependencies {
@@ -355,11 +373,29 @@ allprojects {
 
     repositories {
         mavenLocal()
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
             mavenCentral()
             jcenter()
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
             maven { url 'https://repo.gradle.org/gradle/libs-releases' }
-        maven { url "$artifactory_contextUrl/corda-dev" }
+            maven { url "${artifactory_contextUrl}/corda-dev" }
+        }
     }
 
     configurations {

docs/build.gradle

@@ -0,0 +1,122 @@
+import org.apache.tools.ant.taskdefs.condition.Os
+
+apply plugin: 'org.jetbrains.dokka'
+apply plugin: 'net.corda.plugins.publish-utils'
+apply plugin: 'maven-publish'
+apply plugin: 'com.jfrog.artifactory'
+
+def internalPackagePrefixes(sourceDirs) {
+    def prefixes = []
+    // Kotlin allows packages to deviate from the directory structure, but let's assume they don't:
+    sourceDirs.collect { sourceDir ->
+        sourceDir.traverse(type: groovy.io.FileType.DIRECTORIES) {
+            if (it.name == 'internal') {
+                prefixes.add sourceDir.toPath().relativize(it.toPath()).toString().replace(File.separator, '.')
+            }
+        }
+    }
+    prefixes
+}
+
+ext {
+    // TODO: Add '../client/jfx/src/main/kotlin' and '../client/mock/src/main/kotlin' if we decide to make them into public API
+    dokkaSourceDirs = files('../core/src/main/kotlin', '../client/rpc/src/main/kotlin', '../finance/workflows/src/main/kotlin', '../finance/contracts/src/main/kotlin', '../client/jackson/src/main/kotlin',
+            '../testing/test-utils/src/main/kotlin', '../testing/node-driver/src/main/kotlin')
+    internalPackagePrefixes = internalPackagePrefixes(dokkaSourceDirs)
+    archivedApiDocsBaseFilename = 'api-docs'
+}
+
+dokka {
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/kotlin")
+}
+
+task dokkaJavadoc(type: org.jetbrains.dokka.gradle.DokkaTask) {
+    outputFormat = "javadoc"
+    outputDirectory = file("${rootProject.rootDir}/docs/build/html/api/javadoc")
+}
+
+[dokka, dokkaJavadoc].collect {
+    it.configure {
+        moduleName = 'corda'
+        processConfigurations = ['compile']
+        sourceDirs = dokkaSourceDirs
+        includes = ['packages.md']
+        jdkVersion = 8
+        externalDocumentationLink {
+            url = new URL("http://fasterxml.github.io/jackson-core/javadoc/2.9/")
+        }
+        externalDocumentationLink {
+            url = new URL("https://docs.oracle.com/javafx/2/api/")
+        }
+        externalDocumentationLink {
+            url = new URL("http://www.bouncycastle.org/docs/docs1.5on/")
+        }
+        internalPackagePrefixes.collect { packagePrefix ->
+            packageOptions {
+                prefix = packagePrefix
+                suppress = true
+            }
+        }
+    }
+}
+
+task apidocs(dependsOn: ['dokka', 'dokkaJavadoc']) {
+    group "Documentation"
+    description "Build API documentation"
+}
+
+task makeHTMLDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-html.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-html.sh"
+    }
+}
+
+task makePDFDocs(type: Exec){
+    if (Os.isFamily(Os.FAMILY_WINDOWS)) {
+        commandLine "docker", "run", "--rm", "-v", "${project.projectDir}:/opt/docs_builder", "-v", "${project.projectDir}/..:/opt", "corda/docs-builder:latest", "bash", "-c", "make-docsite-pdf.sh"
+    } else {
+        commandLine "bash", "-c",  "docker run --rm --user \$(id -u):\$(id -g) -v ${project.projectDir}:/opt/docs_builder -v ${project.projectDir}/..:/opt corda/docs-builder:latest bash -c make-docsite-pdf.sh"
+    }
+}
+
+task makeDocs(dependsOn: ['makeHTMLDocs', 'makePDFDocs'])
+apidocs.shouldRunAfter makeDocs
+
+task archiveApiDocs(type: Tar) {
+    dependsOn apidocs
+    from buildDir
+    include 'html/**'
+    extension 'tgz'
+    compression Compression.GZIP
+}
+
+publishing {
+    publications {
+        if (System.getProperty('publishApiDocs') != null) {
+            archivedApiDocs(MavenPublication) {
+                artifact archiveApiDocs {
+                    artifactId archivedApiDocsBaseFilename
+                }
+            }
+        }
+    }
+}
+
+artifactoryPublish {
+    publications('archivedApiDocs')
+    version = version.replaceAll('-SNAPSHOT', '')
+    publishPom = false
+}
+
+artifactory {
+    publish {
+        contextUrl = artifactory_contextUrl
+        repository {
+            repoKey = 'corda-dependencies-dev'
+            username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+            password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+        }
+    }
+}

settings.gradle

@@ -2,9 +2,27 @@ pluginManagement {
     ext.artifactory_contextUrl = 'https://software.r3.com/artifactory'
 
     repositories {
+        // Use system environment to activate caching with Artifactory,
+        // because it is actually easier to pass that during parallel build.
+        // NOTE: it has to be a name of a virtual repository with all
+        // required remote or local repositories!
+        if (System.getenv("CORDA_USE_CACHE")) {
+            maven {
+                name "R3 Maven remote repositories"
+                url "${artifactory_contextUrl}/${System.getenv("CORDA_USE_CACHE")}"
+                authentication {
+                    basic(BasicAuthentication)
+                }
+                credentials {
+                    username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+                    password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+                }
+            }
+        } else {
             mavenLocal()
             gradlePluginPortal()
-        maven { url "$artifactory_contextUrl/corda-dependencies" }
+            maven { url "${artifactory_contextUrl}/corda-dependencies" }
+        }
     }
 }
 // The project is named 'corda-project' and not 'corda' because if this is named the same as the

testing/node-driver/src/main/java/net/corda/testing/driver/SharedMemoryIncremental.java

@@ -17,7 +17,7 @@ import java.nio.channels.FileChannel;
  * import sun.misc.Unsafe;
  * import sun.nio.ch.DirectBuffer;
  */
-class SharedMemoryIncremental extends PortAllocation {
+public class SharedMemoryIncremental extends PortAllocation {
 
     static private final int DEFAULT_START_PORT = 10_000;
     static private final int FIRST_EPHEMERAL_PORT = 30_000;