ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2024-12-19 21:17:58 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/release/os/4.3' into merge_4.3_to_4.4

Browse Source 
# Conflicts:
#	constants.properties
This commit is contained in:
stefano 2019-10-08 15:57:14 +01:00
commit 0488a5ac5d
57 changed files with 4216 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
.ci/dev/integration/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,59 @@
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda - Run Tests') {
stage('Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelIntegrationTest"
if (env.CHANGE_ID) {
pullRequest.createStatus(status: 'success',
context: 'continuous-integration/jenkins/pr-merge/integrationTest',
description: 'Integration Tests Passed',
targetUrl: "${env.JOB_URL}/testResults")
}
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

61
.ci/dev/regression/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,61 @@
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda Pull Request - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda Pull Request - Run Tests') {
stage('Unit and Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitAndIntegrationTest"
}
}
stage('Slow Integration Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelSlowIntegrationTest"
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

59
.ci/dev/unit/Jenkinsfile vendored Normal file
View File

@ -0,0 +1,59 @@
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
options { timestamps() }
environment {
DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}"
EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}"
BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}"
}
stages {
stage('Corda Pull Request - Generate Build Image') {
steps {
withCredentials([string(credentialsId: 'container_reg_passwd', variable: 'DOCKER_PUSH_PWD')]) {
sh "./gradlew " +
"-Dkubenetize=true " +
"-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " +
"-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " +
"-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" clean pushBuildImage"
}
sh "kubectl auth can-i get pods"
}
}
stage('Corda Pull Request - Run Tests') {
stage('Unit Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitTest"
if (env.CHANGE_ID) {
pullRequest.createStatus(status: 'success',
context: 'continuous-integration/jenkins/pr-merge/unitTest',
description: 'Unit Tests Passed',
targetUrl: "${env.JOB_URL}/testResults")
}
}
}
}
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
cleanup {
deleteDir() /* clean up our workspace */
}
}
}

181
.idea/codeStyles/Project.xml generated
View File

@ -40,187 +40,6 @@
<option name="EXTENDS_LIST_WRAP" value="0" />
<option name="ASSIGNMENT_WRAP" value="0" />
<option name="WRAP_ON_TYPING" value="0" />
<option name="arrangementSettings" />
<option name="forceArrangeMenuAvailable" value="false" />
<option name="CODE_STYLE_DEFAULTS" value="KOTLIN_OFFICIAL" />
<option name="RIGHT_MARGIN" value="140" />
<option name="LINE_COMMENT_AT_FIRST_COLUMN" value="true" />
<option name="BLOCK_COMMENT_AT_FIRST_COLUMN" value="true" />
<option name="LINE_COMMENT_ADD_SPACE" value="false" />
<option name="KEEP_LINE_BREAKS" value="true" />
<option name="KEEP_FIRST_COLUMN_COMMENT" value="true" />
<option name="KEEP_CONTROL_STATEMENT_IN_ONE_LINE" value="true" />
<option name="KEEP_BLANK_LINES_IN_DECLARATIONS" value="1" />
<option name="KEEP_BLANK_LINES_IN_CODE" value="1" />
<option name="KEEP_BLANK_LINES_BETWEEN_PACKAGE_DECLARATION_AND_HEADER" value="2" />
<option name="KEEP_BLANK_LINES_BEFORE_RBRACE" value="0" />
<option name="BLANK_LINES_BEFORE_PACKAGE" value="0" />
<option name="BLANK_LINES_AFTER_PACKAGE" value="1" />
<option name="BLANK_LINES_BEFORE_IMPORTS" value="1" />
<option name="BLANK_LINES_AFTER_IMPORTS" value="1" />
<option name="BLANK_LINES_AROUND_CLASS" value="1" />
<option name="BLANK_LINES_AROUND_FIELD" value="0" />
<option name="BLANK_LINES_AROUND_METHOD" value="1" />
<option name="BLANK_LINES_BEFORE_METHOD_BODY" value="0" />
<option name="BLANK_LINES_AROUND_FIELD_IN_INTERFACE" value="0" />
<option name="BLANK_LINES_AROUND_METHOD_IN_INTERFACE" value="1" />
<option name="BLANK_LINES_AFTER_CLASS_HEADER" value="0" />
<option name="BLANK_LINES_AFTER_ANONYMOUS_CLASS_HEADER" value="0" />
<option name="BLANK_LINES_BEFORE_CLASS_END" value="0" />
<option name="BRACE_STYLE" value="1" />
<option name="CLASS_BRACE_STYLE" value="1" />
<option name="METHOD_BRACE_STYLE" value="1" />
<option name="LAMBDA_BRACE_STYLE" value="1" />
<option name="USE_FLYING_GEESE_BRACES" value="false" />
<option name="DO_NOT_INDENT_TOP_LEVEL_CLASS_MEMBERS" value="false" />
<option name="ELSE_ON_NEW_LINE" value="false" />
<option name="WHILE_ON_NEW_LINE" value="false" />
<option name="CATCH_ON_NEW_LINE" value="false" />
<option name="FINALLY_ON_NEW_LINE" value="false" />
<option name="INDENT_CASE_FROM_SWITCH" value="true" />
<option name="CASE_STATEMENT_ON_NEW_LINE" value="true" />
<option name="INDENT_BREAK_FROM_CASE" value="true" />
<option name="SPECIAL_ELSE_IF_TREATMENT" value="true" />
<option name="ALIGN_MULTILINE_CHAINED_METHODS" value="false" />
<option name="ALIGN_MULTILINE_PARAMETERS" value="true" />
<option name="ALIGN_MULTILINE_PARAMETERS_IN_CALLS" value="false" />
<option name="ALIGN_MULTILINE_RESOURCES" value="true" />
<option name="ALIGN_MULTILINE_FOR" value="true" />
<option name="INDENT_WHEN_CASES" value="true" />
<option name="ALIGN_MULTILINE_BINARY_OPERATION" value="false" />
<option name="ALIGN_MULTILINE_ASSIGNMENT" value="false" />
<option name="ALIGN_MULTILINE_TERNARY_OPERATION" value="false" />
<option name="ALIGN_MULTILINE_THROWS_LIST" value="false" />
<option name="ALIGN_THROWS_KEYWORD" value="false" />
<option name="ALIGN_MULTILINE_EXTENDS_LIST" value="false" />
<option name="ALIGN_MULTILINE_METHOD_BRACKETS" value="false" />
<option name="ALIGN_MULTILINE_PARENTHESIZED_EXPRESSION" value="false" />
<option name="ALIGN_MULTILINE_ARRAY_INITIALIZER_EXPRESSION" value="false" />
<option name="ALIGN_GROUP_FIELD_DECLARATIONS" value="false" />
<option name="ALIGN_CONSECUTIVE_VARIABLE_DECLARATIONS" value="false" />
<option name="ALIGN_CONSECUTIVE_ASSIGNMENTS" value="false" />
<option name="ALIGN_SUBSEQUENT_SIMPLE_METHODS" value="false" />
<option name="SPACE_AROUND_ASSIGNMENT_OPERATORS" value="true" />
<option name="SPACE_AROUND_LOGICAL_OPERATORS" value="true" />
<option name="SPACE_AROUND_EQUALITY_OPERATORS" value="true" />
<option name="SPACE_AROUND_RELATIONAL_OPERATORS" value="true" />
<option name="SPACE_AROUND_BITWISE_OPERATORS" value="true" />
<option name="SPACE_AROUND_ADDITIVE_OPERATORS" value="true" />
<option name="SPACE_AROUND_MULTIPLICATIVE_OPERATORS" value="true" />
<option name="SPACE_AROUND_SHIFT_OPERATORS" value="true" />
<option name="SPACE_AROUND_UNARY_OPERATOR" value="false" />
<option name="SPACE_AROUND_LAMBDA_ARROW" value="true" />
<option name="SPACE_AROUND_METHOD_REF_DBL_COLON" value="false" />
<option name="SPACE_AFTER_COMMA" value="true" />
<option name="SPACE_AFTER_COMMA_IN_TYPE_ARGUMENTS" value="true" />
<option name="SPACE_BEFORE_COMMA" value="false" />
<option name="SPACE_AFTER_SEMICOLON" value="true" />
<option name="SPACE_BEFORE_SEMICOLON" value="false" />
<option name="SPACE_WITHIN_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_METHOD_CALL_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_EMPTY_METHOD_CALL_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_METHOD_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_EMPTY_METHOD_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_IF_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_WHILE_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_FOR_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_TRY_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_CATCH_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_SWITCH_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_SYNCHRONIZED_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_CAST_PARENTHESES" value="false" />
<option name="SPACE_WITHIN_BRACKETS" value="false" />
<option name="SPACE_WITHIN_BRACES" value="false" />
<option name="SPACE_WITHIN_ARRAY_INITIALIZER_BRACES" value="false" />
<option name="SPACE_WITHIN_EMPTY_ARRAY_INITIALIZER_BRACES" value="false" />
<option name="SPACE_AFTER_TYPE_CAST" value="true" />
<option name="SPACE_BEFORE_METHOD_CALL_PARENTHESES" value="false" />
<option name="SPACE_BEFORE_METHOD_PARENTHESES" value="false" />
<option name="SPACE_BEFORE_IF_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_WHILE_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_FOR_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_TRY_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_CATCH_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_SWITCH_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_SYNCHRONIZED_PARENTHESES" value="true" />
<option name="SPACE_BEFORE_CLASS_LBRACE" value="true" />
<option name="SPACE_BEFORE_METHOD_LBRACE" value="true" />
<option name="SPACE_BEFORE_IF_LBRACE" value="true" />
<option name="SPACE_BEFORE_ELSE_LBRACE" value="true" />
<option name="SPACE_BEFORE_WHILE_LBRACE" value="true" />
<option name="SPACE_BEFORE_FOR_LBRACE" value="true" />
<option name="SPACE_BEFORE_DO_LBRACE" value="true" />
<option name="SPACE_BEFORE_SWITCH_LBRACE" value="true" />
<option name="SPACE_BEFORE_TRY_LBRACE" value="true" />
<option name="SPACE_BEFORE_CATCH_LBRACE" value="true" />
<option name="SPACE_BEFORE_FINALLY_LBRACE" value="true" />
<option name="SPACE_BEFORE_SYNCHRONIZED_LBRACE" value="true" />
<option name="SPACE_BEFORE_ARRAY_INITIALIZER_LBRACE" value="false" />
<option name="SPACE_BEFORE_ANNOTATION_ARRAY_INITIALIZER_LBRACE" value="false" />
<option name="SPACE_BEFORE_ELSE_KEYWORD" value="true" />
<option name="SPACE_BEFORE_WHILE_KEYWORD" value="true" />
<option name="SPACE_BEFORE_CATCH_KEYWORD" value="true" />
<option name="SPACE_BEFORE_FINALLY_KEYWORD" value="true" />
<option name="SPACE_BEFORE_QUEST" value="true" />
<option name="SPACE_AFTER_QUEST" value="true" />
<option name="SPACE_BEFORE_COLON" value="true" />
<option name="SPACE_AFTER_COLON" value="true" />
<option name="SPACE_BEFORE_TYPE_PARAMETER_LIST" value="false" />
<option name="CALL_PARAMETERS_WRAP" value="0" />
<option name="PREFER_PARAMETERS_WRAP" value="false" />
<option name="CALL_PARAMETERS_LPAREN_ON_NEXT_LINE" value="false" />
<option name="CALL_PARAMETERS_RPAREN_ON_NEXT_LINE" value="false" />
<option name="METHOD_PARAMETERS_WRAP" value="0" />
<option name="METHOD_PARAMETERS_LPAREN_ON_NEXT_LINE" value="false" />
<option name="METHOD_PARAMETERS_RPAREN_ON_NEXT_LINE" value="false" />
<option name="RESOURCE_LIST_WRAP" value="0" />
<option name="RESOURCE_LIST_LPAREN_ON_NEXT_LINE" value="false" />
<option name="RESOURCE_LIST_RPAREN_ON_NEXT_LINE" value="false" />
<option name="EXTENDS_LIST_WRAP" value="0" />
<option name="THROWS_LIST_WRAP" value="0" />
<option name="EXTENDS_KEYWORD_WRAP" value="0" />
<option name="THROWS_KEYWORD_WRAP" value="0" />
<option name="METHOD_CALL_CHAIN_WRAP" value="1" />
<option name="WRAP_FIRST_METHOD_IN_CALL_CHAIN" value="false" />
<option name="PARENTHESES_EXPRESSION_LPAREN_WRAP" value="false" />
<option name="PARENTHESES_EXPRESSION_RPAREN_WRAP" value="false" />
<option name="BINARY_OPERATION_WRAP" value="0" />
<option name="BINARY_OPERATION_SIGN_ON_NEXT_LINE" value="false" />
<option name="TERNARY_OPERATION_WRAP" value="0" />
<option name="TERNARY_OPERATION_SIGNS_ON_NEXT_LINE" value="false" />
<option name="MODIFIER_LIST_WRAP" value="false" />
<option name="KEEP_SIMPLE_BLOCKS_IN_ONE_LINE" value="false" />
<option name="KEEP_SIMPLE_METHODS_IN_ONE_LINE" value="false" />
<option name="KEEP_SIMPLE_LAMBDAS_IN_ONE_LINE" value="false" />
<option name="KEEP_SIMPLE_CLASSES_IN_ONE_LINE" value="false" />
<option name="KEEP_MULTIPLE_EXPRESSIONS_IN_ONE_LINE" value="false" />
<option name="FOR_STATEMENT_WRAP" value="0" />
<option name="FOR_STATEMENT_LPAREN_ON_NEXT_LINE" value="false" />
<option name="FOR_STATEMENT_RPAREN_ON_NEXT_LINE" value="false" />
<option name="ARRAY_INITIALIZER_WRAP" value="0" />
<option name="ARRAY_INITIALIZER_LBRACE_ON_NEXT_LINE" value="false" />
<option name="ARRAY_INITIALIZER_RBRACE_ON_NEXT_LINE" value="false" />
<option name="ASSIGNMENT_WRAP" value="0" />
<option name="PLACE_ASSIGNMENT_SIGN_ON_NEXT_LINE" value="false" />
<option name="LABELED_STATEMENT_WRAP" value="2" />
<option name="WRAP_COMMENTS" value="false" />
<option name="ASSERT_STATEMENT_WRAP" value="0" />
<option name="ASSERT_STATEMENT_COLON_ON_NEXT_LINE" value="false" />
<option name="IF_BRACE_FORCE" value="0" />
<option name="DOWHILE_BRACE_FORCE" value="0" />
<option name="WHILE_BRACE_FORCE" value="0" />
<option name="FOR_BRACE_FORCE" value="0" />
<option name="WRAP_LONG_LINES" value="false" />
<option name="METHOD_ANNOTATION_WRAP" value="2" />
<option name="CLASS_ANNOTATION_WRAP" value="2" />
<option name="FIELD_ANNOTATION_WRAP" value="2" />
<option name="PARAMETER_ANNOTATION_WRAP" value="0" />
<option name="VARIABLE_ANNOTATION_WRAP" value="0" />
<option name="SPACE_BEFORE_ANOTATION_PARAMETER_LIST" value="false" />
<option name="SPACE_WITHIN_ANNOTATION_PARENTHESES" value="false" />
<option name="ENUM_CONSTANTS_WRAP" value="0" />
<option name="FORCE_REARRANGE_MODE" value="0" />
<option name="WRAP_ON_TYPING" value="0" />
</codeStyleSettings>
</code_scheme>
</component>

1
.idea/codeStyles/codeStyleConfig.xml generated
View File

@ -1,5 +1,6 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
<option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
</state>
</component>

52
Jenkinsfile vendored
View File

@ -1,4 +1,7 @@
killall_jobs()
@Library('existing-build-control')
import static com.r3.build.BuildControl.killAllExistingBuildsForJob
killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger())
pipeline {
agent { label 'k8s' }
@ -35,48 +38,27 @@ pipeline {
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelIntegrationTest"
}
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
}
stage('Unit Tests') {
steps {
sh "./gradlew " +
"-DbuildId=\"\${BUILD_ID}\" " +
"-Dkubenetize=true " +
"-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
" allParallelUnitTest"
}
}
// stage('Unit Tests') {
// steps {
// sh "./gradlew " +
// "-DbuildId=\"\${BUILD_ID}\" " +
// "-Dkubenetize=true " +
// "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" +
// " allParallelUnitTest"
// }
// post {
// always {
// junit '**/build/test-results-xml/**/*.xml'
// }
// }
// }
}
}
}
}
@NonCPS
def killall_jobs() {
def jobname = env.JOB_NAME
def buildnum = env.BUILD_NUMBER.toInteger()
def job = Jenkins.instance.getItemByFullName(jobname)
for (build in job.builds) {
if (!build.isBuilding()) {
continue;
post {
always {
junit '**/build/test-results-xml/**/*.xml'
}
if (buildnum == build.getNumber().toInteger()) {
continue
cleanup {
deleteDir() /* clean up our workspace */
}
echo "Killing task = ${build}"
build.doStop();
}
}

2
README.md
View File

@ -1,5 +1,5 @@
<p align="center">
<img src="https://www.corda.net/wp-content/uploads/2016/11/fg005_corda_b.png" alt="Corda" width="500">
<img src="https://www.corda.net/wp-content/themes/corda/assets/images/crda-logo-big.svg" alt="Corda" width="500">
</p>
<a href="https://ci-master.corda.r3cev.com/viewType.html?buildTypeId=Corda_CordaBuild&tab=buildTypeStatusDiv&guest=1"><img src="https://ci.corda.r3cev.com/app/rest/builds/buildType:Corda_CordaBuild/statusIcon"/></a> [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)

42
build.gradle
View File

@ -1,8 +1,10 @@
import net.corda.testing.DistributedTesting
import net.corda.testing.ImageBuilding
import net.corda.testing.Distribution
import net.corda.testing.ParallelTestGroup
import static org.gradle.api.JavaVersion.VERSION_1_8
import static org.gradle.api.JavaVersion.VERSION_11
import static org.gradle.api.JavaVersion.VERSION_1_8
buildscript {
// For sharing constants between builds
@ -25,10 +27,12 @@ buildscript {
if (JavaVersion.current() == JavaVersion.VERSION_11) {
ext.quasar_version = constants.getProperty("quasarVersion11")
ext.quasar_classifier = constants.getProperty("quasarClassifier11")
ext.jdkClassifier = constants.getProperty("jdkClassifier11")
}
else {
ext.quasar_version = constants.getProperty("quasarVersion")
ext.quasar_classifier = constants.getProperty("quasarClassifier")
ext.jdkClassifier = constants.getProperty("jdkClassifier")
}
ext.quasar_exclusions = [
'co.paralleluniverse**',
@ -121,8 +125,7 @@ buildscript {
// has been compiled by a more recent version of the Java Runtime (class file version 55.0)
ext.fontawesomefx_commons_version = '11.0'
ext.fontawesomefx_fontawesome_version = '4.7.0-11'
}
else {
} else {
ext.fontawesomefx_commons_version = '8.15'
ext.fontawesomefx_fontawesome_version = '4.7.0-5'
}
@ -245,7 +248,7 @@ allprojects {
jacoco {
// JDK11 official support (https://github.com/jacoco/jacoco/releases/tag/v0.8.3)
toolVersion = "0.8.3"
toolVersion = "0.8.3"
}
tasks.withType(JavaCompile) {
@ -316,6 +319,12 @@ allprojects {
}
}
jar {
// JDK11 built and published artifacts to include classifier
if (jdkClassifier != null)
archiveClassifier = jdkClassifier
}
group 'net.corda'
version "$corda_release_version"
@ -496,7 +505,8 @@ bintrayConfig {
'corda-common-configuration-parsing',
'corda-common-validation',
'corda-common-logging',
'corda-tools-network-builder'
'corda-tools-network-builder',
'corda-tools-checkpoint-agent'
]
license {
name = 'Apache-2.0'
@ -588,25 +598,37 @@ buildScan {
task allParallelIntegrationTest(type: ParallelTestGroup) {
testGroups "integrationTest"
numberOfShards 15
numberOfShards 10
streamOutput false
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
task allParallelSlowIntegrationTest(type: ParallelTestGroup) {
testGroups "slowIntegrationTest"
numberOfShards 4
streamOutput false
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
task allParallelUnitTest(type: ParallelTestGroup) {
testGroups "test"
numberOfShards 15
numberOfShards 10
streamOutput false
coresPerFork 3
coresPerFork 5
memoryInGbPerFork 6
distribute Distribution.CLASS
}
task allParallelUnitAndIntegrationTest(type: ParallelTestGroup) {
testGroups "test", "integrationTest"
numberOfShards 20
streamOutput false
numberOfShards 15
streamOutput true
coresPerFork 6
memoryInGbPerFork 10
distribute Distribution.CLASS
}
apply plugin: ImageBuilding
apply plugin: DistributedTesting

3
buildSrc/build.gradle
View File

@ -39,5 +39,8 @@ dependencies {
compile 'commons-codec:commons-codec:1.13'
compile "io.github.classgraph:classgraph:$class_graph_version"
compile "com.bmuschko:gradle-docker-plugin:5.0.0"
compile 'org.apache.commons:commons-csv:1.1'
compile group: 'org.jetbrains', name: 'annotations', version: '13.0'
testCompile "junit:junit:$junit_version"
testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
}

164
buildSrc/src/main/groovy/net/corda/testing/BucketingAllocator.java Normal file
View File

@ -0,0 +1,164 @@
package net.corda.testing;
//Why Java?! because sometimes types are useful.
import groovy.lang.Tuple2;
import org.gradle.api.tasks.TaskAction;
import org.jetbrains.annotations.NotNull;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public class BucketingAllocator {
private List<Tuple2<TestLister, Object>> sources = new ArrayList<>();
private final List<TestsForForkContainer> forkContainers;
private final Supplier<List<Tuple2<String, Double>>> timedTestsProvider;
public BucketingAllocator(Integer forkCount, Supplier<List<Tuple2<String, Double>>> timedTestsProvider) {
this.forkContainers = IntStream.range(0, forkCount).mapToObj(TestsForForkContainer::new).collect(Collectors.toList());
this.timedTestsProvider = timedTestsProvider;
}
public void addSource(TestLister source, Object testTask) {
sources.add(new Tuple2<>(source, testTask));
}
public List<String> getTestsForForkAndTestTask(Integer fork, Object testTask) {
return forkContainers.get(fork).getTestsForTask(testTask);
}
@TaskAction
public void generateTestPlan() {
List<Tuple2<String, Double>> allTestsFromCSV = timedTestsProvider.get();
List<Tuple2<String, Object>> allDiscoveredTests = getTestsOnClasspathOfTestingTasks();
List<TestBucket> matchedTests = matchClasspathTestsToCSV(allTestsFromCSV, allDiscoveredTests);
//use greedy algo - for each testbucket find the currently smallest container and add to it
allocateTestsToForks(matchedTests);
forkContainers.forEach(TestsForForkContainer::freeze);
printSummary();
}
private void printSummary() {
forkContainers.forEach(container -> {
System.out.println("####### TEST PLAN SUMMARY ( " + container.forkIdx + " ) #######");
System.out.println("Duration: " + container.getCurrentDuration());
System.out.println("Number of tests: " + container.testsForFork.stream().mapToInt(b -> b.foundTests.size()).sum());
System.out.println("Tests to Run: ");
container.testsForFork.forEach(tb -> {
System.out.println(tb.nameWithAsterix);
tb.foundTests.forEach(ft -> System.out.println("\t" + ft.getFirst() + ", " + ft.getSecond()));
});
});
}
private void allocateTestsToForks(@NotNull List<TestBucket> matchedTests) {
matchedTests.forEach(matchedTestBucket -> {
TestsForForkContainer smallestContainer = Collections.min(forkContainers, Comparator.comparing(TestsForForkContainer::getCurrentDuration));
smallestContainer.addBucket(matchedTestBucket);
});
}
private List<TestBucket> matchClasspathTestsToCSV(List<Tuple2<String, Double>> allTestsFromCSV, @NotNull List<Tuple2<String, Object>> allDiscoveredTests) {
return allDiscoveredTests.stream().map(tuple -> {
String testName = tuple.getFirst();
Object task = tuple.getSecond();
String noAsterixName = testName.substring(0, testName.length() - 1);
//2DO [can this filtering algorithm be improved - the test names are sorted, it should be possible to do something using binary search]
List<Tuple2<String, Double>> matchingTests = allTestsFromCSV.stream().filter(testFromCSV -> testFromCSV.getFirst().startsWith(noAsterixName)).collect(Collectors.toList());
return new TestBucket(task, testName, noAsterixName, matchingTests);
}).sorted(Comparator.comparing(TestBucket::getDuration).reversed()).collect(Collectors.toList());
}
private List<Tuple2<String, Object>> getTestsOnClasspathOfTestingTasks() {
return sources.stream().map(source -> {
TestLister lister = source.getFirst();
Object testTask = source.getSecond();
return lister.getAllTestsDiscovered().stream().map(test -> new Tuple2<>(test, testTask)).collect(Collectors.toList());
}).flatMap(Collection::stream).collect(Collectors.toList());
}
public static class TestBucket {
final Object testTask;
final String nameWithAsterix;
final String nameWithoutAsterix;
final List<Tuple2<String, Double>> foundTests;
final Double duration;
public TestBucket(Object testTask, String nameWithAsterix, String nameWithoutAsterix, List<Tuple2<String, Double>> foundTests) {
this.testTask = testTask;
this.nameWithAsterix = nameWithAsterix;
this.nameWithoutAsterix = nameWithoutAsterix;
this.foundTests = foundTests;
duration = Math.max(foundTests.stream().mapToDouble(tp -> Math.max(tp.getSecond(), 10)).sum(), 10);
}
public Double getDuration() {
return duration;
}
@Override
public String toString() {
return "TestBucket{" +
"testTask=" + testTask +
", nameWithAsterix='" + nameWithAsterix + '\'' +
", nameWithoutAsterix='" + nameWithoutAsterix + '\'' +
", foundTests=" + foundTests +
", duration=" + duration +
'}';
}
}
public static class TestsForForkContainer {
private Double runningDuration = 0.0;
private final Integer forkIdx;
private final List<TestBucket> testsForFork = Collections.synchronizedList(new ArrayList<>());
private final Map<Object, List<TestBucket>> frozenTests = new HashMap<>();
public TestsForForkContainer(Integer forkIdx) {
this.forkIdx = forkIdx;
}
public void addBucket(TestBucket tb) {
this.testsForFork.add(tb);
this.runningDuration = runningDuration + tb.duration;
}
public Double getCurrentDuration() {
return runningDuration;
}
public void freeze() {
testsForFork.forEach(tb -> {
frozenTests.computeIfAbsent(tb.testTask, i -> new ArrayList<>()).add(tb);
});
}
public List<String> getTestsForTask(Object task) {
return frozenTests.getOrDefault(task, Collections.emptyList()).stream().map(it -> it.nameWithAsterix).collect(Collectors.toList());
}
public List<TestBucket> getBucketsForFork() {
return new ArrayList<>(testsForFork);
}
@Override
public String toString() {
return "TestsForForkContainer{" +
"runningDuration=" + runningDuration +
", forkIdx=" + forkIdx +
", testsForFork=" + testsForFork +
", frozenTests=" + frozenTests +
'}';
}
}
}

69
buildSrc/src/main/groovy/net/corda/testing/BucketingAllocatorTask.java Normal file
View File

@ -0,0 +1,69 @@
package net.corda.testing;
import groovy.lang.Tuple2;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVRecord;
import org.gradle.api.DefaultTask;
import org.gradle.api.tasks.TaskAction;
import org.gradle.api.tasks.testing.Test;
import javax.inject.Inject;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.function.Supplier;
import java.util.stream.Collectors;
public class BucketingAllocatorTask extends DefaultTask {
private static final String DEFAULT_TESTING_TEST_TIMES_CSV = "testing/test-times.csv";
private final BucketingAllocator allocator;
@Inject
public BucketingAllocatorTask(Integer forkCount) {
Supplier<List<Tuple2<String, Double>>> defaultTestCSV = () -> {
try {
FileReader csvSource = new FileReader(new File(BucketingAllocatorTask.this.getProject().getRootDir(), DEFAULT_TESTING_TEST_TIMES_CSV));
return fromCSV(csvSource);
} catch (IOException e) {
return Collections.emptyList();
}
};
this.allocator = new BucketingAllocator(forkCount, defaultTestCSV);
}
public void addSource(TestLister source, Test testTask) {
allocator.addSource(source, testTask);
this.dependsOn(source);
}
public List<String> getTestsForForkAndTestTask(Integer fork, Test testTask) {
return allocator.getTestsForForkAndTestTask(fork, testTask);
}
@TaskAction
public void allocate() {
allocator.generateTestPlan();
}
public static List<Tuple2<String, Double>> fromCSV(Reader reader) throws IOException {
String name = "Test Name";
String duration = "Duration(ms)";
List<CSVRecord> records = CSVFormat.DEFAULT.withHeader().parse(reader).getRecords();
return records.stream().map(record -> {
try{
String testName = record.get(name);
String testDuration = record.get(duration);
return new Tuple2<>(testName, Math.max(Double.parseDouble(testDuration), 10));
}catch (IllegalArgumentException | IllegalStateException e){
return null;
}
}).filter(Objects::nonNull).sorted(Comparator.comparing(Tuple2::getFirst)).collect(Collectors.toList());
}
}

39
buildSrc/src/main/groovy/net/corda/testing/DistributedTesting.groovy
View File

@ -10,6 +10,7 @@ import org.gradle.api.tasks.testing.Test
*/
class DistributedTesting implements Plugin<Project> {
static def getPropertyAsInt(Project proj, String property, Integer defaultValue) {
return proj.hasProperty(property) ? Integer.parseInt(proj.property(property).toString()) : defaultValue
}
@ -17,10 +18,16 @@ class DistributedTesting implements Plugin<Project> {
@Override
void apply(Project project) {
if (System.getProperty("kubenetize") != null) {
def forks = getPropertyAsInt(project, "dockerForks", 1)
ensureImagePluginIsApplied(project)
ImageBuilding imagePlugin = project.plugins.getPlugin(ImageBuilding)
DockerPushImage imageBuildingTask = imagePlugin.pushTask
String providedTag = System.getProperty("docker.tag")
BucketingAllocatorTask globalAllocator = project.tasks.create("bucketingAllocator", BucketingAllocatorTask, forks)
def requestedTasks = project.gradle.startParameter.taskNames.collect { project.tasks.findByPath(it) }
//in each subproject
//1. add the task to determine all tests within the module
@ -28,9 +35,19 @@ class DistributedTesting implements Plugin<Project> {
//3. KubesTest will invoke these test tasks in a parallel fashion on a remote k8s cluster
project.subprojects { Project subProject ->
subProject.tasks.withType(Test) { Test task ->
ListTests testListerTask = createTestListingTasks(task, subProject)
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, testListerTask)
KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
println "Evaluating ${task.getPath()}"
if (task in requestedTasks && !task.hasProperty("ignoreForDistribution")) {
println "Modifying ${task.getPath()}"
ListTests testListerTask = createTestListingTasks(task, subProject)
globalAllocator.addSource(testListerTask, task)
Test modifiedTestTask = modifyTestTaskForParallelExecution(subProject, task, globalAllocator)
} else {
println "Skipping modification of ${task.getPath()} as it's not scheduled for execution"
}
if (!task.hasProperty("ignoreForDistribution")) {
KubesTest parallelTestTask = generateParallelTestingTask(subProject, task, imageBuildingTask, providedTag)
}
}
}
@ -62,6 +79,7 @@ class DistributedTesting implements Plugin<Project> {
taskToExecuteName = testGrouping.groups.join("And")
memoryGbPerFork = testGrouping.gbOfMemory
numberOfCoresPerFork = testGrouping.coresToUse
distribution = testGrouping.distribution
doFirst {
dockerTag = dockerTag = providedTag ? ImageBuilding.registryName + ":" + providedTag : (imageBuildingTask.imageName.get() + ":" + imageBuildingTask.tag.get())
}
@ -101,24 +119,21 @@ class DistributedTesting implements Plugin<Project> {
return createdParallelTestTask as KubesTest
}
private Test modifyTestTaskForParallelExecution(Project subProject, Test task, ListTests testListerTask) {
subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${testListerTask.getPath()}")
private Test modifyTestTaskForParallelExecution(Project subProject, Test task, BucketingAllocatorTask globalAllocator) {
subProject.logger.info("modifying task: ${task.getPath()} to depend on task ${globalAllocator.getPath()}")
def reportsDir = new File(new File(subProject.rootProject.getBuildDir(), "test-reports"), subProject.name + "-" + task.name)
task.configure {
dependsOn testListerTask
dependsOn globalAllocator
binResultsDir new File(reportsDir, "binary")
reports.junitXml.destination new File(reportsDir, "xml")
maxHeapSize = "6g"
doFirst {
filter {
def fork = getPropertyAsInt(subProject, "dockerFork", 0)
def forks = getPropertyAsInt(subProject, "dockerForks", 1)
def shuffleSeed = 42
subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (${fork}, ${forks}, ${shuffleSeed})")
List<String> includes = testListerTask.getTestsForFork(
subProject.logger.info("requesting tests to include in testing task ${task.getPath()} (idx: ${fork})")
List<String> includes = globalAllocator.getTestsForForkAndTestTask(
fork,
forks,
shuffleSeed)
task)
subProject.logger.info "got ${includes.size()} tests to include into testing task ${task.getPath()}"
if (includes.size() == 0) {

2
buildSrc/src/main/groovy/net/corda/testing/ImageBuilding.groovy
View File

@ -48,6 +48,8 @@ class ImageBuilding implements Plugin<Project> {
if (!mavenDir.exists()) {
mavenDir.mkdirs()
}
logger.info("Will use: ${gradleDir.absolutePath} for caching gradle artifacts")
}
dependsOn buildDockerImageForSource

17
buildSrc/src/main/groovy/net/corda/testing/KubesTest.groovy
View File

@ -39,6 +39,8 @@ class KubesTest extends DefaultTask {
int numberOfPods = 20
int timeoutInMinutesForPodToStart = 60
Distribution distribution = Distribution.METHOD
@TaskAction
void runTestsOnKubes() {
@ -78,7 +80,8 @@ class KubesTest extends DefaultTask {
}
List<CompletableFuture<KubePodResult>> futures = IntStream.range(0, numberOfPods).mapToObj({ i ->
String podName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
String potentialPodName = (taskToExecuteName + "-" + stableRunId + suffix + i).toLowerCase()
String podName = potentialPodName.substring(0, Math.min(potentialPodName.size(), 62))
runBuild(client, namespace, numberOfPods, i, podName, printOutput, 3)
}).collect(Collectors.toList())
this.testOutput = Collections.synchronizedList(futures.collect { it -> it.get().binaryResults }.flatten())
@ -115,10 +118,11 @@ class KubesTest extends DefaultTask {
CompletableFuture<KubePodResult> waiter = new CompletableFuture<>()
ExecListener execListener = buildExecListenerForPod(podName, errChannelStream, waiter, result)
stdOutIs.connect(stdOutOs)
String[] buildCommand = getBuildCommand(numberOfPods, podIdx)
ExecWatch execWatch = client.pods().inNamespace(namespace).withName(podName)
.writingOutput(stdOutOs)
.writingErrorChannel(errChannelStream)
.usingListener(execListener).exec(getBuildCommand(numberOfPods, podIdx))
.usingListener(execListener).exec(buildCommand)
startLogPumping(outputFile, stdOutIs, podIdx, printOutput)
KubePodResult execResult = waiter.join()
@ -183,20 +187,21 @@ class KubesTest extends DefaultTask {
ExecListener buildExecListenerForPod(podName, errChannelStream, CompletableFuture<KubePodResult> waitingFuture, KubePodResult result) {
new ExecListener() {
final Long start = System.currentTimeMillis()
@Override
void onOpen(Response response) {
project.logger.lifecycle("Build started on pod " + podName)
project.logger.lifecycle("Build started on pod $podName")
}
@Override
void onFailure(Throwable t, Response response) {
project.logger.lifecycle("Received error from rom pod " + podName)
project.logger.lifecycle("Received error from rom pod $podName")
waitingFuture.completeExceptionally(t)
}
@Override
void onClose(int code, String reason) {
project.logger.lifecycle("Received onClose() from pod " + podName + " with returnCode=" + code)
project.logger.lifecycle("Received onClose() from pod ${podName}, build took: ${(System.currentTimeMillis() - start) / 1000} seconds")
try {
def errChannelContents = errChannelStream.toString()
Status status = Serialization.unmarshal(errChannelContents, Status.class);
@ -277,7 +282,7 @@ class KubesTest extends DefaultTask {
"let x=1 ; while [ \${x} -ne 0 ] ; do echo \"Waiting for DNS\" ; curl services.gradle.org > /dev/null 2>&1 ; x=\$? ; sleep 1 ; done ; " +
"cd /tmp/source ; " +
"let y=1 ; while [ \${y} -ne 0 ] ; do echo \"Preparing build directory\" ; ./gradlew testClasses integrationTestClasses --parallel 2>&1 ; y=\$? ; sleep 1 ; done ;" +
"./gradlew -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
"./gradlew -D${ListTests.DISTRIBUTION_PROPERTY}=${distribution.name()} -Dkubenetize -PdockerFork=" + podIdx + " -PdockerForks=" + numberOfPods + " $fullTaskToExecutePath --info 2>&1 ;" +
"let rs=\$? ; sleep 10 ; exit \${rs}"]
}

70
buildSrc/src/main/groovy/net/corda/testing/ListTests.groovy
View File

@ -37,10 +37,17 @@ class ListShufflerAndAllocator {
}
}
class ListTests extends DefaultTask {
interface TestLister {
List<String> getAllTestsDiscovered()
}
class ListTests extends DefaultTask implements TestLister {
public static final String DISTRIBUTION_PROPERTY = "distributeBy"
FileCollection scanClassPath
List<String> allTests
Distribution distribution = System.getProperty(DISTRIBUTION_PROPERTY) ? Distribution.valueOf(System.getProperty(DISTRIBUTION_PROPERTY)) : Distribution.METHOD
def getTestsForFork(int fork, int forks, Integer seed) {
def gitSha = new BigInteger(project.hasProperty("corda_revision") ? project.property("corda_revision").toString() : "0", 36)
@ -51,24 +58,53 @@ class ListTests extends DefaultTask {
return new ListShufflerAndAllocator(allTests).getTestsForFork(fork, forks, seedToUse)
}
@Override
public List<String> getAllTestsDiscovered() {
return new ArrayList<>(allTests)
}
@TaskAction
def discoverTests() {
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c ->
c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
}.flatten()
.toSet()
switch (distribution) {
case Distribution.METHOD:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c ->
c.getMethodInfo().filter { m -> m.hasAnnotation("org.junit.Test") }.collect { m -> c.name + "." + m.name + "*" }
}.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
case Distribution.CLASS:
Collection<String> results = new ClassGraph()
.enableClassInfo()
.enableMethodInfo()
.ignoreClassVisibility()
.ignoreMethodVisibility()
.enableAnnotationInfo()
.overrideClasspath(scanClassPath)
.scan()
.getClassesWithMethodAnnotation("org.junit.Test")
.collect { c -> (c.getSubclasses() + Collections.singletonList(c)) }
.flatten()
.collect { ClassInfo c -> c.name + "*" }.flatten()
.toSet()
this.allTests = results.stream().sorted().collect(Collectors.toList())
break
}
}
}
public enum Distribution {
CLASS, METHOD
}

14
buildSrc/src/main/groovy/net/corda/testing/ParallelTestGroup.groovy
View File

@ -1,9 +1,9 @@
package net.corda.testing
import org.gradle.api.DefaultTask
import org.gradle.api.tasks.TaskAction
class ParallelTestGroup extends DefaultTask {
Distribution distribution = Distribution.METHOD
List<String> groups = new ArrayList<>()
int shardCount = 20
@ -11,20 +11,24 @@ class ParallelTestGroup extends DefaultTask {
int gbOfMemory = 4
boolean printToStdOut = true
void numberOfShards(int shards){
void numberOfShards(int shards) {
this.shardCount = shards
}
void coresPerFork(int cores){
void distribute(Distribution dist){
this.distribution = dist
}
void coresPerFork(int cores) {
this.coresToUse = cores
}
void memoryInGbPerFork(int gb){
void memoryInGbPerFork(int gb) {
this.gbOfMemory = gb
}
//when this is false, only containers will "failed" exit codes will be printed to stdout
void streamOutput(boolean print){
void streamOutput(boolean print) {
this.printToStdOut = print
}

54
buildSrc/src/test/java/net/corda/testing/BucketingAllocatorTest.java Normal file
View File

@ -0,0 +1,54 @@
package net.corda.testing;
import org.hamcrest.collection.IsIterableContainingInAnyOrder;
import org.junit.Assert;
import org.junit.Test;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.hamcrest.CoreMatchers.is;
public class BucketingAllocatorTest {
@Test
public void shouldAlwaysBucketTestsEvenIfNotInTimedFile() {
BucketingAllocator bucketingAllocator = new BucketingAllocator(1, Collections::emptyList);
Object task = new Object();
bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
bucketingAllocator.generateTestPlan();
List<String> testsForForkAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
Assert.assertThat(testsForForkAndTestTask, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
}
@Test
public void shouldAllocateTestsAcrossForksEvenIfNoMatchingTestsFound() {
BucketingAllocator bucketingAllocator = new BucketingAllocator(2, Collections::emptyList);
Object task = new Object();
bucketingAllocator.addSource(() -> Arrays.asList("SomeTestingClass*", "AnotherTestingClass*"), task);
bucketingAllocator.generateTestPlan();
List<String> testsForForkOneAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(0, task);
List<String> testsForForkTwoAndTestTask = bucketingAllocator.getTestsForForkAndTestTask(1, task);
Assert.assertThat(testsForForkOneAndTestTask.size(), is(1));
Assert.assertThat(testsForForkTwoAndTestTask.size(), is(1));
List<String> allTests = Stream.of(testsForForkOneAndTestTask, testsForForkTwoAndTestTask).flatMap(Collection::stream).collect(Collectors.toList());
Assert.assertThat(allTests, IsIterableContainingInAnyOrder.containsInAnyOrder("SomeTestingClass*", "AnotherTestingClass*"));
}
}

2
client/rpc/src/integration-test/kotlin/net/corda/client/rpc/FlowsExecutionModeRpcTest.kt
View File

@ -14,7 +14,7 @@ import org.assertj.core.api.Assertions.assertThat
import org.junit.Before
import org.junit.Test
class FlowsExecutionModeTests : NodeBasedTest(listOf("net.corda.finance.contracts", CashSchemaV1::class.packageName)) {
class FlowsExecutionModeTests : NodeBasedTest(emptyList()) {
private val rpcUser = User("user1", "test", permissions = setOf(Permissions.all()))
private lateinit var node: NodeWithInfo

9
client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt
View File

@ -17,6 +17,7 @@ import net.corda.core.context.Trace
import net.corda.core.context.Trace.InvocationId
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.*
import net.corda.core.internal.messaging.InternalCordaRPCOps
import net.corda.core.messaging.CordaRPCOps
import net.corda.core.messaging.RPCOps
import net.corda.core.serialization.SerializationContext
@ -286,8 +287,12 @@ class RPCClientProxyHandler(
}
private fun produceMethodFullyQualifiedName(method: Method) : String {
// For CordaRPCOps send method only - for backwards compatibility
return if (CordaRPCOps::class.java == rpcOpsClass) {
/*
* Until version 4.3, rpc calls did not include class names.
* Up to this version, only CordaRPCOps and InternalCordaRPCOps were supported.
* So, for these classes only methods are sent across the wire to preserve backwards compatibility.
*/
return if (CordaRPCOps::class.java == rpcOpsClass || InternalCordaRPCOps::class.java == rpcOpsClass) {
method.name
} else {
rpcOpsClass.name + CLASS_METHOD_DIVIDER + method.name

2
config/dev/log4j2.xml
View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="info" packages="net.corda.common.logging">
<Configuration status="info" packages="net.corda.common.logging" shutdownHook="disable">
<Properties>
<Property name="log-path">${sys:log-path:-logs}</Property>

4
constants.properties
View File

@ -3,7 +3,7 @@
# their own projects. So don't get fancy with syntax!
cordaVersion=4.4-SNAPSHOT
gradlePluginsVersion=5.0.3
gradlePluginsVersion=5.0.4
kotlinVersion=1.2.71
java8MinUpdateVersion=171
# ***************************************************************#
@ -17,6 +17,8 @@ quasarVersion=0.7.10
quasarClassifier=jdk8
# Quasar version to use with Java 11:
quasarVersion11=0.8.0
# Specify a classifier for Java 11 built artifacts
jdkClassifier11=jdk11
proguardVersion=6.1.1
bouncycastleVersion=1.60
classgraphVersion=4.8.41

3
core-deterministic/testing/data/build.gradle
View File

@ -22,6 +22,9 @@ dependencies {
jar.enabled = false
test {
ext {
ignoreForDistribution = true
}
filter {
// Running this class is the whole point, so include it explicitly.
includeTestsMatching "net.corda.deterministic.data.GenerateData"

35
core/src/main/kotlin/net/corda/core/internal/cordapp/CordappResolver.kt
View File

@ -15,6 +15,8 @@ object CordappResolver {
private val logger = loggerFor<CordappResolver>()
private val cordappClasses: ConcurrentHashMap<String, Set<Cordapp>> = ConcurrentHashMap()
private val insideInMemoryTest: Boolean by lazy { insideInMemoryTest() }
// TODO Use the StackWalker API once we migrate to Java 9+
private var cordappResolver: () -> Cordapp? = {
Exception().stackTrace
@ -25,9 +27,12 @@ object CordappResolver {
?.single()
}
/*
/**
* Associates class names with CorDapps or logs a warning when a CorDapp is already registered for a given class.
* This could happen when trying to run different versions of the same CorDapp on the same node.
*
* @throws IllegalStateException when multiple CorDapps are registered for the same contract class,
* since this can lead to undefined behaviour.
*/
@Synchronized
fun register(cordapp: Cordapp) {
@ -39,12 +44,30 @@ object CordappResolver {
notAlreadyRegisteredClasses.forEach { cordappClasses[it] = setOf(cordapp) }
for ((className, registeredCordapps) in alreadyRegistered) {
if (registeredCordapps.any { it.jarHash == cordapp.jarHash }) continue
if (className in contractClasses) {
logger.error("ATTENTION: More than one CorDapp installed on the node for contract $className. Please remove the previous version when upgrading to a new version.")
for ((registeredClassName, registeredCordapps) in alreadyRegistered) {
val duplicateCordapps = registeredCordapps.filter { it.jarHash == cordapp.jarHash }.toSet()
if (duplicateCordapps.isNotEmpty()) {
logger.warnOnce("The CorDapp (name: ${cordapp.info.shortName}, file: ${cordapp.name}) " +
"is installed multiple times on the node. The following files correspond to the exact same content: " +
"${duplicateCordapps.map { it.name }}")
continue
}
cordappClasses[className] = registeredCordapps + cordapp
// During in-memory tests, the spawned nodes share the same CordappResolver, so detected conflicts can be spurious.
if (registeredClassName in contractClasses && !insideInMemoryTest) {
throw IllegalStateException("More than one CorDapp installed on the node for contract $registeredClassName. " +
"Please remove the previous version when upgrading to a new version.")
}
cordappClasses[registeredClassName] = registeredCordapps + cordapp
}
}
private fun insideInMemoryTest(): Boolean {
return Exception().stackTrace.any {
it.className.startsWith("net.corda.testing.node.internal.InternalMockNetwork") ||
it.className.startsWith("net.corda.testing.node.internal.InProcessNode") ||
it.className.startsWith("net.corda.testing.node.MockServices")
}
}

31
core/src/test/kotlin/net/corda/core/internal/cordapp/CordappResolverTest.kt
View File

@ -2,9 +2,11 @@ package net.corda.core.internal.cordapp
import net.corda.core.crypto.SecureHash
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.After
import org.junit.Before
import org.junit.Test
import java.lang.IllegalStateException
import kotlin.test.assertEquals
class CordappResolverTest {
@ -49,19 +51,42 @@ class CordappResolverTest {
}
@Test
fun `when different cordapps are registered for the same class, the resolver returns null`() {
fun `when different cordapps are registered for the same (non-contract) class, the resolver returns null`() {
CordappResolver.register(CordappImpl.TEST_INSTANCE.copy(
contractClassNames = listOf(javaClass.name),
contractClassNames = listOf("ContractClass1"),
minimumPlatformVersion = 3,
targetPlatformVersion = 222,
jarHash = SecureHash.randomSHA256()
))
CordappResolver.register(CordappImpl.TEST_INSTANCE.copy(
contractClassNames = listOf(javaClass.name),
contractClassNames = listOf("ContractClass2"),
minimumPlatformVersion = 2,
targetPlatformVersion = 456,
jarHash = SecureHash.randomSHA256()
))
assertThat(CordappResolver.currentCordapp).isNull()
}
@Test
fun `when different cordapps are registered for the same (contract) class, the resolver throws an exception`() {
val firstCordapp = CordappImpl.TEST_INSTANCE.copy(
contractClassNames = listOf(javaClass.name),
minimumPlatformVersion = 3,
targetPlatformVersion = 222,
jarHash = SecureHash.randomSHA256()
)
val secondCordapp = CordappImpl.TEST_INSTANCE.copy(
contractClassNames = listOf(javaClass.name),
minimumPlatformVersion = 2,
targetPlatformVersion = 456,
jarHash = SecureHash.randomSHA256()
)
CordappResolver.register(firstCordapp)
assertThatThrownBy { CordappResolver.register(secondCordapp) }
.isInstanceOf(IllegalStateException::class.java)
.hasMessageContaining("More than one CorDapp installed on the node for contract ${javaClass.name}. " +
"Please remove the previous version when upgrading to a new version.")
}
}

18
detekt-baseline.xml
View File

@ -184,7 +184,6 @@
<ID>ComplexMethod:NodeVaultService.kt$NodeVaultService$private fun makeUpdates(batch: Iterable&lt;CoreTransaction&gt;, statesToRecord: StatesToRecord, previouslySeen: Boolean): List&lt;Vault.Update&lt;ContractState&gt;&gt;</ID>
<ID>ComplexMethod:ObjectDiffer.kt$ObjectDiffer$fun diff(a: Any?, b: Any?): DiffTree?</ID>
<ID>ComplexMethod:Obligation.kt$Obligation$override fun verify(tx: LedgerTransaction)</ID>
<ID>ComplexMethod:PersistentNetworkMapCache.kt$PersistentNetworkMapCache$override fun addNodes(nodes: List&lt;NodeInfo&gt;)</ID>
<ID>ComplexMethod:QuasarInstrumentationHook.kt$QuasarInstrumentationHookAgent.Companion$@JvmStatic fun premain(argumentsString: String?, instrumentation: Instrumentation)</ID>
<ID>ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$ private fun close(notify: Boolean = true)</ID>
<ID>ComplexMethod:RPCClientProxyHandler.kt$RPCClientProxyHandler$// The handler for Artemis messages. private fun artemisMessageHandler(message: ClientMessage)</ID>
@ -3021,18 +3020,6 @@
<ID>MaxLineLength:PersistentIdentityMigrationNewTable.kt$PersistentIdentityMigrationNewTable$throw PersistentIdentitiesMigrationException("Cannot migrate persistent identities as liquibase failed to provide a suitable database connection")</ID>
<ID>MaxLineLength:PersistentIdentityMigrationNewTableTest.kt$PersistentIdentityMigrationNewTableTest$session.save(PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded))</ID>
<ID>MaxLineLength:PersistentIdentityMigrationNewTableTest.kt$PersistentIdentityMigrationNewTableTest$val identityService = makeTestIdentityService(PersistentIdentityMigrationNewTableTest.dummyNotary.identity, BOB_IDENTITY, ALICE_IDENTITY)</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService$ @Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class) private fun verifyAndRegisterIdentity(trustAnchor: TrustAnchor, identity: PartyAndCertificate): PartyAndCertificate?</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService$// Allows us to eliminate keys we know belong to others by using the cache contents that might have been seen during other identity activity. // Concentrating activity on the identity cache works better than spreading checking across identity and key management, because we cache misses too. fun stripNotOurKeys(keys: Iterable&lt;PublicKey&gt;): Iterable&lt;PublicKey&gt;</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService$@Throws(UnknownAnonymousPartyException::class) override</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService$fun loadIdentities(identities: Collection&lt;PartyAndCertificate&gt; = emptySet(), confidentialIdentities: Collection&lt;PartyAndCertificate&gt; = emptySet())</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService$log.warn("Certificate validation failed for ${identity.name} against trusted root ${trustAnchor.trustedCert.subjectX500Principal}.")</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService${ // If there is no entry in the legal keyToPartyAndCert table then the party must be a confidential identity so we perform // a lookup in the keyToName table. If an entry for that public key exists, then we attempt look up the associated node's // PartyAndCertificate. val name = keyToName[party.owningKey.toStringShort()] if (name != null) { // This should never return null as this node would not be able to communicate with the node providing a confidential // identity unless its NodeInfo/PartyAndCertificate were available. wellKnownPartyFromX500Name(name) } else { null } }</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService${ // This should never return null as this node would not be able to communicate with the node providing a confidential // identity unless its NodeInfo/PartyAndCertificate were available. wellKnownPartyFromX500Name(name) }</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService${ // Update the three tables as necessary. We definitely store the public key and map it to a party and we optionally update // the public key to external ID mapping table. This block will only ever be reached when registering keys generated on // other because when a node generates its own keys "registerKeyToParty" is automatically called by KeyManagementService.freshKey. registerKeyToParty(publicKey, party) hashToKey[publicKeyHash] = publicKey if (externalId != null) { registerKeyToExternalId(publicKey, externalId) } }</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService.Companion$fun createHashToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap&lt;String, PublicKey, PersistentHashToPublicKey, String&gt;</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService.Companion$fun createKeyToPartyAndCertMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap&lt;String, PartyAndCertificate, PersistentPublicKeyHashToCertificate, String&gt;</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService.Companion$fun createKeyToX500Map(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap&lt;String, CordaX500Name, PersistentPublicKeyHashToParty, String&gt;</ID>
<ID>MaxLineLength:PersistentIdentityService.kt$PersistentIdentityService.Companion$fun createX500ToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap&lt;CordaX500Name, String, PersistentPartyToPublicKeyHash, String&gt;</ID>
<ID>MaxLineLength:PersistentIdentityServiceTests.kt$PersistentIdentityServiceTests$listOf("Organisation A", "Organisation B", "Organisation C") .map { getTestPartyAndCertificate(CordaX500Name(organisation = it, locality = "London", country = "GB"), generateKeyPair().public) }</ID>
<ID>MaxLineLength:PersistentIdentityServiceTests.kt$PersistentIdentityServiceTests$val alicente = getTestPartyAndCertificate(CordaX500Name(organisation = "Alicente Worldwide", locality = "London", country = "GB"), generateKeyPair().public)</ID>
<ID>MaxLineLength:PersistentMap.kt$PersistentMap$ExplicitRemoval&lt;K, V, E, EK&gt; : RemovalListener</ID>
@ -3042,6 +3029,7 @@
<ID>MaxLineLength:PersistentNetworkMapCache.kt$PersistentNetworkMapCache$"SELECT n FROM ${NodeInfoSchemaV1.PersistentNodeInfo::class.java.name} n JOIN n.legalIdentitiesAndCerts l WHERE l.name = :name"</ID>
<ID>MaxLineLength:PersistentNetworkMapCache.kt$PersistentNetworkMapCache$"SELECT n FROM ${NodeInfoSchemaV1.PersistentNodeInfo::class.java.name} n JOIN n.legalIdentitiesAndCerts l WHERE l.owningKeyHash = :owningKeyHash"</ID>
<ID>MaxLineLength:PersistentNetworkMapCache.kt$PersistentNetworkMapCache$val info = findByIdentityKey(session, nodeInfo.legalIdentitiesAndCerts.first().owningKey).singleOrNull { it.serial == nodeInfo.serial }</ID>
<ID>MaxLineLength:PersistentNetworkMapCache.kt$PersistentNetworkMapCache$val newNodes = mutableListOf&lt;NodeInfo&gt;() val updatedNodes = mutableListOf&lt;Pair&lt;NodeInfo, NodeInfo&gt;&gt;() nodes.map { it to getNodesByLegalIdentityKey(it.legalIdentities.first().owningKey).firstOrNull() } .forEach { (node, previousNode) -&gt; when { previousNode == null -&gt; { logger.info("No previous node found for ${node.legalIdentities.first().name}") if (verifyAndRegisterIdentities(node)) { newNodes.add(node) } } previousNode.serial &gt; node.serial -&gt; { logger.info("Discarding older nodeInfo for ${node.legalIdentities.first().name}") } previousNode != node -&gt; { logger.info("Previous node was found for ${node.legalIdentities.first().name} as: $previousNode") // TODO We should be adding any new identities as well if (verifyIdentities(node)) { updatedNodes.add(node to previousNode) } } else -&gt; logger.info("Previous node was identical to incoming one - doing nothing") } } /** * This algorithm protects against database failure (eg. attempt to persist a nodeInfo entry larger than permissible by the * database X500Name) without sacrificing performance incurred by attempting to flush nodeInfo's individually. * Upon database transaction failure, the list of new nodeInfo's is split in half, and then each half is persisted independently. * This continues recursively until all valid nodeInfo's are persisted, and failed ones reported as warnings. */ recursivelyUpdateNodes(newNodes.map { nodeInfo -&gt; Pair(nodeInfo, MapChange.Added(nodeInfo)) } + updatedNodes.map { (nodeInfo, previousNodeInfo) -&gt; Pair(nodeInfo, MapChange.Modified(nodeInfo, previousNodeInfo)) })</ID>
<ID>MaxLineLength:PersistentNetworkMapCacheTest.kt$PersistentNetworkMapCacheTest$private val charlieNetMapCache = PersistentNetworkMapCache(TestingNamedCacheFactory(), database, InMemoryIdentityService(trustRoot = DEV_ROOT_CA.certificate))</ID>
<ID>MaxLineLength:PersistentScheduledFlowRepository.kt$PersistentScheduledFlowRepository$private</ID>
<ID>MaxLineLength:PersistentScheduledFlowRepository.kt$PersistentScheduledFlowRepository$return Pair(StateRef(SecureHash.parse(txId), index), ScheduledStateRef(StateRef(SecureHash.parse(txId), index), scheduledStateRecord.scheduledAt))</ID>
@ -4224,7 +4212,7 @@
<ID>TooGenericExceptionCaught:InternalUtils.kt$ex: Exception</ID>
<ID>TooGenericExceptionCaught:InternalUtils.kt$th: Throwable</ID>
<ID>TooGenericExceptionCaught:IssueCash.kt$IssueCash$e: Exception</ID>
<ID>TooGenericExceptionCaught:JVMAgentUtil.kt$JVMAgentUtil$e: Exception</ID>
<ID>TooGenericExceptionCaught:JVMAgentUtil.kt$JVMAgentUtil$e: Throwable</ID>
<ID>TooGenericExceptionCaught:JacksonSupport.kt$JacksonSupport.PartyDeserializer$e: Exception</ID>
<ID>TooGenericExceptionCaught:JacksonSupport.kt$JacksonSupport.PublicKeyDeserializer$e: Exception</ID>
<ID>TooGenericExceptionCaught:JacksonSupport.kt$JacksonSupport.SecureHashDeserializer$e: Exception</ID>
@ -4393,7 +4381,7 @@
<ID>TooManyFunctions:P2PMessagingClient.kt$P2PMessagingClient : SingletonSerializeAsTokenMessagingServiceAddressToArtemisQueueResolver</ID>
<ID>TooManyFunctions:PathUtils.kt$net.corda.core.internal.PathUtils.kt</ID>
<ID>TooManyFunctions:Perceivable.kt$net.corda.finance.contracts.universal.Perceivable.kt</ID>
<ID>TooManyFunctions:PersistentIdentityService.kt$PersistentIdentityService : SingletonSerializeAsTokenIdentityService</ID>
<ID>TooManyFunctions:PersistentIdentityService.kt$PersistentIdentityService : SingletonSerializeAsTokenIdentityServiceInternal</ID>
<ID>TooManyFunctions:PersistentNetworkMapCache.kt$PersistentNetworkMapCache : NetworkMapCacheInternalSingletonSerializeAsToken</ID>
<ID>TooManyFunctions:PortfolioApi.kt$PortfolioApi</ID>
<ID>TooManyFunctions:PropertyDescriptor.kt$net.corda.serialization.internal.amqp.PropertyDescriptor.kt</ID>

6
docker/src/bash/generate-config.sh
View File

@ -19,8 +19,8 @@ function generateTestnetConfig() {
MY_P2P_PORT=${MY_P2P_PORT} \
MY_RPC_PORT=${MY_RPC_PORT} \
MY_RPC_ADMIN_PORT=${MY_RPC_ADMIN_PORT} \
NETWORKMAP_URL='https://map.testnet.corda.network' \
DOORMAN_URL='https://doorman.testnet.corda.network' \
NETWORKMAP_URL='https://netmap.testnet.r3.com' \
DOORMAN_URL='https://doorman.testnet.r3.com/' \
java -jar config-exporter.jar "TEST-NET-COMBINE" "node.conf" "/opt/corda/starting-node.conf" "${CONFIG_FOLDER}/node.conf"
}
@ -62,7 +62,7 @@ function downloadTestnetCerts() {
: ${COUNTRY:? '$COUNTRY (the country used when registering for Testnet) must be set as environment variable'}
curl -L -d "{\"x500Name\":{\"locality\":\"${LOCALITY}\", \"country\":\"${COUNTRY}\"}, \"configType\": \"INSTALLSCRIPT\", \"include\": { \"systemdServices\": false, \"cordapps\": false, \"cordaJar\": false, \"cordaWebserverJar\": false, \"scripts\": false} }" \
-H 'Content-Type: application/json' \
-X POST "https://testnet.corda.network/api/user/node/generate/one-time-key/redeem/$ONE_TIME_DOWNLOAD_KEY" \
-X POST "https://onboarder.prod.ws.r3.com/api/user/node/generate/one-time-key/redeem/$ONE_TIME_DOWNLOAD_KEY" \
-o "${CERTIFICATES_FOLDER}/certs.zip"
fi
rm -rf ${CERTIFICATES_FOLDER}/*.jks

8
docs/source/corda-configuration-file.rst
View File

@ -28,6 +28,11 @@ This prevents configuration errors when mixing keys containing ``.`` wrapped wit
``"dataSourceProperties.dataSourceClassName" = "val"`` in `Reference.conf`_ would be not overwritten by the property
``dataSourceProperties.dataSourceClassName = "val2"`` in *node.conf*.
.. warning:: If a property is defined twice the last one will take precedence. The library currently used for parsing HOCON
currently does not provide a way to catch duplicates when parsing files and will silently override values for the same key.
For example having ``key=initialValue`` defined first in node.conf and later on down the
lines ``key=overridingValue`` will result into the value being ``overridingValue``.
By default the node will fail to start in presence of unknown property keys.
To alter this behaviour, the ``on-unknown-config-keys`` command-line argument can be set to ``IGNORE`` (default is ``FAIL``).
@ -46,6 +51,9 @@ JVM options
java -Dcorda.rpcSettings.ssl.keyStorePassword=mypassword -jar node.jar
.. note:: If the same field is overriden by both an environment variable and system property, the system property
takes precedence.
Configuration file fields
-------------------------

5
docs/source/node-administration.rst
View File

@ -25,6 +25,11 @@ If you need to switch to synchronous logging (e.g. for debugging/testing purpose
by adding ``-DLog4jContextSelector=org.apache.logging.log4j.core.selector.ClassLoaderContextSelector`` to the node's
command line or to the ``jvmArgs`` section of the node configuration (see :doc:`corda-configuration-file`).
.. warning:: Ensure that ``shutdownHook="disable"`` is set if you are overriding the log4j2 configuration file
otherwise logs will not be flushed properly on shutdown and loss may occur. The option is set in the ``Configuration``
tag of the log4j configuration file, for example ``<Configuration ... shutdownHook="disable">``. This is because
Corda overrides the default log4j2 shutdown logic in order to make sure it gets shut down correctly.
Example
+++++++

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/bridging/AMQPBridgeManager.kt
View File

@ -83,7 +83,7 @@ class AMQPBridgeManager(config: MutualSslConfiguration,
}
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
try {
MDC.put("queueName", queueName)
MDC.put("targets", targets.joinToString(separator = ";") { it.toString() })

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt
View File

@ -46,7 +46,7 @@ internal class ConnectionStateMachine(private val serverMode: Boolean,
}
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
try {
MDC.put("serverMode", serverMode.toString())
MDC.put("localLegalName", localLegalName)

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/EventProcessor.kt
View File

@ -41,7 +41,7 @@ internal class EventProcessor(channel: Channel,
}
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
try {
MDC.put("serverMode", serverMode.toString())
MDC.put("localLegalName", localLegalName)

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/netty/AMQPChannelHandler.kt
View File

@ -49,7 +49,7 @@ internal class AMQPChannelHandler(private val serverMode: Boolean,
private var badCert: Boolean = false
private fun withMDC(block: () -> Unit) {
val oldMDC = MDC.getCopyOfContextMap()
val oldMDC = MDC.getCopyOfContextMap() ?: emptyMap<String, String>()
try {
MDC.put("serverMode", serverMode.toString())
MDC.put("remoteAddress", remoteAddress.toString())

5
node/capsule/build.gradle
View File

@ -36,7 +36,10 @@ capsule {
task buildCordaJAR(type: FatCapsule, dependsOn: project(':node').tasks.jar) {
applicationClass 'net.corda.node.Corda'
archiveName "corda-${corda_release_version}.jar"
archiveBaseName = 'corda'
archiveVersion = corda_release_version
archiveClassifier = jdkClassifier
archiveName = archiveFileName.get()
applicationSource = files(
project(':node').configurations.runtimeClasspath,
project(':node').tasks.jar,

9
node/src/integration-test/kotlin/net/corda/node/AddressBindingFailureTests.kt
View File

@ -43,7 +43,8 @@ class AddressBindingFailureTests {
driver(DriverParameters(startNodesInProcess = false,
notarySpecs = listOf(NotarySpec(notaryName)),
notaryCustomOverrides = mapOf("p2pAddress" to address.toString()),
portAllocation = portAllocation)
portAllocation = portAllocation,
cordappsForAllNodes = emptyList())
) {} }.isInstanceOfSatisfying(IllegalStateException::class.java) { error ->
assertThat(error.message).contains("Unable to start notaries")
@ -56,7 +57,11 @@ class AddressBindingFailureTests {
ServerSocket(0).use { socket ->
val address = InetSocketAddress("localhost", socket.localPort).toNetworkHostAndPort()
driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList(), inMemoryDB = false, portAllocation = portAllocation)) {
driver(DriverParameters(startNodesInProcess = true,
notarySpecs = emptyList(),
inMemoryDB = false,
portAllocation = portAllocation,
cordappsForAllNodes = emptyList())) {
assertThatThrownBy { startNode(customOverrides = overrides(address)).getOrThrow() }.isInstanceOfSatisfying(AddressBindingException::class.java) { exception ->
assertThat(exception.addresses).contains(address).withFailMessage("Expected addresses to contain $address but was ${exception.addresses}.")

59
node/src/integration-test/kotlin/net/corda/node/services/network/PersistentNetworkMapCacheTest.kt
View File

@ -1,5 +1,6 @@
package net.corda.node.services.network
import net.corda.core.identity.CordaX500Name
import net.corda.core.node.NodeInfo
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.node.internal.schemas.NodeInfoSchemaV1
@ -20,6 +21,17 @@ class PersistentNetworkMapCacheTest {
private companion object {
val ALICE = TestIdentity(ALICE_NAME, 70)
val BOB = TestIdentity(BOB_NAME, 80)
val CHARLIE = TestIdentity(CHARLIE_NAME, 90)
val LONG_X500_NAME = CordaX500Name(
commonName = "AB123456789012345678901234567890123456789012345678901234567890",
organisationUnit = "AB123456789012345678901234567890123456789012345678901234567890",
organisation = "Long Plc",
locality = "AB123456789012345678901234567890123456789012345678901234567890",
state = "AB123456789012345678901234567890123456789012345678901234567890",
country= "IT")
val LONG_PLC = TestIdentity(LONG_X500_NAME, 95)
val LONGER_PLC = TestIdentity(LONG_X500_NAME.copy(organisation = "Longer Plc"), 96)
}
@Rule
@ -100,6 +112,53 @@ class PersistentNetworkMapCacheTest {
assertThat(nodeInfos).hasSize(2)
}
@Test
fun `negative test - attempt to insert invalid node info`() {
charlieNetMapCache.addNode(createNodeInfo(listOf(LONG_PLC)))
assertThat(charlieNetMapCache.allNodes).hasSize(0)
}
@Test
fun `negative test - attempt to update existing node with invalid node info`() {
charlieNetMapCache.addNode(createNodeInfo(listOf(ALICE)))
val aliceUpdate = TestIdentity(LONG_X500_NAME, ALICE.keyPair)
charlieNetMapCache.addNode(createNodeInfo(listOf(aliceUpdate)))
assertThat(charlieNetMapCache.allNodes).hasSize(1)
assertThat(charlieNetMapCache.getNodeByLegalName(ALICE_NAME)).isNotNull
assertThat(charlieNetMapCache.getNodeByLegalName(LONG_X500_NAME)).isNull()
}
@Test
fun `negative test - insert two valid node infos and one invalid one`() {
charlieNetMapCache.addNodes(listOf(createNodeInfo(listOf(ALICE)),
createNodeInfo(listOf(BOB)),
createNodeInfo(listOf(LONG_PLC))))
assertThat(charlieNetMapCache.allNodes).hasSize(2)
assertThat(charlieNetMapCache.allNodes.flatMap { it.legalIdentities }).isEqualTo(listOf(ALICE.party, BOB.party))
}
@Test
fun `negative test - insert three valid node infos and two invalid ones`() {
charlieNetMapCache.addNodes(listOf(createNodeInfo(listOf(LONG_PLC)),
createNodeInfo(listOf(ALICE)),
createNodeInfo(listOf(BOB)),
createNodeInfo(listOf(CHARLIE)),
createNodeInfo(listOf(LONGER_PLC))))
assertThat(charlieNetMapCache.allNodes).hasSize(3)
assertThat(charlieNetMapCache.allNodes.flatMap { it.legalIdentities }).isEqualTo(listOf(ALICE.party, BOB.party, CHARLIE.party))
}
@Test
fun `negative test - insert one valid node info then attempt to add one invalid node info and update the existing valid nodeinfo`() {
charlieNetMapCache.addNode(createNodeInfo(listOf(ALICE)))
val aliceUpdate = TestIdentity(LONG_X500_NAME, ALICE.keyPair)
charlieNetMapCache.addNodes(listOf(createNodeInfo(listOf(aliceUpdate)),
createNodeInfo(listOf(LONGER_PLC)), createNodeInfo(listOf(BOB))))
assertThat(charlieNetMapCache.allNodes).hasSize(2)
assertThat(charlieNetMapCache.getNodeByLegalName(ALICE_NAME)).isNotNull
assertThat(charlieNetMapCache.getNodeByLegalName(BOB_NAME)).isNotNull
}
private fun createNodeInfo(identities: List<TestIdentity>,
address: NetworkHostAndPort = NetworkHostAndPort("localhost", portCounter++)): NodeInfo {
return NodeInfo(

13
node/src/main/kotlin/net/corda/node/internal/DbExceptionHandler.kt → node/src/main/kotlin/net/corda/node/internal/GeneralExceptionHandler.kt
View File

@ -5,14 +5,21 @@ import org.slf4j.LoggerFactory
import java.sql.SQLException
/**
* If a thread dies because it can't connect to the database, the node ends up in an inconsistent state.
*
* Cater for all type of unrecoverable [VirtualMachineError] in which the node may end up in an inconsistent state.
* Fail fast and hard.
*/
class DbExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
class GeneralExceptionHandler(private val parentHandler: Thread.UncaughtExceptionHandler? = null) : Thread.UncaughtExceptionHandler {
override fun uncaughtException(t: Thread?, e: Throwable?) {
// fail fast with minimal overhead and further processing
if (e is VirtualMachineError) {
System.err.println("${e.message}")
Runtime.getRuntime().halt(1)
}
// the error is a database connection issue - pull the rug
if (e is Error && e.cause is SQLException) {
else if (e is Error && e.cause is SQLException) {
errorAndTerminate("Thread ${t!!.name} failed due to database connection error. This is unrecoverable, terminating node.", e)
}

5
node/src/main/kotlin/net/corda/node/internal/Node.kt
View File

@ -462,11 +462,10 @@ open class Node(configuration: NodeConfiguration,
}
/**
* Register a default exception handler for all threads that terminates the process if the database connection goes away and
* cannot be recovered.
* Register a default exception handler for all threads that terminate the process due to an unrecoverable Virtual Machine error.
*/
private fun registerDefaultExceptionHandler() {
Thread.setDefaultUncaughtExceptionHandler(DbExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
Thread.setDefaultUncaughtExceptionHandler(GeneralExceptionHandler(Thread.getDefaultUncaughtExceptionHandler()))
}
/**

22
node/src/main/kotlin/net/corda/node/services/api/IdentityServiceInternal.kt Normal file
View File

@ -0,0 +1,22 @@
package net.corda.node.services.api
import net.corda.core.identity.PartyAndCertificate
import net.corda.core.node.services.IdentityService
import net.corda.core.utilities.contextLogger
import java.security.InvalidAlgorithmParameterException
import java.security.cert.CertificateExpiredException
import java.security.cert.CertificateNotYetValidException
interface IdentityServiceInternal : IdentityService {
private companion object {
val log = contextLogger()
}
/** This method exists so it can be mocked with doNothing, rather than having to make up a possibly invalid return value. */
fun justVerifyAndRegisterIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean = false) {
verifyAndRegisterIdentity(identity, isNewRandomIdentity)
}
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
fun verifyAndRegisterIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean): PartyAndCertificate?
}

14
node/src/main/kotlin/net/corda/node/services/identity/InMemoryIdentityService.kt
View File

@ -11,6 +11,7 @@ import net.corda.core.node.services.IdentityService
import net.corda.core.serialization.SingletonSerializeAsToken
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.trace
import net.corda.node.services.api.IdentityServiceInternal
import net.corda.node.services.persistence.WritablePublicKeyToOwningIdentityCache
import net.corda.nodeapi.internal.crypto.X509Utilities
import net.corda.nodeapi.internal.crypto.x509Certificates
@ -32,7 +33,7 @@ import kotlin.collections.LinkedHashSet
class InMemoryIdentityService(
identities: List<PartyAndCertificate> = emptyList(),
override val trustRoot: X509Certificate
) : SingletonSerializeAsToken(), IdentityService {
) : SingletonSerializeAsToken(), IdentityServiceInternal {
companion object {
private val log = contextLogger()
}
@ -60,6 +61,11 @@ class InMemoryIdentityService(
return verifyAndRegisterIdentity(trustAnchor, identity)
}
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
override fun verifyAndRegisterIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean): PartyAndCertificate? {
return verifyAndRegisterIdentity(trustAnchor, identity)
}
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
private fun verifyAndRegisterIdentity(trustAnchor: TrustAnchor, identity: PartyAndCertificate): PartyAndCertificate? {
// Validate the chain first, before we do anything clever with it
@ -82,12 +88,12 @@ class InMemoryIdentityService(
val firstPath = X509Utilities.buildCertPath(identityCertChain.slice(idx until identityCertChain.size))
verifyAndRegisterIdentity(trustAnchor, PartyAndCertificate(firstPath))
}
return registerIdentity(identity)
return registerIdentity(identity, false)
}
private fun registerIdentity(identity: PartyAndCertificate): PartyAndCertificate? {
private fun registerIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean): PartyAndCertificate? {
val identityCertChain = identity.certPath.x509Certificates
log.trace { "Registering identity $identity" }
log.trace { "Registering identity $identity isNewRandomIdentity=${isNewRandomIdentity}" }
keyToPartyAndCerts[identity.owningKey] = identity
// Always keep the first party we registered, as that's the well known identity
nameToKey.computeIfAbsent(identity.name) {identity.owningKey}

81
node/src/main/kotlin/net/corda/node/services/identity/PersistentIdentityService.kt
View File

@ -13,6 +13,7 @@ import net.corda.core.serialization.SingletonSerializeAsToken
import net.corda.core.utilities.MAX_HASH_HEX_SIZE
import net.corda.core.utilities.contextLogger
import net.corda.core.utilities.debug
import net.corda.node.services.api.IdentityServiceInternal
import net.corda.node.services.keys.BasicHSMKeyManagementService
import net.corda.node.services.persistence.PublicKeyHashToExternalId
import net.corda.node.services.persistence.WritablePublicKeyToOwningIdentityCache
@ -43,7 +44,7 @@ import kotlin.streams.toList
* cached for efficient lookup.
*/
@ThreadSafe
class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), IdentityService {
class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), IdentityServiceInternal {
companion object {
private val log = contextLogger()
@ -56,7 +57,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
const val IDENTITY_COLUMN_NAME = "identity_value"
const val NAME_COLUMN_NAME = "name"
fun createKeyToPartyAndCertMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, PartyAndCertificate, PersistentPublicKeyHashToCertificate, String> {
fun createKeyToPartyAndCertMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, PartyAndCertificate,
PersistentPublicKeyHashToCertificate, String> {
return AppendOnlyPersistentMap(
cacheFactory = cacheFactory,
name = "PersistentIdentityService_keyToPartyAndCert",
@ -74,7 +76,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
)
}
fun createX500ToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<CordaX500Name, String, PersistentPartyToPublicKeyHash, String> {
fun createX500ToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<CordaX500Name, String,
PersistentPartyToPublicKeyHash, String> {
return AppendOnlyPersistentMap(
cacheFactory = cacheFactory,
name = "PersistentIdentityService_nameToKey",
@ -89,7 +92,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
)
}
fun createKeyToX500Map(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, CordaX500Name, PersistentPublicKeyHashToParty, String> {
fun createKeyToX500Map(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, CordaX500Name,
PersistentPublicKeyHashToParty, String> {
return AppendOnlyPersistentMap(
cacheFactory = cacheFactory,
name = "PersistentIdentityService_keyToName",
@ -106,7 +110,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
persistentEntityClass = PersistentPublicKeyHashToParty::class.java)
}
fun createHashToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, PublicKey, PersistentHashToPublicKey, String> {
fun createHashToKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<String, PublicKey, PersistentHashToPublicKey,
String> {
return AppendOnlyPersistentMap(
cacheFactory = cacheFactory,
name = "PersistentIdentityService_hashToKey",
@ -207,7 +212,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
notaryIdentityCache.addAll(notaryIdentities)
}
fun loadIdentities(identities: Collection<PartyAndCertificate> = emptySet(), confidentialIdentities: Collection<PartyAndCertificate> = emptySet()) {
fun loadIdentities(identities: Collection<PartyAndCertificate> = emptySet(), confidentialIdentities: Collection<PartyAndCertificate> =
emptySet()) {
identities.forEach {
val key = mapToKey(it)
keyToPartyAndCert.addWithDuplicatesAllowed(key, it, false)
@ -222,7 +228,14 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
override fun verifyAndRegisterIdentity(identity: PartyAndCertificate): PartyAndCertificate? {
return verifyAndRegisterIdentity(trustAnchor, identity)
return verifyAndRegisterIdentity(identity, false)
}
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
override fun verifyAndRegisterIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean): PartyAndCertificate? {
return database.transaction {
verifyAndRegisterIdentity(trustAnchor, identity, isNewRandomIdentity)
}
}
/**
@ -230,16 +243,17 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
*
* @param trustAnchor The trust anchor that will verify the identity's validity
* @param identity The identity to verify
* @param isNewRandomIdentity true if the identity will not have been registered before (e.g. because it is randomly generated by ourselves).
* @param isNewRandomIdentity true if identity will not have been registered before (e.g. because it is randomly generated by us)
*/
@Throws(CertificateExpiredException::class, CertificateNotYetValidException::class, InvalidAlgorithmParameterException::class)
private fun verifyAndRegisterIdentity(trustAnchor: TrustAnchor, identity: PartyAndCertificate): PartyAndCertificate? {
// Validate the chain first, before we do anything clever with it
private fun verifyAndRegisterIdentity(trustAnchor: TrustAnchor, identity: PartyAndCertificate, isNewRandomIdentity: Boolean = false):
PartyAndCertificate? {
// Validate the chain first, before we do anything clever with it
val identityCertChain = identity.certPath.x509Certificates
try {
identity.verify(trustAnchor)
} catch (e: CertPathValidatorException) {
log.warn("Certificate validation failed for ${identity.name} against trusted root ${trustAnchor.trustedCert.subjectX500Principal}.")
log.warn("Certificate validation failed for ${identity.name} against trusted root ${trustAnchor.trustedCert.subjectX500Principal}.")
log.warn("Certificate path :")
identityCertChain.reversed().forEachIndexed { index, certificate ->
val space = (0 until index).joinToString("") { " " }
@ -249,24 +263,32 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
}
// Ensure we record the first identity of the same name, first
val wellKnownCert = identityCertChain.single { CertRole.extract(it)?.isWellKnown ?: false }
if (wellKnownCert != identity.certificate) {
if (wellKnownCert != identity.certificate && !isNewRandomIdentity) {
val idx = identityCertChain.lastIndexOf(wellKnownCert)
val firstPath = X509Utilities.buildCertPath(identityCertChain.slice(idx until identityCertChain.size))
verifyAndRegisterIdentity(trustAnchor, PartyAndCertificate(firstPath))
}
return registerIdentity(identity)
return registerIdentity(identity, isNewRandomIdentity)
}
private fun registerIdentity(identity: PartyAndCertificate): PartyAndCertificate? {
private fun registerIdentity(identity: PartyAndCertificate, isNewRandomIdentity: Boolean): PartyAndCertificate? {
log.debug { "Registering identity $identity" }
val identityCertChain = identity.certPath.x509Certificates
val key = mapToKey(identity)
return database.transaction {
keyToPartyAndCert.addWithDuplicatesAllowed(key, identity, false)
nameToKey.addWithDuplicatesAllowed(identity.name, key, false)
keyToName.addWithDuplicatesAllowed(key, identity.name, false)
if (isNewRandomIdentity) {
// Because this is supposed to be new and random, there's no way we have it in the database already, so skip the this check
keyToPartyAndCert[key] = identity
val parentId = identityCertChain[1].publicKey.toStringShort()
keyToPartyAndCert[parentId]
return keyToPartyAndCert[parentId]
} else {
return database.transaction {
keyToPartyAndCert.addWithDuplicatesAllowed(key, identity, false)
nameToKey.addWithDuplicatesAllowed(identity.name, key, false)
keyToName.addWithDuplicatesAllowed(key, identity.name, false)
val parentId = identityCertChain[1].publicKey.toStringShort()
keyToPartyAndCert[parentId]
}
}
}
@ -305,13 +327,13 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
// If we cannot find it then we perform a lookup on the public key to X500 name table
val legalIdentity = super.wellKnownPartyFromAnonymous(party)
if (legalIdentity == null) {
// If there is no entry in the legal keyToPartyAndCert table then the party must be a confidential identity so we perform
// a lookup in the keyToName table. If an entry for that public key exists, then we attempt look up the associated node's
// PartyAndCertificate.
// If there is no entry in the legal keyToPartyAndCert table then the party must be a confidential identity so we
// perform a lookup in the keyToName table. If an entry for that public key exists, then we attempt look up the
// associated node's PartyAndCertificate.
val name = keyToName[party.owningKey.toStringShort()]
if (name != null) {
// This should never return null as this node would not be able to communicate with the node providing a confidential
// identity unless its NodeInfo/PartyAndCertificate were available.
// This should never return null as this node would not be able to communicate with the node providing a
// confidential identity unless its NodeInfo/PartyAndCertificate were available.
wellKnownPartyFromX500Name(name)
} else {
null
@ -332,12 +354,14 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
}
@Throws(UnknownAnonymousPartyException::class)
override fun assertOwnership(party: Party, anonymousParty: AnonymousParty) = database.transaction { super.assertOwnership(party, anonymousParty) }
override fun assertOwnership(party: Party, anonymousParty: AnonymousParty) = database.transaction { super.assertOwnership(party,
anonymousParty) }
lateinit var ourNames: Set<CordaX500Name>
// Allows us to eliminate keys we know belong to others by using the cache contents that might have been seen during other identity activity.
// Concentrating activity on the identity cache works better than spreading checking across identity and key management, because we cache misses too.
// Allows us to eliminate keys we know belong to others by using the cache contents that might have been seen during other identity
// activity. Concentrating activity on the identity cache works better than spreading checking across identity and key management,
// because we cache misses too.
fun stripNotOurKeys(keys: Iterable<PublicKey>): Iterable<PublicKey> {
return keys.filter { (@Suppress("DEPRECATION") certificateFromKey(it))?.name in ourNames }
}
@ -351,7 +375,8 @@ class PersistentIdentityService(cacheFactory: NamedCacheFactory) : SingletonSeri
if (existingEntryForKey == null) {
// Update the three tables as necessary. We definitely store the public key and map it to a party and we optionally update
// the public key to external ID mapping table. This block will only ever be reached when registering keys generated on
// other because when a node generates its own keys "registerKeyToParty" is automatically called by KeyManagementService.freshKey.
// other because when a node generates its own keys "registerKeyToParty" is automatically called by
// KeyManagementService.freshKey.
registerKeyToParty(publicKey, party)
hashToKey[publicKeyHash] = publicKey
if (externalId != null) {

8
node/src/main/kotlin/net/corda/node/services/keys/KMSUtils.kt
View File

@ -5,6 +5,7 @@ import net.corda.core.identity.PartyAndCertificate
import net.corda.core.internal.CertRole
import net.corda.core.node.services.IdentityService
import net.corda.core.utilities.days
import net.corda.node.services.api.IdentityServiceInternal
import net.corda.nodeapi.internal.crypto.CertificateType
import net.corda.nodeapi.internal.crypto.ContentSignerBuilder
import net.corda.nodeapi.internal.crypto.X509Utilities
@ -44,7 +45,12 @@ fun freshCertificate(identityService: IdentityService,
window)
val ourCertPath = X509Utilities.buildCertPath(ourCertificate, issuer.certPath.x509Certificates)
val anonymisedIdentity = PartyAndCertificate(ourCertPath)
identityService.verifyAndRegisterIdentity(anonymisedIdentity)
if (identityService is IdentityServiceInternal) {
identityService.justVerifyAndRegisterIdentity(anonymisedIdentity, true)
} else {
identityService.verifyAndRegisterIdentity(anonymisedIdentity)
}
return anonymisedIdentity
}

45
node/src/main/kotlin/net/corda/node/services/network/PersistentNetworkMapCache.kt
View File

@ -34,6 +34,7 @@ import rx.subjects.PublishSubject
import java.security.PublicKey
import java.util.*
import javax.annotation.concurrent.ThreadSafe
import javax.persistence.PersistenceException
/** Database-based network map cache. */
@ThreadSafe
@ -184,20 +185,44 @@ open class PersistentNetworkMapCache(cacheFactory: NamedCacheFactory,
else -> logger.info("Previous node was identical to incoming one - doing nothing")
}
}
/**
* This algorithm protects against database failure (eg. attempt to persist a nodeInfo entry larger than permissible by the
* database X500Name) without sacrificing performance incurred by attempting to flush nodeInfo's individually.
* Upon database transaction failure, the list of new nodeInfo's is split in half, and then each half is persisted independently.
* This continues recursively until all valid nodeInfo's are persisted, and failed ones reported as warnings.
*/
recursivelyUpdateNodes(newNodes.map { nodeInfo -> Pair(nodeInfo, MapChange.Added(nodeInfo)) } +
updatedNodes.map { (nodeInfo, previousNodeInfo) -> Pair(nodeInfo, MapChange.Modified(nodeInfo, previousNodeInfo)) })
}
}
database.transaction {
updatedNodes.forEach { (node, previousNode) ->
//updated
updateInfoDB(node, session)
changePublisher.onNext(MapChange.Modified(node, previousNode))
}
newNodes.forEach { node ->
//new
updateInfoDB(node, session)
changePublisher.onNext(MapChange.Added(node))
private fun recursivelyUpdateNodes(nodeUpdates: List<Pair<NodeInfo, MapChange>>) {
try {
persistNodeUpdates(nodeUpdates)
}
catch (e: PersistenceException) {
if (nodeUpdates.isNotEmpty()) {
when {
nodeUpdates.size > 1 -> {
// persist first half
val nodeUpdatesLow = nodeUpdates.subList(0, (nodeUpdates.size / 2))
recursivelyUpdateNodes(nodeUpdatesLow)
// persist second half
val nodeUpdatesHigh = nodeUpdates.subList((nodeUpdates.size / 2), nodeUpdates.size)
recursivelyUpdateNodes(nodeUpdatesHigh)
}
else -> logger.warn("Failed to add or update node with info: ${nodeUpdates.single()}")
}
}
}
}
private fun persistNodeUpdates(nodeUpdates: List<Pair<NodeInfo, MapChange>>) {
database.transaction {
nodeUpdates.forEach { (nodeInfo, change) ->
updateInfoDB(nodeInfo, session)
changePublisher.onNext(change)
}
}
}

3
node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumper.kt
View File

@ -151,8 +151,9 @@ class CheckpointDumper(private val checkpointStorage: CheckpointStorage, private
private fun checkpointAgentRunning(): Boolean {
val agentProperties = getJvmAgentProperties(log)
val pattern = "(.+)?checkpoint-agent(-.+)?\\.jar.*".toRegex()
return agentProperties.values.any { value ->
value is String && value.contains("checkpoint-agent.jar")
value is String && value.contains(pattern)
}
}

8
node/src/main/kotlin/net/corda/node/services/rpc/RPCServer.kt
View File

@ -14,6 +14,7 @@ import net.corda.core.context.Trace.InvocationId
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.LifeCycle
import net.corda.core.internal.NamedCacheFactory
import net.corda.core.internal.messaging.InternalCordaRPCOps
import net.corda.core.messaging.CordaRPCOps
import net.corda.core.messaging.RPCOps
import net.corda.core.serialization.SerializationContext
@ -158,7 +159,12 @@ class RPCServer(
opsList.forEach { ops ->
listOfApplicableInterfacesRec(ops.javaClass).toSet().forEach { interfaceClass ->
val groupedMethods = with(interfaceClass) {
if(interfaceClass == CordaRPCOps::class.java) {
/*
* Until version 4.3, rpc calls did not include class names.
* Up to this version, only CordaRPCOps and InternalCordaRPCOps were supported.
* So, for these classes methods are registered without their class name as well to preserve backwards compatibility.
*/
if(interfaceClass == CordaRPCOps::class.java || interfaceClass == InternalCordaRPCOps::class.java) {
methods.groupBy { it.name }
} else {
methods.groupBy { interfaceClass.name + CLASS_METHOD_DIVIDER + it.name }

4
node/src/main/kotlin/net/corda/node/utilities/JVMAgentUtil.kt
View File

@ -16,8 +16,8 @@ object JVMAgentUtil {
return try {
val vm = VirtualMachine.attach(jvmPid)
return vm.agentProperties
} catch (e: Exception) {
log.warn("Unable to determine whether checkpoint agent is running: ${e.message}.\n" +
} catch (e: Throwable) {
log.warn("Unable to determine whether agent is running: ${e.message}.\n" +
"You may need to pass in -Djdk.attach.allowAttachSelf=true if running on a Java 9 or later VM")
Properties()
}

18
node/src/main/kotlin/net/corda/node/utilities/errorAndTerminate.kt
View File

@ -10,12 +10,16 @@ import kotlin.concurrent.thread
*/
@Synchronized
fun errorAndTerminate(message: String, e: Throwable?) {
thread {
val log = LoggerFactory.getLogger("errorAndTerminate")
log.error(message, e)
}
try {
thread {
val log = LoggerFactory.getLogger("errorAndTerminate")
log.error(message, e)
}
// give the logger a chance to flush the error message before killing the node
Thread.sleep(10.seconds.toMillis())
Runtime.getRuntime().halt(1)
// give the logger a chance to flush the error message before killing the node
Thread.sleep(10.seconds.toMillis())
}
finally {
Runtime.getRuntime().halt(1)
}
}

2
node/src/main/resources/migration/node-core.changelog-v14-table.xml
View File

@ -13,6 +13,8 @@
<constraints nullable="false"/>
</column>
</createTable>
<addPrimaryKey columnNames="pk_hash" constraintName="node_identities_no_cert_pkey"
tableName="node_identities_no_cert"/>
</changeSet>
</databaseChangeLog>

21
node/src/test/kotlin/net/corda/node/services/network/NetworkMapClientTest.kt
View File

@ -2,6 +2,7 @@ package net.corda.node.services.network
import net.corda.core.crypto.Crypto
import net.corda.core.crypto.sha256
import net.corda.core.identity.CordaX500Name
import net.corda.core.internal.sign
import net.corda.core.serialization.serialize
import net.corda.core.utilities.seconds
@ -71,6 +72,26 @@ class NetworkMapClientTest {
assertEquals(nodeInfo2, networkMapClient.getNodeInfo(nodeInfoHash2))
}
@Test
fun `negative test - registered invalid node is added to the network map`() {
val invalidLongNodeName = CordaX500Name(
commonName = "AB123456789012345678901234567890123456789012345678901234567890",
organisationUnit = "AB123456789012345678901234567890123456789012345678901234567890",
organisation = "Long Plc",
locality = "AB123456789012345678901234567890123456789012345678901234567890",
state = "AB123456789012345678901234567890123456789012345678901234567890",
country= "IT")
val (nodeInfo, signedNodeInfo) = createNodeInfoAndSigned(invalidLongNodeName)
networkMapClient.publish(signedNodeInfo)
val nodeInfoHash = nodeInfo.serialize().sha256()
assertThat(networkMapClient.getNetworkMap().payload.nodeInfoHashes).containsExactly(nodeInfoHash)
assertEquals(nodeInfo, networkMapClient.getNodeInfo(nodeInfoHash))
}
@Test
fun `errors return a meaningful error message`() {
val nodeInfoBuilder = TestNodeInfoBuilder()

1
settings.gradle
View File

@ -100,4 +100,3 @@ include 'core-deterministic:testing:verifier'
include 'serialization-deterministic'
include 'tools:checkpoint-agent'
findProject(':tools:checkpoint-agent')?.name = 'checkpoint-agent'

2
testing/Dockerfile
View File

@ -1,4 +1,4 @@
FROM stefanotestingcr.azurecr.io/buildbase:latest
COPY . /tmp/source
CMD ls /tmp/gradle && cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info
CMD cd /tmp/source && GRADLE_USER_HOME=/tmp/gradle ./gradlew clean testClasses integrationTestClasses --parallel --info

2
testing/DockerfileBase
View File

@ -2,7 +2,7 @@ FROM ubuntu:18.04
ENV GRADLE_USER_HOME=/tmp/gradle
RUN mkdir /tmp/gradle && mkdir -p /home/root/.m2/repository
RUN apt-get update && apt-get install -y curl && \
RUN apt-get update && apt-get install -y curl libatomic1 && \
curl -O https://d3pxv6yz143wms.cloudfront.net/8.222.10.1/java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
apt-get install -y java-common && dpkg -i java-1.8.0-amazon-corretto-jdk_8.222.10-1_amd64.deb && \
apt-get clean && \

3224
testing/test-times.csv Normal file
View File

File diff suppressed because it is too large Load Diff

5
testing/testserver/testcapsule/build.gradle
View File

@ -26,7 +26,10 @@ capsule {
task buildWebserverJar(type: FatCapsule, dependsOn: project(':node').tasks.jar) {
applicationClass 'net.corda.webserver.WebServer'
archiveName "corda-testserver-${corda_release_version}.jar"
archiveBaseName = 'corda-testserver'
archiveVersion = corda_release_version
archiveClassifier = jdkClassifier
archiveName = archiveFileName.get()
applicationSource = files(
project(':testing:testserver').configurations.runtimeClasspath,
project(':testing:testserver').tasks.jar,

8
tools/checkpoint-agent/build.gradle
View File

@ -25,6 +25,8 @@ repositories {
apply plugin: 'kotlin'
apply plugin: 'idea'
apply plugin: 'net.corda.plugins.publish-utils'
apply plugin: 'com.jfrog.artifactory'
description 'A javaagent to allow hooking into Kryo checkpoints'
@ -49,7 +51,7 @@ dependencies {
}
jar {
archiveName = "${project.name}.jar"
archiveBaseName = "${project.name}"
manifest {
attributes(
'Premain-Class': 'net.corda.tools.CheckpointAgent',
@ -62,3 +64,7 @@ jar {
}
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
}
publish {
name 'corda-tools-checkpoint-agent'
}

6
tools/explorer/capsule/build.gradle
View File

@ -17,13 +17,15 @@ capsule {
task buildExplorerJAR(type: FatCapsule, dependsOn: project(':tools:explorer').tasks.jar) {
applicationClass 'net.corda.explorer.Main'
archiveName "node-explorer-${corda_release_version}.jar"
archiveBaseName = 'node-explorer'
archiveVersion = corda_release_version
archiveClassifier = jdkClassifier
archiveName = archiveFileName.get()
applicationSource = files(
project(':tools:explorer').configurations.runtimeClasspath,
project(':tools:explorer').tasks.jar,
project(':tools:explorer').sourceSets.main.java.outputDir.toString() + '/ExplorerCaplet.class'
)
classifier 'fat'
capsuleManifest {
applicationVersion = corda_release_version

5
tools/network-builder/build.gradle
View File

@ -73,7 +73,7 @@ processResources {
shadowJar {
baseName = 'network-builder'
classifier = null
archiveClassifier = jdkClassifier
version = null
zip64 true
}
@ -83,12 +83,11 @@ assemble.dependsOn buildNetworkBuilder
artifacts {
publish shadowJar {
classifier = ""
archiveClassifier = jdkClassifier
}
}
jar {
classifier "ignore"
enabled = false
}

4
tools/shell-cli/build.gradle
View File

@ -35,12 +35,12 @@ assemble.dependsOn buildShellCli
artifacts {
publish shadowJar {
classifier = ""
archiveClassifier = jdkClassifier
}
}
jar {
classifier "ignore"
archiveClassifier = "ignore"
enabled = false
}