From 03ab258fc2d0f40badd73f78c81e5ec6badf2940 Mon Sep 17 00:00:00 2001 From: Stefan Iliev <46542846+StefanIliev545@users.noreply.github.com> Date: Tue, 29 Oct 2019 17:55:58 +0000 Subject: [PATCH 01/18] Revert "CORDA-3307 - add support for environment variables in linux (#5523)" (#5643) This reverts commit c882b221a5a5b6028c6828dd912f9e1680a158d5. --- detekt-baseline.xml | 10 ++- docs/source/changelog.rst | 3 - docs/source/corda-configuration-file.rst | 7 -- .../net/corda/node/NodeConfigParsingTests.kt | 87 ------------------- .../corda/node/NodeStartupPerformanceTests.kt | 2 +- .../node/services/config/ConfigUtilities.kt | 67 +------------- .../services/config/ShadowingException.kt | 7 -- .../kotlin/net/corda/testing/driver/Driver.kt | 78 +---------------- .../testing/node/internal/DriverDSLImpl.kt | 50 ++++------- .../testing/node/internal/ProcessUtilities.kt | 20 +---- .../corda/testing/node/internal/RPCDriver.kt | 5 +- 11 files changed, 32 insertions(+), 304 deletions(-) delete mode 100644 node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt delete mode 100644 node/src/main/kotlin/net/corda/node/services/config/ShadowingException.kt diff --git a/detekt-baseline.xml b/detekt-baseline.xml index 1db37ae52c..c47aaf3ad0 100644 --- a/detekt-baseline.xml +++ b/detekt-baseline.xml @@ -689,6 +689,8 @@ LongParameterList:Driver.kt$DriverParameters$( isDebug: Boolean, driverDirectory: Path, portAllocation: PortAllocation, debugPortAllocation: PortAllocation, systemProperties: Map<String, String>, useTestClock: Boolean, startNodesInProcess: Boolean, waitForAllNodesToFinish: Boolean, notarySpecs: List<NotarySpec>, extraCordappPackagesToScan: List<String>, jmxPolicy: JmxPolicy, networkParameters: NetworkParameters, cordappsForAllNodes: Set<TestCordapp>? ) LongParameterList:DriverDSL.kt$DriverDSL$( defaultParameters: NodeParameters = NodeParameters(), providedName: CordaX500Name? = defaultParameters.providedName, rpcUsers: List<User> = defaultParameters.rpcUsers, verifierType: VerifierType = defaultParameters.verifierType, customOverrides: Map<String, Any?> = defaultParameters.customOverrides, startInSameProcess: Boolean? = defaultParameters.startInSameProcess, maximumHeapSize: String = defaultParameters.maximumHeapSize ) LongParameterList:DriverDSL.kt$DriverDSL$( defaultParameters: NodeParameters = NodeParameters(), providedName: CordaX500Name? = defaultParameters.providedName, rpcUsers: List<User> = defaultParameters.rpcUsers, verifierType: VerifierType = defaultParameters.verifierType, customOverrides: Map<String, Any?> = defaultParameters.customOverrides, startInSameProcess: Boolean? = defaultParameters.startInSameProcess, maximumHeapSize: String = defaultParameters.maximumHeapSize, logLevelOverride: String? = defaultParameters.logLevelOverride ) + LongParameterList:DriverDSLImpl.kt$( isDebug: Boolean = DriverParameters().isDebug, driverDirectory: Path = DriverParameters().driverDirectory, portAllocation: PortAllocation = DriverParameters().portAllocation, debugPortAllocation: PortAllocation = DriverParameters().debugPortAllocation, systemProperties: Map<String, String> = DriverParameters().systemProperties, useTestClock: Boolean = DriverParameters().useTestClock, startNodesInProcess: Boolean = DriverParameters().startNodesInProcess, extraCordappPackagesToScan: List<String> = @Suppress("DEPRECATION") DriverParameters().extraCordappPackagesToScan, waitForAllNodesToFinish: Boolean = DriverParameters().waitForAllNodesToFinish, notarySpecs: List<NotarySpec> = DriverParameters().notarySpecs, jmxPolicy: JmxPolicy = DriverParameters().jmxPolicy, networkParameters: NetworkParameters = DriverParameters().networkParameters, compatibilityZone: CompatibilityZoneParams? = null, notaryCustomOverrides: Map<String, Any?> = DriverParameters().notaryCustomOverrides, inMemoryDB: Boolean = DriverParameters().inMemoryDB, cordappsForAllNodes: Collection<TestCordappInternal>? = null, dsl: DriverDSLImpl.() -> A ) + LongParameterList:DriverDSLImpl.kt$DriverDSLImpl.Companion$( config: NodeConfig, quasarJarPath: String, debugPort: Int?, overriddenSystemProperties: Map<String, String>, maximumHeapSize: String, logLevelOverride: String?, vararg extraCmdLineFlag: String ) LongParameterList:DummyFungibleContract.kt$DummyFungibleContract$(inputs: List<State>, outputs: List<State>, tx: LedgerTransaction, issueCommand: CommandWithParties<Commands.Issue>, currency: Currency, issuer: PartyAndReference) LongParameterList:IRS.kt$FloatingRatePaymentEvent$(date: LocalDate = this.date, accrualStartDate: LocalDate = this.accrualStartDate, accrualEndDate: LocalDate = this.accrualEndDate, dayCountBasisDay: DayCountBasisDay = this.dayCountBasisDay, dayCountBasisYear: DayCountBasisYear = this.dayCountBasisYear, fixingDate: LocalDate = this.fixingDate, notional: Amount<Currency> = this.notional, rate: Rate = this.rate) LongParameterList:IRS.kt$InterestRateSwap$(floatingLeg: FloatingLeg, fixedLeg: FixedLeg, calculation: Calculation, common: Common, oracle: Party, notary: Party) @@ -725,6 +727,8 @@ LongParameterList:ParametersUtilities.kt$( notaries: List<NotaryInfo> = emptyList(), minimumPlatformVersion: Int = 1, modifiedTime: Instant = Instant.now(), maxMessageSize: Int = 10485760, // TODO: Make this configurable and consistence across driver, bootstrapper, demobench and NetworkMapServer maxTransactionSize: Int = maxMessageSize * 50, whitelistedContractImplementations: Map<String, List<AttachmentId>> = emptyMap(), epoch: Int = 1, eventHorizon: Duration = 30.days, packageOwnership: Map<String, PublicKey> = emptyMap() ) LongParameterList:PersistentUniquenessProvider.kt$PersistentUniquenessProvider$( states: List<StateRef>, txId: SecureHash, callerIdentity: Party, requestSignature: NotarisationRequestSignature, timeWindow: TimeWindow?, references: List<StateRef> ) LongParameterList:PhysicalLocationStructures.kt$WorldCoordinate$(screenWidth: Double, screenHeight: Double, topLatitude: Double, bottomLatitude: Double, leftLongitude: Double, rightLongitude: Double) + LongParameterList:ProcessUtilities.kt$ProcessUtilities$( arguments: List<String>, classPath: List<String> = defaultClassPath, workingDirectory: Path? = null, jdwpPort: Int? = null, extraJvmArguments: List<String> = emptyList(), maximumHeapSize: String? = null ) + LongParameterList:ProcessUtilities.kt$ProcessUtilities$( className: String, arguments: List<String>, classPath: List<String> = defaultClassPath, workingDirectory: Path? = null, jdwpPort: Int? = null, extraJvmArguments: List<String> = emptyList(), maximumHeapSize: String? = null ) LongParameterList:QueryCriteria.kt$QueryCriteria.FungibleAssetQueryCriteria$( participants: List<AbstractParty>? = this.participants, owner: List<AbstractParty>? = this.owner, quantity: ColumnPredicate<Long>? = this.quantity, issuer: List<AbstractParty>? = this.issuer, issuerRef: List<OpaqueBytes>? = this.issuerRef, status: Vault.StateStatus = this.status, contractStateTypes: Set<Class<out ContractState>>? = this.contractStateTypes ) LongParameterList:QueryCriteria.kt$QueryCriteria.FungibleAssetQueryCriteria$( participants: List<AbstractParty>? = this.participants, owner: List<AbstractParty>? = this.owner, quantity: ColumnPredicate<Long>? = this.quantity, issuer: List<AbstractParty>? = this.issuer, issuerRef: List<OpaqueBytes>? = this.issuerRef, status: Vault.StateStatus = this.status, contractStateTypes: Set<Class<out ContractState>>? = this.contractStateTypes, relevancyStatus: Vault.RelevancyStatus = this.relevancyStatus ) LongParameterList:QueryCriteria.kt$QueryCriteria.LinearStateQueryCriteria$( participants: List<AbstractParty>? = this.participants, uuid: List<UUID>? = this.uuid, externalId: List<String>? = this.externalId, status: Vault.StateStatus = this.status, contractStateTypes: Set<Class<out ContractState>>? = this.contractStateTypes, relevancyStatus: Vault.RelevancyStatus = this.relevancyStatus ) @@ -732,6 +736,7 @@ LongParameterList:QueryCriteria.kt$QueryCriteria.VaultQueryCriteria$( status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED, contractStateTypes: Set<Class<out ContractState>>? = null, stateRefs: List<StateRef>? = null, notary: List<AbstractParty>? = null, softLockingCondition: SoftLockingCondition? = null, timeCondition: TimeCondition? = null, relevancyStatus: Vault.RelevancyStatus = Vault.RelevancyStatus.ALL, constraintTypes: Set<Vault.ConstraintInfo.Type> = emptySet(), constraints: Set<Vault.ConstraintInfo> = emptySet(), participants: List<AbstractParty>? = null, externalIds: List<UUID> = emptyList() ) LongParameterList:QueryCriteria.kt$QueryCriteria.VaultQueryCriteria$( status: Vault.StateStatus = this.status, contractStateTypes: Set<Class<out ContractState>>? = this.contractStateTypes, stateRefs: List<StateRef>? = this.stateRefs, notary: List<AbstractParty>? = this.notary, softLockingCondition: SoftLockingCondition? = this.softLockingCondition, timeCondition: TimeCondition? = this.timeCondition ) LongParameterList:RPCClient.kt$RPCClient$( rpcOpsClass: Class<I>, username: String, password: String, externalTrace: Trace? = null, impersonatedActor: Actor? = null, targetLegalIdentity: CordaX500Name? = null ) + LongParameterList:RPCDriver.kt$( isDebug: Boolean = false, driverDirectory: Path = Paths.get("build") / "rpc-driver" / getTimestampAsDirectoryName(), portAllocation: PortAllocation = globalPortAllocation, debugPortAllocation: PortAllocation = globalDebugPortAllocation, systemProperties: Map<String, String> = emptyMap(), useTestClock: Boolean = false, startNodesInProcess: Boolean = false, waitForNodesToFinish: Boolean = false, extraCordappPackagesToScan: List<String> = emptyList(), notarySpecs: List<NotarySpec> = emptyList(), externalTrace: Trace? = null, @Suppress("DEPRECATION") jmxPolicy: JmxPolicy = JmxPolicy(), networkParameters: NetworkParameters = testNetworkParameters(), notaryCustomOverrides: Map<String, Any?> = emptyMap(), inMemoryDB: Boolean = true, cordappsForAllNodes: Collection<TestCordappInternal>? = null, dsl: RPCDriverDSL.() -> A ) LongParameterList:RPCDriver.kt$RPCDriverDSL$( rpcUser: User = rpcTestUser, nodeLegalName: CordaX500Name = fakeNodeLegalName, configuration: RPCServerConfiguration = RPCServerConfiguration.DEFAULT, listOps: List<I>, brokerHandle: RpcBrokerHandle, queueDrainTimeout: Duration = 5.seconds ) LongParameterList:RPCDriver.kt$RPCDriverDSL$( rpcUser: User = rpcTestUser, nodeLegalName: CordaX500Name = fakeNodeLegalName, configuration: RPCServerConfiguration = RPCServerConfiguration.DEFAULT, ops: I, brokerHandle: RpcBrokerHandle, queueDrainTimeout: Duration = 5.seconds ) LongParameterList:RPCDriver.kt$RPCDriverDSL$( rpcUser: User = rpcTestUser, nodeLegalName: CordaX500Name = fakeNodeLegalName, maxFileSize: Int = MAX_MESSAGE_SIZE, maxBufferedBytesPerClient: Long = 10L * MAX_MESSAGE_SIZE, configuration: RPCServerConfiguration = RPCServerConfiguration.DEFAULT, ops: I, queueDrainTimeout: Duration = 5.seconds ) @@ -1762,6 +1767,7 @@ MaxLineLength:ConfigUtilities.kt$ // TODO Move this to KeyStoreConfigHelpers. fun NodeConfiguration.configureWithDevSSLCertificate(cryptoService: CryptoService? = null) MaxLineLength:ConfigUtilities.kt$// Problems: // - Forces you to have a primary constructor with all fields of name and type matching the configuration file structure. // - Encourages weak bean-like types. // - Cannot support a many-to-one relationship between configuration file structures and configuration domain type. This is essential for versioning of the configuration files. // - It's complicated and based on reflection, meaning problems with it are typically found at runtime. // - It doesn't support validation errors in a structured way. If something goes wrong, it throws exceptions, which doesn't support good usability practices like displaying all the errors at once. fun <T : Any> Config.parseAs(clazz: KClass<T>, onUnknownKeys: ((Set<String>, logger: Logger) -> Unit) = UnknownConfigKeysPolicy.FAIL::handle, nestedPath: String? = null, baseDirectory: Path? = null): T MaxLineLength:ConfigUtilities.kt$// TODO Move this to KeyStoreConfigHelpers. fun MutualSslConfiguration.configureDevKeyAndTrustStores(myLegalName: CordaX500Name, signingCertificateStore: FileBasedCertificateStoreSupplier, certificatesDirectory: Path, cryptoService: CryptoService? = null) + MaxLineLength:ConfigUtilities.kt$ConfigHelper$return ConfigFactory.parseMap(toProperties().filterKeys { (it as String).startsWith(CORDA_PROPERTY_PREFIX) }.mapKeys { (it.key as String).removePrefix(CORDA_PROPERTY_PREFIX) }) MaxLineLength:ConfigUtilities.kt$ConfigHelper$val smartDevMode = CordaSystemUtils.isOsMac() || (CordaSystemUtils.isOsWindows() && !CordaSystemUtils.getOsName().toLowerCase().contains("server")) MaxLineLength:ConfigUtilities.kt$fun Any?.toConfigValue(): ConfigValue MaxLineLength:ConfigUtilities.kt$inline fun <reified T : Any> Config.parseAs(noinline onUnknownKeys: ((Set<String>, logger: Logger) -> Unit) = UnknownConfigKeysPolicy.FAIL::handle): T @@ -2074,8 +2080,9 @@ MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$private MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val flowOverrideConfig = FlowOverrideConfig(parameters.flowOverrides.map { FlowOverride(it.key.canonicalName, it.value.canonicalName) }) MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val jdbcUrl = "jdbc:h2:mem:persistence${inMemoryCounter.getAndIncrement()};DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=100" + MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl$val process = startOutOfProcessNode(config, quasarJarPath, debugPort, systemProperties, parameters.maximumHeapSize, parameters.logLevelOverride) MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion$private operator fun Config.plus(property: Pair<String, Any>) - MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion${ log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, " + "debug port is " + (debugPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) val systemProperties = mutableMapOf( "name" to config.corda.myLegalName, "visualvm.display.name" to "corda-${config.corda.myLegalName}" ) debugPort?.let { systemProperties += "log4j2.level" to "debug" systemProperties += "log4j2.debug" to "true" } systemProperties += inheritFromParentProcess() systemProperties += overriddenSystemProperties // See experimental/quasar-hook/README.md for how to generate. val excludePattern = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;" + "com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;" + "com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;" + "io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;" + "org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;" + "org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;" + "org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**;org.jolokia**;" + "com.lmax**;picocli**;liquibase**;com.github.benmanes**;org.json**;org.postgresql**;nonapi.io.github.classgraph**;)" val extraJvmArguments = systemProperties.removeResolvedClasspath().map { "-D${it.key}=${it.value}" } + "-javaagent:$quasarJarPath=$excludePattern" val loggingLevel = when { logLevelOverride != null -> logLevelOverride debugPort == null -> "INFO" else -> "DEBUG" } val arguments = mutableListOf( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { it.addAll(extraCmdLineFlag) }.toList() // The following dependencies are excluded from the classpath of the created JVM, so that the environment resembles a real one as close as possible. // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.) or irrelevant testing libraries (test, corda-mock etc.). // TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164. val exclude = listOf("samples", "finance", "integrationTest", "test", "corda-mock", "com.opengamma.strata") val cp = ProcessUtilities.defaultClassPath.filterNot { cpEntry -> exclude.any { token -> cpEntry.contains("${File.separatorChar}$token") } || cpEntry.endsWith("-tests.jar") } return ProcessUtilities.startJavaProcess( className = "net.corda.node.Corda", // cannot directly get class for this, so just use string arguments = arguments, jdwpPort = debugPort, extraJvmArguments = extraJvmArguments, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, classPath = cp, environmentVariables = environmentVariables ) } + MaxLineLength:DriverDSLImpl.kt$DriverDSLImpl.Companion${ log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) val systemProperties = mutableMapOf( "name" to config.corda.myLegalName, "visualvm.display.name" to "corda-${config.corda.myLegalName}" ) debugPort?.let { systemProperties += "log4j2.level" to "debug" systemProperties += "log4j2.debug" to "true" } systemProperties += inheritFromParentProcess() systemProperties += overriddenSystemProperties // See experimental/quasar-hook/README.md for how to generate. val excludePattern = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;" + "com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;" + "com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;" + "io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;" + "org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;" + "org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;" + "org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**;org.jolokia**;" + "com.lmax**;picocli**;liquibase**;com.github.benmanes**;org.json**;org.postgresql**;nonapi.io.github.classgraph**;)" val extraJvmArguments = systemProperties.removeResolvedClasspath().map { "-D${it.key}=${it.value}" } + "-javaagent:$quasarJarPath=$excludePattern" val loggingLevel = when { logLevelOverride != null -> logLevelOverride debugPort == null -> "INFO" else -> "DEBUG" } val arguments = mutableListOf( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { it += extraCmdLineFlag }.toList() // The following dependencies are excluded from the classpath of the created JVM, so that the environment resembles a real one as close as possible. // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.) or irrelevant testing libraries (test, corda-mock etc.). // TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164. val exclude = listOf("samples", "finance", "integrationTest", "test", "corda-mock", "com.opengamma.strata") val cp = ProcessUtilities.defaultClassPath.filterNot { cpEntry -> exclude.any { token -> cpEntry.contains("${File.separatorChar}$token") } || cpEntry.endsWith("-tests.jar") } return ProcessUtilities.startJavaProcess( className = "net.corda.node.Corda", // cannot directly get class for this, so just use string arguments = arguments, jdwpPort = debugPort, extraJvmArguments = extraJvmArguments, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, classPath = cp ) } MaxLineLength:DriverDSLImpl.kt$InternalDriverDSL$ fun <A> pollUntilNonNull(pollName: String, pollInterval: Duration = DEFAULT_POLL_INTERVAL, warnCount: Int = DEFAULT_WARN_COUNT, check: () -> A?): CordaFuture<A> MaxLineLength:DriverDSLImpl.kt$InternalDriverDSL$ fun pollUntilTrue(pollName: String, pollInterval: Duration = DEFAULT_POLL_INTERVAL, warnCount: Int = DEFAULT_WARN_COUNT, check: () -> Boolean): CordaFuture<Unit> MaxLineLength:DriverDSLImpl.kt$fun DriverDSL.startNode(providedName: CordaX500Name, devMode: Boolean, parameters: NodeParameters = NodeParameters()): CordaFuture<NodeHandle> @@ -3852,6 +3859,7 @@ SpreadOperator:DemoBench.kt$DemoBench.Companion$(DemoBench::class.java, *args) SpreadOperator:DevCertificatesTest.kt$DevCertificatesTest$(*oldX509Certificates) SpreadOperator:DockerInstantiator.kt$DockerInstantiator$(*it.toTypedArray()) + SpreadOperator:DriverDSLImpl.kt$DriverDSLImpl$( config, quasarJarPath, debugPort, systemProperties, "512m", null, *extraCmdLineFlag ) SpreadOperator:DummyContract.kt$DummyContract.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owner.owningKey), /* OUTPUT */ StateAndContract(state, PROGRAM_ID) ) SpreadOperator:DummyContract.kt$DummyContract.Companion$(*items) SpreadOperator:DummyContractV2.kt$DummyContractV2.Companion$( /* INPUTS */ *priors.toTypedArray(), /* COMMAND */ Command(cmd, priorState.owners.map { it.owningKey }), /* OUTPUT */ StateAndContract(state, DummyContractV2.PROGRAM_ID) ) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d0d079bbed..e7ff69555c 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -80,9 +80,6 @@ Unreleased Note that it's a responsibility of a client application to handle RPC reconnection in case this happens. See :ref:`setting_jvm_args` and :ref:`memory_usage_and_tuning` for further details. -* Environment variables and system properties can now be provided with underscore separators instead of dots. Neither are case sensitive. - See :ref:`overriding config values ` for more information. - .. _changelog_v4.1: Version 4.1 diff --git a/docs/source/corda-configuration-file.rst b/docs/source/corda-configuration-file.rst index e44b92b10c..6cbbc141d9 100644 --- a/docs/source/corda-configuration-file.rst +++ b/docs/source/corda-configuration-file.rst @@ -39,8 +39,6 @@ To alter this behaviour, the ``on-unknown-config-keys`` command-line argument ca Overriding values from node.conf -------------------------------- -.. _corda_configuration_file_overriding_config: - Environment variables For example: ``${NODE_TRUST_STORE_PASSWORD}`` would be replaced by the contents of environment variable ``NODE_TRUST_STORE_PASSWORD`` (see: :ref:`hiding-sensitive-data` section). @@ -56,11 +54,6 @@ JVM options .. note:: If the same field is overriden by both an environment variable and system property, the system property takes precedence. -.. note:: Underscores can be used in instead of dots. For example overriding the ``p2pAddress`` with an environment variable can be done - by specifying ``CORDA_P2PADDRESS=host:port``. Variables and properties are not case sensitive. Corda will warn you if a variable - prefixed with ``CORDA`` cannot be mapped to a valid property. Shadowing occurs when two properties - of the same type with the same key are defined. For example having ``CORDA_P2PADDRESS=host:port`` and ``corda_p2paddress=host1:port1`` - will raise an exception on startup. This is to prevent hard to spot mistakes. Configuration file fields ------------------------- diff --git a/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt b/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt deleted file mode 100644 index 42a9dc95fe..0000000000 --- a/node/src/integration-test/kotlin/net/corda/node/NodeConfigParsingTests.kt +++ /dev/null @@ -1,87 +0,0 @@ -package net.corda.node - -import net.corda.core.utilities.getOrThrow -import net.corda.node.logging.logFile -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.NodeParameters -import net.corda.testing.driver.driver -import net.corda.testing.driver.internal.incrementalPortAllocation -import org.assertj.core.api.Assertions.assertThatThrownBy -import org.junit.Test -import org.junit.jupiter.api.assertThrows - -class NodeConfigParsingTests { - - @Test - fun `config is overriden by underscore variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("corda_sshd_port" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator)) { - val hasSsh = startNode().get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assert(hasSsh) - } - } - - @Test - fun `config is overriden by case insensitive underscore variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("cORDa_sShD_pOrt" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator)) { - val hasSsh = startNode().get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assert(hasSsh) - } - } - - @Test - fun `config is overriden by case insensitive dot variable`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf("cOrda.sShD.pOrt" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator)) { - val hasSsh = startNode(NodeParameters()).get() - .logFile() - .readLines() - .filter { it.contains("SSH server listening on port") } - .any { it.contains(sshPort.toString()) } - assert(hasSsh) - } - } - - @Test - fun `shadowing is forbidden`() { - val portAllocator = incrementalPortAllocation() - val sshPort = portAllocator.nextPort() - - driver(DriverParameters( - environmentVariables = mapOf( - "cOrda_sShD_POrt" to sshPort.toString(), - "cOrda.sShD.pOrt" to sshPort.toString()), - startNodesInProcess = false, - portAllocation = portAllocator, - notarySpecs = emptyList())) { - - assertThatThrownBy { - startNode().getOrThrow() - } - } - } -} diff --git a/node/src/integration-test/kotlin/net/corda/node/NodeStartupPerformanceTests.kt b/node/src/integration-test/kotlin/net/corda/node/NodeStartupPerformanceTests.kt index 12c434a69d..a9f2586013 100644 --- a/node/src/integration-test/kotlin/net/corda/node/NodeStartupPerformanceTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/NodeStartupPerformanceTests.kt @@ -24,4 +24,4 @@ class NodeStartupPerformanceTests { println(times.map { it / 1_000_000.0 }) } } -} \ No newline at end of file +} diff --git a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt index 7715d64153..1f914c5c01 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt @@ -1,7 +1,6 @@ package net.corda.node.services.config import com.typesafe.config.Config -import com.typesafe.config.ConfigException import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigParseOptions import net.corda.cliutils.CordaSystemUtils @@ -9,7 +8,6 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.internal.createDirectories import net.corda.core.internal.div import net.corda.core.internal.exists -import net.corda.node.internal.Node import net.corda.nodeapi.internal.DEV_CA_KEY_STORE_PASS import net.corda.nodeapi.internal.config.FileBasedCertificateStoreSupplier import net.corda.nodeapi.internal.config.MutualSslConfiguration @@ -21,10 +19,7 @@ import net.corda.nodeapi.internal.installDevNodeCaCertPath import net.corda.nodeapi.internal.loadDevCaTrustStore import net.corda.nodeapi.internal.registerDevP2pCertificates import org.slf4j.LoggerFactory -import java.lang.IllegalStateException import java.nio.file.Path -import kotlin.reflect.KClass -import kotlin.reflect.full.memberProperties fun configOf(vararg pairs: Pair): Config = ConfigFactory.parseMap(mapOf(*pairs)) operator fun Config.plus(overrides: Map): Config = ConfigFactory.parseMap(overrides).withFallback(this) @@ -67,69 +62,9 @@ object ConfigHelper { return finalConfig } - private fun getCaseSensitivePropertyPath(target : KClass?, path : List) : String { - - require(path.isNotEmpty()) { "Path to config property cannot be empty." } - - val lookFor = path.first() - target?.memberProperties?.forEach { - if (it.name.toLowerCase() == lookFor.toLowerCase()) { - return if (path.size > 1) - "${it.name}." + - getCaseSensitivePropertyPath( - (it.getter.returnType.classifier as KClass<*>), - path.subList(1, path.size)) - else - it.name - } - } - - return "" - } - - /* - * Gets - */ private fun Config.cordaEntriesOnly(): Config { - val cordaPropOccurrences = mutableSetOf() - val badKeyConversions = mutableSetOf() - - return ConfigFactory.parseMap( - toProperties() - .mapKeys { - val newKey = (it.key as String) - .replace("_", ".") - .toLowerCase() - - if (newKey.contains(CORDA_PROPERTY_PREFIX) && cordaPropOccurrences.contains(newKey)) { - throw ShadowingException(it.key.toString(), newKey) - } - - cordaPropOccurrences.add(newKey) - newKey.let { key -> - if (!key.contains(CORDA_PROPERTY_PREFIX)) - return@let key - - val nodeConfKey = key.removePrefix(CORDA_PROPERTY_PREFIX) - val configPath = getCaseSensitivePropertyPath( - NodeConfigurationImpl::class, - nodeConfKey.split(".") - ) - - if (nodeConfKey.length != configPath.length) { - Node.printWarning( - "${it.key} (property or environment variable) cannot be mapped to an existing Corda" + - " config property and thus won't be used as a config override!" + - " It won't be passed as a config override! If that was the intention " + - " double check the spelling and ensure there is such config key.") - badKeyConversions.add(configPath) - } - CORDA_PROPERTY_PREFIX + configPath - } - }.filterKeys { it.startsWith(CORDA_PROPERTY_PREFIX) } - .mapKeys { it.key.removePrefix(CORDA_PROPERTY_PREFIX) } - .filterKeys { !badKeyConversions.contains(it) }) + return ConfigFactory.parseMap(toProperties().filterKeys { (it as String).startsWith(CORDA_PROPERTY_PREFIX) }.mapKeys { (it.key as String).removePrefix(CORDA_PROPERTY_PREFIX) }) } } diff --git a/node/src/main/kotlin/net/corda/node/services/config/ShadowingException.kt b/node/src/main/kotlin/net/corda/node/services/config/ShadowingException.kt deleted file mode 100644 index 4778fe4284..0000000000 --- a/node/src/main/kotlin/net/corda/node/services/config/ShadowingException.kt +++ /dev/null @@ -1,7 +0,0 @@ -package net.corda.node.services.config - -import com.typesafe.config.ConfigException - -class ShadowingException(definedProperty : String, convertedProperty : String) - : ConfigException( - "Environment variable $definedProperty is shadowing another property transformed to $convertedProperty") \ No newline at end of file diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt index 79ba6b4e8d..a5085085d4 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt @@ -199,8 +199,7 @@ fun driver(defaultParameters: DriverParameters = DriverParameters(), dsl: Dr networkParameters = defaultParameters.networkParameters, notaryCustomOverrides = defaultParameters.notaryCustomOverrides, inMemoryDB = defaultParameters.inMemoryDB, - cordappsForAllNodes = uncheckedCast(defaultParameters.cordappsForAllNodes), - environmentVariables = defaultParameters.environmentVariables + cordappsForAllNodes = uncheckedCast(defaultParameters.cordappsForAllNodes) ), coerce = { it }, dsl = dsl @@ -256,46 +255,10 @@ data class DriverParameters( val networkParameters: NetworkParameters = testNetworkParameters(notaries = emptyList()), val notaryCustomOverrides: Map = emptyMap(), val inMemoryDB: Boolean = true, - val cordappsForAllNodes: Collection? = null, - val environmentVariables : Map = emptyMap() + val cordappsForAllNodes: Collection? = null ) { constructor(cordappsForAllNodes: Collection) : this(isDebug = false, cordappsForAllNodes = cordappsForAllNodes) - constructor( - isDebug: Boolean = false, - driverDirectory: Path = Paths.get("build") / "node-driver" / getTimestampAsDirectoryName(), - portAllocation: PortAllocation = incrementalPortAllocation(), - debugPortAllocation: PortAllocation = incrementalPortAllocation(), - systemProperties: Map = emptyMap(), - useTestClock: Boolean = false, - startNodesInProcess: Boolean = false, - waitForAllNodesToFinish: Boolean = false, - notarySpecs: List = listOf(NotarySpec(DUMMY_NOTARY_NAME)), - extraCordappPackagesToScan: List = emptyList(), - @Suppress("DEPRECATION") jmxPolicy: JmxPolicy = JmxPolicy(), - networkParameters: NetworkParameters = testNetworkParameters(notaries = emptyList()), - notaryCustomOverrides: Map = emptyMap(), - inMemoryDB: Boolean = true, - cordappsForAllNodes: Collection? = null - ) : this( - isDebug, - driverDirectory, - portAllocation, - debugPortAllocation, - systemProperties, - useTestClock, - startNodesInProcess, - waitForAllNodesToFinish, - notarySpecs, - extraCordappPackagesToScan, - jmxPolicy, - networkParameters, - notaryCustomOverrides, - inMemoryDB, - cordappsForAllNodes, - environmentVariables = emptyMap() - ) - constructor( isDebug: Boolean = false, driverDirectory: Path = Paths.get("build") / "node-driver" / getTimestampAsDirectoryName(), @@ -410,7 +373,6 @@ data class DriverParameters( fun withNotaryCustomOverrides(notaryCustomOverrides: Map): DriverParameters = copy(notaryCustomOverrides = notaryCustomOverrides) fun withInMemoryDB(inMemoryDB: Boolean): DriverParameters = copy(inMemoryDB = inMemoryDB) fun withCordappsForAllNodes(cordappsForAllNodes: Collection?): DriverParameters = copy(cordappsForAllNodes = cordappsForAllNodes) - fun withEnvironmentVariables(variables : Map): DriverParameters = copy(environmentVariables = variables) fun copy( isDebug: Boolean, @@ -471,40 +433,4 @@ data class DriverParameters( notaryCustomOverrides = emptyMap(), cordappsForAllNodes = cordappsForAllNodes ) - - @Suppress("LongParameterList") - fun copy( - isDebug: Boolean, - driverDirectory: Path, - portAllocation: PortAllocation, - debugPortAllocation: PortAllocation, - systemProperties: Map, - useTestClock: Boolean, - startNodesInProcess: Boolean, - waitForAllNodesToFinish: Boolean, - notarySpecs: List, - extraCordappPackagesToScan: List, - jmxPolicy: JmxPolicy, - networkParameters: NetworkParameters, - notaryCustomOverrides: Map, - inMemoryDB: Boolean, - cordappsForAllNodes: Collection? - ) = this.copy( - isDebug = isDebug, - driverDirectory = driverDirectory, - portAllocation = portAllocation, - debugPortAllocation = debugPortAllocation, - systemProperties = systemProperties, - useTestClock = useTestClock, - startNodesInProcess = startNodesInProcess, - waitForAllNodesToFinish = waitForAllNodesToFinish, - notarySpecs = notarySpecs, - extraCordappPackagesToScan = extraCordappPackagesToScan, - jmxPolicy = jmxPolicy, - networkParameters = networkParameters, - notaryCustomOverrides = notaryCustomOverrides, - inMemoryDB = inMemoryDB, - cordappsForAllNodes = cordappsForAllNodes, - environmentVariables = emptyMap() - ) } \ No newline at end of file diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt index c34d07b2c4..6d3fa042ea 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt @@ -91,8 +91,7 @@ class DriverDSLImpl( val networkParameters: NetworkParameters, val notaryCustomOverrides: Map, val inMemoryDB: Boolean, - val cordappsForAllNodes: Collection?, - val environmentVariables : Map + val cordappsForAllNodes: Collection? ) : InternalDriverDSL { private var _executorService: ScheduledExecutorService? = null @@ -289,11 +288,9 @@ class DriverDSLImpl( } else { startOutOfProcessMiniNode( config, - arrayOf( - "initial-registration", - "--network-root-truststore=${rootTruststorePath.toAbsolutePath()}", - "--network-root-truststore-password=$rootTruststorePassword" - ) + "initial-registration", + "--network-root-truststore=${rootTruststorePath.toAbsolutePath()}", + "--network-root-truststore-password=$rootTruststorePassword" ).map { config } } } @@ -453,7 +450,7 @@ class DriverDSLImpl( } else { // TODO The config we use here is uses a hardocded p2p port which changes when the node is run proper // This causes two node info files to be generated. - startOutOfProcessMiniNode(config, arrayOf("generate-node-info")).map { + startOutOfProcessMiniNode(config, "generate-node-info").map { // Once done we have to read the signed node info file that's been generated val nodeInfoFile = config.corda.baseDirectory.list { paths -> paths.filter { it.fileName.toString().startsWith(NodeInfoFilesCopier.NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() @@ -539,7 +536,7 @@ class DriverDSLImpl( * Start the node with the given flag which is expected to start the node for some function, which once complete will * terminate the node. */ - private fun startOutOfProcessMiniNode(config: NodeConfig, extraCmdLineFlag: Array = emptyArray()): CordaFuture { + private fun startOutOfProcessMiniNode(config: NodeConfig, vararg extraCmdLineFlag: String): CordaFuture { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null val process = startOutOfProcessNode( config, @@ -548,8 +545,7 @@ class DriverDSLImpl( systemProperties, "512m", null, - environmentVariables, - extraCmdLineFlag + *extraCmdLineFlag ) return poll(executorService, "$extraCmdLineFlag (${config.corda.myLegalName})") { @@ -606,15 +602,7 @@ class DriverDSLImpl( nodeFuture } else { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null - val process = startOutOfProcessNode( - config, - quasarJarPath, - debugPort, - systemProperties, - parameters.maximumHeapSize, - parameters.logLevelOverride, - environmentVariables - ) + val process = startOutOfProcessNode(config, quasarJarPath, debugPort, systemProperties, parameters.maximumHeapSize, parameters.logLevelOverride) // Destroy the child process when the parent exits.This is needed even when `waitForAllNodesToFinish` is // true because we don't want orphaned processes in the case that the parent process is terminated by the @@ -738,7 +726,6 @@ class DriverDSLImpl( } } - @Suppress("LongParameterList") private fun startOutOfProcessNode( config: NodeConfig, quasarJarPath: String, @@ -746,11 +733,9 @@ class DriverDSLImpl( overriddenSystemProperties: Map, maximumHeapSize: String, logLevelOverride: String?, - environmentVariables : Map, - extraCmdLineFlag: Array = emptyArray() - ): Process { - log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, " + - "debug port is " + (debugPort ?: "not enabled")) + vararg extraCmdLineFlag: String + ): Process { + log.info("Starting out-of-process Node ${config.corda.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled")) // Write node.conf writeConfig(config.corda.baseDirectory, "node.conf", config.typesafe.toNodeOnly()) @@ -789,7 +774,7 @@ class DriverDSLImpl( "--base-directory=${config.corda.baseDirectory}", "--logging-level=$loggingLevel", "--no-local-shell").also { - it.addAll(extraCmdLineFlag) + it += extraCmdLineFlag }.toList() // The following dependencies are excluded from the classpath of the created JVM, so that the environment resembles a real one as close as possible. @@ -807,8 +792,7 @@ class DriverDSLImpl( extraJvmArguments = extraJvmArguments, workingDirectory = config.corda.baseDirectory, maximumHeapSize = maximumHeapSize, - classPath = cp, - environmentVariables = environmentVariables + classPath = cp ) } @@ -1029,8 +1013,7 @@ fun genericDriver( networkParameters = defaultParameters.networkParameters, notaryCustomOverrides = defaultParameters.notaryCustomOverrides, inMemoryDB = defaultParameters.inMemoryDB, - cordappsForAllNodes = uncheckedCast(defaultParameters.cordappsForAllNodes), - environmentVariables = defaultParameters.environmentVariables + cordappsForAllNodes = uncheckedCast(defaultParameters.cordappsForAllNodes) ) ) val shutdownHook = addShutdownHook(driverDsl::shutdown) @@ -1107,7 +1090,6 @@ class SplitCompatibilityZoneParams( override fun config() : NetworkServicesConfig = config } -@Suppress("LongParameterList") fun internalDriver( isDebug: Boolean = DriverParameters().isDebug, driverDirectory: Path = DriverParameters().driverDirectory, @@ -1125,7 +1107,6 @@ fun internalDriver( notaryCustomOverrides: Map = DriverParameters().notaryCustomOverrides, inMemoryDB: Boolean = DriverParameters().inMemoryDB, cordappsForAllNodes: Collection? = null, - environmentVariables: Map = emptyMap(), dsl: DriverDSLImpl.() -> A ): A { return genericDriver( @@ -1145,8 +1126,7 @@ fun internalDriver( networkParameters = networkParameters, notaryCustomOverrides = notaryCustomOverrides, inMemoryDB = inMemoryDB, - cordappsForAllNodes = cordappsForAllNodes, - environmentVariables = environmentVariables + cordappsForAllNodes = cordappsForAllNodes ), coerce = { it }, dsl = dsl diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt index cb87b756ff..224d525ca4 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt @@ -7,29 +7,17 @@ import java.time.ZonedDateTime import java.time.format.DateTimeFormatter object ProcessUtilities { - @Suppress("LongParameterList") inline fun startJavaProcess( arguments: List, classPath: List = defaultClassPath, workingDirectory: Path? = null, jdwpPort: Int? = null, extraJvmArguments: List = emptyList(), - maximumHeapSize: String? = null, - environmentVariables: Map = emptyMap() + maximumHeapSize: String? = null ): Process { - return startJavaProcess( - C::class.java.name, - arguments, - classPath, - workingDirectory, - jdwpPort, - extraJvmArguments, - maximumHeapSize, - environmentVariables - ) + return startJavaProcess(C::class.java.name, arguments, classPath, workingDirectory, jdwpPort, extraJvmArguments, maximumHeapSize) } - @Suppress("LongParameterList") fun startJavaProcess( className: String, arguments: List, @@ -37,8 +25,7 @@ object ProcessUtilities { workingDirectory: Path? = null, jdwpPort: Int? = null, extraJvmArguments: List = emptyList(), - maximumHeapSize: String? = null, - environmentVariables: Map = emptyMap() + maximumHeapSize: String? = null ): Process { val command = mutableListOf().apply { add(javaPath) @@ -51,7 +38,6 @@ object ProcessUtilities { } return ProcessBuilder(command).apply { inheritIO() - environment().putAll(environmentVariables) environment()["CLASSPATH"] = classPath.joinToString(File.pathSeparator) if (workingDirectory != null) { // Timestamp may be handy if the same process started, killed and then re-started. Without timestamp diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/RPCDriver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/RPCDriver.kt index b8d0bda519..c1caf61551 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/RPCDriver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/RPCDriver.kt @@ -104,7 +104,6 @@ val fakeNodeLegalName = CordaX500Name(organisation = "Not:a:real:name", locality private val globalPortAllocation = incrementalPortAllocation() private val globalDebugPortAllocation = incrementalPortAllocation() -@Suppress("LongParameterList") fun rpcDriver( isDebug: Boolean = false, driverDirectory: Path = Paths.get("build") / "rpc-driver" / getTimestampAsDirectoryName(), @@ -122,7 +121,6 @@ fun rpcDriver( notaryCustomOverrides: Map = emptyMap(), inMemoryDB: Boolean = true, cordappsForAllNodes: Collection? = null, - environmentVariables: Map = emptyMap(), dsl: RPCDriverDSL.() -> A ): A { return genericDriver( @@ -143,8 +141,7 @@ fun rpcDriver( networkParameters = networkParameters, notaryCustomOverrides = notaryCustomOverrides, inMemoryDB = inMemoryDB, - cordappsForAllNodes = cordappsForAllNodes, - environmentVariables = environmentVariables + cordappsForAllNodes = cordappsForAllNodes ), externalTrace ), coerce = { it }, From 289528350045b8a51d8042cbe395e5f2315f4965 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Wed, 30 Oct 2019 17:47:13 +0000 Subject: [PATCH 02/18] CORDA-3388: Restore mapping of 'java.lang.Void -> void' (#5650) --- .../serialization/internal/amqp/SerializerFactoryBuilder.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index a038af6fe2..4e7fbb466b 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -27,7 +27,8 @@ object SerializerFactoryBuilder { Float::class, Int::class, Long::class, - Short::class + Short::class, + Void::class ).associate { klazz -> klazz.javaObjectType to klazz.javaPrimitiveType }) as Map, Class<*>> From 547e6d9edd0c2cf172a4ea833e984f81337915b6 Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 31 Oct 2019 15:56:25 +0000 Subject: [PATCH 03/18] use jobs to preallocate nodes instead of Pods as they support auto delete --- .../net/corda/testing/PodAllocator.java | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java index e0a11a5fb3..96a880e598 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java +++ b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java @@ -1,8 +1,9 @@ package net.corda.testing; import io.fabric8.kubernetes.api.model.Pod; -import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.batch.Job; +import io.fabric8.kubernetes.api.model.batch.JobBuilder; import io.fabric8.kubernetes.client.Config; import io.fabric8.kubernetes.client.ConfigBuilder; import io.fabric8.kubernetes.client.DefaultKubernetesClient; @@ -48,15 +49,15 @@ public class PodAllocator { KubernetesClient client = new DefaultKubernetesClient(config); - List podsToRequest = IntStream.range(0, number).mapToObj(i -> buildPod("pa-" + prefix + i, coresPerPod, memoryPerPod)).collect(Collectors.toList()); - podsToRequest.forEach(requestedPod -> { - String msg = "PreAllocating " + requestedPod.getMetadata().getName(); + List podsToRequest = IntStream.range(0, number).mapToObj(i -> buildJob("pa-" + prefix + i, coresPerPod, memoryPerPod)).collect(Collectors.toList()); + podsToRequest.forEach(requestedJob -> { + String msg = "PreAllocating " + requestedJob.getMetadata().getName(); if (logger instanceof org.gradle.api.logging.Logger) { ((org.gradle.api.logging.Logger) logger).quiet(msg); } else { logger.info(msg); } - client.pods().inNamespace(KubesTest.NAMESPACE).create(requestedPod); + client.batch().jobs().inNamespace(KubesTest.NAMESPACE).create(requestedJob); }); } @@ -69,20 +70,20 @@ public class PodAllocator { .withWebsocketPingInterval(CONNECTION_TIMEOUT) .build(); KubernetesClient client = new DefaultKubernetesClient(config); - Stream podsToDelete = client.pods().inNamespace(KubesTest.NAMESPACE).list() + Stream jobsToDelete = client.batch().jobs().inNamespace(KubesTest.NAMESPACE).list() .getItems() .stream() .sorted(Comparator.comparing(p -> p.getMetadata().getName())) .filter(foundPod -> foundPod.getMetadata().getName().contains(prefix)); - List> deleteFutures = podsToDelete.map(pod -> { + List> deleteFutures = jobsToDelete.map(job -> { CompletableFuture result = new CompletableFuture<>(); - Watch watch = client.pods().inNamespace(pod.getMetadata().getNamespace()).withName(pod.getMetadata().getName()).watch(new Watcher() { + Watch watch = client.pods().inNamespace(job.getMetadata().getNamespace()).withName(job.getMetadata().getName()).watch(new Watcher() { @Override public void eventReceived(Action action, Pod resource) { if (action == Action.DELETED) { result.complete(resource); - String msg = "Successfully deleted pod " + pod.getMetadata().getName(); + String msg = "Successfully deleted job " + job.getMetadata().getName(); if (logger instanceof org.gradle.api.logging.Logger) { ((org.gradle.api.logging.Logger) logger).lifecycle(msg); } else { @@ -93,7 +94,7 @@ public class PodAllocator { @Override public void onClose(KubernetesClientException cause) { - String message = "Failed to delete pod " + pod.getMetadata().getName(); + String message = "Failed to delete job " + job.getMetadata().getName(); if (logger instanceof org.gradle.api.logging.Logger) { ((org.gradle.api.logging.Logger) logger).quiet(message); } else { @@ -102,7 +103,7 @@ public class PodAllocator { result.completeExceptionally(cause); } }); - client.pods().delete(pod); + client.batch().jobs().delete(job); return result; }).collect(Collectors.toList()); @@ -114,8 +115,14 @@ public class PodAllocator { } - Pod buildPod(String podName, Integer coresPerPod, Integer memoryPerPod) { - return new PodBuilder().withNewMetadata().withName(podName).endMetadata() + Job buildJob(String podName, Integer coresPerPod, Integer memoryPerPod) { + return new JobBuilder().withNewMetadata().withName(podName).endMetadata() + .withNewSpec() + .withTtlSecondsAfterFinished(10) + .withNewTemplate() + .withNewMetadata() + .withName(podName + "-pod") + .endMetadata() .withNewSpec() .addNewContainer() .withImage("busybox:latest") @@ -129,6 +136,8 @@ public class PodAllocator { .endContainer() .withRestartPolicy("Never") .endSpec() + .endTemplate() + .endSpec() .build(); } From bd9d8dbdbdf5e7f605350dcdb3ad796854fba441 Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 31 Oct 2019 15:58:05 +0000 Subject: [PATCH 04/18] promote failure to delete to error log level --- .../src/main/groovy/net/corda/testing/PodAllocator.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java index 96a880e598..8359698f85 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java +++ b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java @@ -84,11 +84,7 @@ public class PodAllocator { if (action == Action.DELETED) { result.complete(resource); String msg = "Successfully deleted job " + job.getMetadata().getName(); - if (logger instanceof org.gradle.api.logging.Logger) { - ((org.gradle.api.logging.Logger) logger).lifecycle(msg); - } else { - logger.info(msg); - } + logger.info(msg); } } @@ -96,7 +92,7 @@ public class PodAllocator { public void onClose(KubernetesClientException cause) { String message = "Failed to delete job " + job.getMetadata().getName(); if (logger instanceof org.gradle.api.logging.Logger) { - ((org.gradle.api.logging.Logger) logger).quiet(message); + ((org.gradle.api.logging.Logger) logger).error(message); } else { logger.info(message); } From 7a9ee89deddf86efcc42663cff03a523f58d5a18 Mon Sep 17 00:00:00 2001 From: stefano Date: Thu, 31 Oct 2019 16:51:33 +0000 Subject: [PATCH 05/18] modify watcher to watch jobs rather than pods --- .../src/main/groovy/net/corda/testing/PodAllocator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java index 8359698f85..7dc64678a8 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java +++ b/buildSrc/src/main/groovy/net/corda/testing/PodAllocator.java @@ -76,11 +76,11 @@ public class PodAllocator { .sorted(Comparator.comparing(p -> p.getMetadata().getName())) .filter(foundPod -> foundPod.getMetadata().getName().contains(prefix)); - List> deleteFutures = jobsToDelete.map(job -> { - CompletableFuture result = new CompletableFuture<>(); - Watch watch = client.pods().inNamespace(job.getMetadata().getNamespace()).withName(job.getMetadata().getName()).watch(new Watcher() { + List> deleteFutures = jobsToDelete.map(job -> { + CompletableFuture result = new CompletableFuture<>(); + Watch watch = client.batch().jobs().inNamespace(job.getMetadata().getNamespace()).withName(job.getMetadata().getName()).watch(new Watcher() { @Override - public void eventReceived(Action action, Pod resource) { + public void eventReceived(Action action, Job resource) { if (action == Action.DELETED) { result.complete(resource); String msg = "Successfully deleted job " + job.getMetadata().getName(); From e0701231acc2b635b172b0caf0fabfc9bbc79011 Mon Sep 17 00:00:00 2001 From: James Higgs <45565019+JamesHR3@users.noreply.github.com> Date: Thu, 31 Oct 2019 17:57:52 +0000 Subject: [PATCH 06/18] [DOCS] Add rows for new database tables in node-database.rst (#5654) --- docs/source/node-database.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/node-database.rst b/docs/source/node-database.rst index f5dadba8bb..ee6c8d3c0f 100644 --- a/docs/source/node-database.rst +++ b/docs/source/node-database.rst @@ -91,8 +91,12 @@ By default, the node database has the following tables: +-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | NODE_CONTRACT_UPGRADES | STATE_REF, CONTRACT_CLASS_NAME | +-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| NODE_HASH_TO_KEY | PK_HASH, PUBLIC_KEY | ++-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | NODE_IDENTITIES | PK_HASH, IDENTITY_VALUE | +-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| NODE_IDENTITIES_NO_CERT | PK_HASH, NAME | ++-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | NODE_INFOS | NODE_INFO_ID, NODE_INFO_HASH, PLATFORM_VERSION, SERIAL | +-----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | NODE_INFO_HOSTS | HOST_NAME, PORT, NODE_INFO_ID, HOSTS_ID | From d033fceeef7cb440edf45fb9f91a21fd5c83a44e Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Fri, 1 Nov 2019 09:05:12 +0000 Subject: [PATCH 07/18] CORDA-3365: Reintroduce dependency to fix BFT-Smart notary (#5640) The commons-codec:1.10 library was removed due to a security vulnerability, but in commons-codec:1.13 it appears to have been fixed. --- node/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/node/build.gradle b/node/build.gradle index ce88a4bf56..fc9b6b9238 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -202,6 +202,7 @@ dependencies { // BFT-Smart dependencies compile 'com.github.bft-smart:library:master-v1.1-beta-g6215ec8-87' + compile 'commons-codec:commons-codec:1.13' // Java Atomix: RAFT library compile 'io.atomix.copycat:copycat-client:1.2.3' From 93ff0728128bba3179b8219fae69c45bc45d00a0 Mon Sep 17 00:00:00 2001 From: snedamle <47854724+snedamle@users.noreply.github.com> Date: Fri, 1 Nov 2019 16:19:04 +0530 Subject: [PATCH 08/18] =?UTF-8?q?adding=20one=20point=20to=20whitelist=20c?= =?UTF-8?q?ontract=20constraints=20migration=20-=20signed=E2=80=A6=20(#556?= =?UTF-8?q?8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * adding one point to whitelist contract constraints migration - signed CorDapp JAR must be registered with the CZ network operator * 1. Removing later releases section 2. Changing 4.0 to 4.3 * Changing 4.3 to |corda_version| --- docs/source/cordapp-constraint-migration.rst | 37 ++++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/source/cordapp-constraint-migration.rst b/docs/source/cordapp-constraint-migration.rst index 25761ff1da..5810cd0e93 100644 --- a/docs/source/cordapp-constraint-migration.rst +++ b/docs/source/cordapp-constraint-migration.rst @@ -63,10 +63,10 @@ Hash constraints migration .. note:: These instructions only apply to CorDapp Contract JARs (unless otherwise stated). -Corda 4.0 +Corda |corda_version| ~~~~~~~~~ -Corda 4.0 requires some additional steps to consume and evolve pre-existing on-ledger **hash** constrained states: +Corda |corda_version| requires some additional steps to consume and evolve pre-existing on-ledger **hash** constrained states: 1. All Corda Nodes in the same CZ or business network that may encounter a transaction chain with a hash constrained state must be started using relaxed hash constraint checking mode as described in :ref:`relax_hash_constraints_checking_ref`. @@ -106,13 +106,6 @@ Corda 4.0 requires some additional steps to consume and evolve pre-existing on-l Please also ensure that the original unsigned contracts CorDapp is removed from the ``/cordapps`` folder (this will already be present in the nodes attachments store) to ensure the lookup code in step 2 retrieves the correct signed contract CorDapp JAR. -Later releases -~~~~~~~~~~~~~~ - -The next version of Corda will provide automatic transition of *hash constrained* states. This means that signed CorDapps running on a Corda 4.x node will -automatically propagate any pre-existing on-ledger *hash-constrained* states (and generate *signature-constrained* outputs) when the system property -to break constraints is set. - .. _cz_whitelisted_constraint_migration: CZ whitelisted constraints migration @@ -120,17 +113,31 @@ CZ whitelisted constraints migration .. note:: These instructions only apply to CorDapp Contract JARs (unless otherwise stated). -Corda 4.0 +Corda |corda_version| ~~~~~~~~~ -Corda 4.0 requires some additional steps to consume and evolve pre-existing on-ledger **CZ whitelisted** constrained states: +Corda |corda_version| requires some additional steps to consume and evolve pre-existing on-ledger **CZ whitelisted** constrained states: 1. As the original developer of the CorDapp, the first step is to sign the latest version of the JAR that was released (see :doc:`cordapp-build-systems`). The key used for signing will be used to sign all subsequent releases, so it should be stored appropriately. The JAR can be signed by multiple keys owned by different parties and it will be expressed as a ``CompositeKey`` in the ``SignatureAttachmentConstraint`` (See :doc:`api-core-types`). -2. Any flow that builds transactions using this CorDapp will automatically transition states to use the ``SignatureAttachmentConstraint`` if - no other constraint is specified. Therefore, there are two ways to alter the existing code. +2. The new Corda 4 signed CorDapp JAR must be registered with the CZ network operator (as whitelisted in the network parameters which are distributed + to all nodes in that CZ). The CZ network operator should check that the JAR is signed and not allow any more versions of it to be whitelisted in the future. + From now on the development organisation that signed the JAR is responsible for signing new versions. + + The process of CZ network CorDapp whitelisting depends on how the Corda network is configured: + + - if using a hosted CZ network (such as `The Corda Network `_ or + `UAT Environment `_ ) running an Identity Operator (formerly known as Doorman) and + Network Map Service, you should manually send the hashes of the two JARs to the CZ network operator and request these be added using + their network parameter update process. + + - if using a local network created using the Network Bootstrapper tool, please follow the instructions in + :ref:`Updating the contract whitelist for bootstrapped networks ` to can add both CorDapp Contract JAR hashes. + +3. Any flow that builds transactions using this CorDapp will automatically transition states to use the ``SignatureAttachmentConstraint`` if + no other constraint is specified and the CorDapp continues to be whitelisted. Therefore, there are two ways to alter the existing code. * Do not specify a constraint * Explicitly add a Signature Constraint @@ -165,6 +172,6 @@ The code below details how to explicitly add a Signature Constraint: // Set the Signature constraint on the new state to migrate away from the WhitelistConstraint. .addOutputState(outputState, myContract, new SignatureAttachmentConstraint(ownersKey)) -3. As a node operator you need to add the new signed version of the contracts CorDapp to the ``/cordapps`` folder together with the latest version of the flows jar. +4. As a node operator you need to add the new signed version of the contracts CorDapp to the ``/cordapps`` folder together with the latest version of the flows jar. Please also ensure that the original unsigned contracts CorDapp is removed from the ``/cordapps`` folder (this will already be present in the - nodes attachments store) to ensure the lookup code in step 3 retrieves the correct signed contract CorDapp JAR. + nodes attachments store) to ensure the lookup code in step 3 retrieves the correct signed contract CorDapp JAR. \ No newline at end of file From e7e7de1d05720dc45b712effb8aa01e013ab6b69 Mon Sep 17 00:00:00 2001 From: stefano Date: Fri, 1 Nov 2019 15:52:20 +0000 Subject: [PATCH 09/18] add regression test jenkins file --- .ci/dev/regression/Jenkinsfile | 35 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 5fb46d4e81..9d6140f87b 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,4 +1,4 @@ -@Library('existing-build-control') +@org.gradle.platform.base.Library('existing-build-control') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) @@ -8,7 +8,7 @@ pipeline { options { timestamps() } environment { - DOCKER_TAG_TO_USE = "${UUID.randomUUID().toString().toLowerCase().subSequence(0, 12)}" + DOCKER_TAG_TO_USE = "${env.GIT_COMMIT.subSequence(0, 8)}" EXECUTOR_NUMBER = "${env.EXECUTOR_NUMBER}" BUILD_ID = "${env.BUILD_ID}-${env.JOB_NAME}" } @@ -21,37 +21,28 @@ pipeline { "-Dkubenetize=true " + "-Ddocker.push.password=\"\${DOCKER_PUSH_PWD}\" " + "-Ddocker.work.dir=\"/tmp/\${EXECUTOR_NUMBER}\" " + - "-Ddocker.provided.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " clean pushBuildImage" + "-Ddocker.build.tag=\"\${DOCKER_TAG_TO_USE}\"" + + " clean pushBuildImage preAllocateForParallelRegressionTest --stacktrace" } sh "kubectl auth can-i get pods" } } - stage('Corda Pull Request - Run Tests') { - stage('Unit and Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelUnitAndIntegrationTest" - } - } - stage('Slow Integration Tests') { - steps { - sh "./gradlew " + - "-DbuildId=\"\${BUILD_ID}\" " + - "-Dkubenetize=true " + - "-Ddocker.tag=\"\${DOCKER_TAG_TO_USE}\"" + - " allParallelSlowIntegrationTest" - } + stage('Regression Test') { + steps { + sh "./gradlew " + + "-DbuildId=\"\${BUILD_ID}\" " + + "-Dkubenetize=true " + + "-Ddocker.run.tag=\"\${DOCKER_TAG_TO_USE}\"" + + " deAllocateForParallelRegressionTest parallelRegressionTest --stacktrace" } } } + post { always { + archiveArtifacts artifacts: '**/pod-logs/**/*.log', fingerprint: false junit '**/build/test-results-xml/**/*.xml' } cleanup { From d54f2ddd874059735ecc040f60fbb95ea23aae09 Mon Sep 17 00:00:00 2001 From: stefano Date: Fri, 1 Nov 2019 15:57:08 +0000 Subject: [PATCH 10/18] fix Library import --- .ci/dev/regression/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/dev/regression/Jenkinsfile b/.ci/dev/regression/Jenkinsfile index 9d6140f87b..e18796ef6c 100644 --- a/.ci/dev/regression/Jenkinsfile +++ b/.ci/dev/regression/Jenkinsfile @@ -1,4 +1,4 @@ -@org.gradle.platform.base.Library('existing-build-control') +@Library('existing-build-control') import static com.r3.build.BuildControl.killAllExistingBuildsForJob killAllExistingBuildsForJob(env.JOB_NAME, env.BUILD_NUMBER.toInteger()) From f8b4b334e38b2b1859623b485fcba79022994e52 Mon Sep 17 00:00:00 2001 From: stefano Date: Fri, 1 Nov 2019 16:10:35 +0000 Subject: [PATCH 11/18] truncate pod name from start rather than end --- .../main/groovy/net/corda/testing/KubesTest.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java index b15aa0791f..1135c0464c 100644 --- a/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java +++ b/buildSrc/src/main/groovy/net/corda/testing/KubesTest.java @@ -104,8 +104,7 @@ public class KubesTest extends DefaultTask { } List> futures = IntStream.range(0, numberOfPods).mapToObj(i -> { - String potentialPodName = (taskToExecuteName + "-" + stableRunId + random + i).toLowerCase(); - String podName = potentialPodName.substring(0, Math.min(potentialPodName.length(), 62)); + String podName = generatePodName(stableRunId, random, i); return submitBuild(NAMESPACE, numberOfPods, i, podName, printOutput, 3); }).collect(Collectors.toList()); @@ -125,6 +124,16 @@ public class KubesTest extends DefaultTask { }).collect(Collectors.toList()); } + @NotNull + private String generatePodName(String stableRunId, String random, int i) { + int magicMaxLength = 63; + String provisionalName = taskToExecuteName.toLowerCase() + "-" + stableRunId + "-" + random + "-" + i; + //length = 100 + //100-63 = 37 + //subString(37, 100) -? string of 63 characters + return provisionalName.substring(Math.max(provisionalName.length() - magicMaxLength, 0)); + } + @NotNull private KubernetesClient getKubernetesClient() { io.fabric8.kubernetes.client.Config config = new io.fabric8.kubernetes.client.ConfigBuilder() From 91e6c9783f8021c21c4d5e624e5207ecfb7e2519 Mon Sep 17 00:00:00 2001 From: Barry Date: Sat, 2 Nov 2019 09:07:53 +0000 Subject: [PATCH 12/18] TM-51 Read and write test results to artifactory. (#5597) * TM-51 Prep for reading and writing test results to artifactory. * TM-51 Tests from target branch if no tests for current branch * TM-51 Placeholder for test averaging over runs. * TM-51 Replace slashes in branch names used as tags. * TM-51 More placeholder work for the mean duration work. * TM-51 Write out average tests results as as csv. The csv file should grow and be updated on each run. This includes whether or not we are running unit tests, integration tests and so on. * TM-51 Comment out old junit test archiving, add more comments. * TM-51 Zip task needs to depend on a csv creation task. If there isn't a csv file present, then the zip task doesn't run due to 'NO-SOURCE' * TM-51 Zip task should ignore empty dirs * TM-51 Fix up loading of test results. We were looking for the wrong artifact name. Add a bit more logging. * TM-51 Fix up possible problem with allocating by class distribution. If we encounter a class we haven't seen before, there won't be any tests. This means we should give it some weight. '1' is far too small. * TM-51 Test that we are definitely increment the run count. Tracking down whether the zipped csv file should have incremented. * TM-51 Better default value for missing test/class names. Begin by using mean unit test duration, but we have the option to bump that to the mean class unit tests duration. * TM-51 More debug information around csv writing. We should be incrementing the tests. * TM-51 Reload the csv before updating it. * TM-51 Reduce verbosity of logging. * TM-51 Reinstate unit tests. Remove logging verbosity. * TM-51 Load tests from artifactory in memory and avoid interim file. * TM-51 Better handling of zero duration tests. Ensure we return zero times from junit artifacts which may either be zero or have no recorded time. Before writing the tests duration csv file, store those with a known time, and then store those with zero using the average time. * TM-51 Log whether we have recorded a test. Tracking down the curious case where we seem to not be rerunning the same set of tests on the second run. * TM-51 Capture junit files as well. Trying to track down whether some tests are intermittently run. * TM-51 Change task dependencies to ensure ziptask is triggered. * TM-51 Remove test assertion, and trigger build * TM-51 Add corda/enterprise to artifactory tag name. Moved properties to own file. * TM-51 Remove unnecessary mean class-based duration. * TM-51 Add more BucketingAllocator tests. We need these to nail down its behaviour some more. * TM-51 Further log information. We don't seem to be finding the tests in the 'production' runs which is odd. * TM-51 corda type double set? * TM-51 do not set the project type in the properties. SRP and all that. * TM-51 better plan reporting * TM-51 duration may be zero Another runtime problem that doesn't show in tests. * TM-51 better plan reporting * fix missing space after image id * fix merge issue in DistributedTesting * TM-51 remove unused code when GET/PUT-ting to Artifactory. * TM-51 put tasks in gradle group and tidy up zip task creation * TM-51 Fix the junit XML path. * TM-51 Fix the task graph * TM-51 Less logging --- .idea/codeStyles/Project.xml | 3 + Jenkinsfile | 7 +- .../groovy/net/corda/testing/Artifactory.java | 146 + .../net/corda/testing/BucketingAllocator.java | 89 +- .../corda/testing/BucketingAllocatorTask.java | 39 +- .../corda/testing/DistributedTesting.groovy | 37 +- .../groovy/net/corda/testing/KubesTest.java | 12 +- .../groovy/net/corda/testing/Properties.java | 87 + .../corda/testing/TestDurationArtifacts.java | 429 +++ .../main/groovy/net/corda/testing/Tests.java | 199 + .../net/corda/testing/PropertiesTest.java | 56 + .../testing/TestDurationArtifactsTest.java | 323 ++ .../groovy/net/corda/testing/TestsTest.java | 145 + .../corda/testing/BucketingAllocatorTest.java | 134 +- testing/test-times.csv | 3224 ----------------- 15 files changed, 1628 insertions(+), 3302 deletions(-) create mode 100644 buildSrc/src/main/groovy/net/corda/testing/Artifactory.java create mode 100644 buildSrc/src/main/groovy/net/corda/testing/Properties.java create mode 100644 buildSrc/src/main/groovy/net/corda/testing/TestDurationArtifacts.java create mode 100644 buildSrc/src/main/groovy/net/corda/testing/Tests.java create mode 100644 buildSrc/src/test/groovy/net/corda/testing/PropertiesTest.java create mode 100644 buildSrc/src/test/groovy/net/corda/testing/TestDurationArtifactsTest.java create mode 100644 buildSrc/src/test/groovy/net/corda/testing/TestsTest.java delete mode 100644 testing/test-times.csv diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 0d2d9001a5..ebb620b41a 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -32,6 +32,9 @@