From 7517270ddedc5dee873179babd39a9f524796e29 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Mon, 26 Feb 2018 14:53:13 +0000 Subject: [PATCH 01/14] Improves node.conf page. Shows the defaults used. --- docs/source/corda-configuration-file.rst | 98 +++++++++++++----------- docs/source/node-database.rst | 44 +++++++++-- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/docs/source/corda-configuration-file.rst b/docs/source/corda-configuration-file.rst index 3c41f75d5b..9f954d3bb5 100644 --- a/docs/source/corda-configuration-file.rst +++ b/docs/source/corda-configuration-file.rst @@ -1,64 +1,47 @@ Node configuration ================== +.. contents:: + File location ------------- +When starting a node, the ``corda.jar`` file defaults to reading the node's configuration from a ``node.conf`` file in +the directory from which the command to launch Corda is executed. There are two command-line options to override this +behaviour: -The Corda all-in-one ``corda.jar`` file is generated by the ``gradle buildCordaJAR`` task and defaults to reading configuration -from a ``node.conf`` file in the present working directory. This behaviour can be overidden using the ``--config-file`` -command line option to target configuration files with different names, or different file location (relative paths are -relative to the current working directory). Also, the ``--base-directory`` command line option alters the Corda node -workspace location and if specified a ``node.conf`` configuration file is then expected in the root of the workspace. +* The ``--config-file`` command line option allows you to specify a configuration file with a different name, or at + different file location. Paths are relative to the current working directory -The configuration file templates used for the ``gradle deployNodes`` task are to be found in the ``/config/dev`` folder. -Also note that there is a basic set of defaults loaded from the built in resource file ``/node/src/main/resources/reference.conf`` -of the ``:node`` gradle module. All properties in this can be overidden in the file configuration and for rarely changed -properties this defaulting allows the property to be excluded from the configuration file. +* The ``--base-directory`` command line option allows you to specify the node's workspace location. A ``node.conf`` + configuration file is then expected in the root of this workspace + +If you specify both command line arguments at the same time, the node will fail to start. Format ------ +The Corda configuration file uses the HOCON format which is superset of JSON. Please visit +``_ for further details. -The Corda configuration file uses the HOCON format which is superset of JSON. It has several features which makes it -very useful as a configuration format. Please visit their `page `_ -for further details. - -Examples +Defaults -------- +A set of default configuration options are loaded from the built-in resource file ``/node/src/main/resources/reference.conf``. +This file can be found in the ``:node`` gradle module of the `Corda repository `_. Any +options you do not specify in your own ``node.conf`` file will use these defaults. -General node configuration file for hosting the IRSDemo services. +Here are the contents of the ``reference.conf`` file: -.. literalinclude:: example-code/src/main/resources/example-node.conf +.. literalinclude:: ../../node/src/main/resources/reference.conf :language: javascript -Simple Notary configuration file. - -.. parsed-literal:: - - myLegalName : "O=Notary Service,OU=corda,L=London,C=GB" - keyStorePassword : "cordacadevpass" - trustStorePassword : "trustpass" - p2pAddress : "localhost:12345" - rpcSettings = { - useSsl = false - standAloneBroker = false - address : "my-corda-node:10003" - adminAddress : "my-corda-node:10004" - } - webAddress : "localhost:12347" - notary : { - validating : false - } - devMode : true - compatibilityZoneURL : "https://cz.corda.net" - Fields ------ +The available config fields are listed below. ``baseDirectory`` is available as a substitution value and contains the +absolute path to the node's base directory. -The available config fields are listed below. ``baseDirectory`` is available as a substitution value, containing the absolute -path to the node's base directory. - -:myLegalName: The legal identity of the node acts as a human readable alias to the node's public key and several demos use - this to lookup the NodeInfo. +:myLegalName: The legal identity of the node. This acts as a human-readable alias to the node's public key and can be used with + the network map to look up the node's info. This is the name that is used in the node's certificates (either when requesting them + from the doorman, or when auto-generating them in dev mode). At runtime, Corda checks whether this name matches the + name in the node's certificates. :keyStorePassword: The password to unlock the KeyStore file (``/certificates/sslkeystore.jks``) containing the node certificate and private key. @@ -195,4 +178,33 @@ path to the node's base directory. Otherwise defaults to 10MB :attachmentCacheBound: Optionally specify how many attachments should be cached locally. Note that this includes only the key and - metadata, the content is cached separately and can be loaded lazily. Defaults to 1024. \ No newline at end of file + metadata, the content is cached separately and can be loaded lazily. Defaults to 1024. + +Examples +-------- + +General node configuration file for hosting the IRSDemo services: + +.. literalinclude:: example-code/src/main/resources/example-node.conf + :language: javascript + +Simple notary configuration file: + +.. parsed-literal:: + + myLegalName : "O=Notary Service,OU=corda,L=London,C=GB" + keyStorePassword : "cordacadevpass" + trustStorePassword : "trustpass" + p2pAddress : "localhost:12345" + rpcSettings = { + useSsl = false + standAloneBroker = false + address : "my-corda-node:10003" + adminAddress : "my-corda-node:10004" + } + webAddress : "localhost:12347" + notary : { + validating : false + } + devMode : true + compatibilityZoneURL : "https://cz.corda.net" \ No newline at end of file diff --git a/docs/source/node-database.rst b/docs/source/node-database.rst index e38cde81c9..c429abc082 100644 --- a/docs/source/node-database.rst +++ b/docs/source/node-database.rst @@ -1,16 +1,14 @@ Node database ============= -Currently, nodes store their data in an H2 database. In the future, we plan to support a wide range of databases. - -You can connect directly to a running node's database to see its stored states, transactions and attachments as -follows: +Default in-memory database +-------------------------- +By default, nodes store their data in an H2 database. You can connect directly to a running node's database to see its +stored states, transactions and attachments as follows: * Download the `h2 platform-independent zip `_, unzip the zip, and navigate in a terminal window to the unzipped folder -* Change directories to the bin folder: - - ``cd h2/bin`` +* Change directories to the bin folder: ``cd h2/bin`` * Run the following command to open the h2 web console in a web browser tab: @@ -25,4 +23,34 @@ follows: * Paste this string into the JDBC URL field and click ``Connect``, using the default username and password. You will be presented with a web interface that shows the contents of your node's storage and vault, and provides an -interface for you to query them using SQL. \ No newline at end of file +interface for you to query them using SQL. + +PostgreSQL +---------- +Nodes also have untested support for PostgreSQL 9.6, using PostgreSQL JDBC Driver 42.1.4. + +.. warning:: This is an experimental community contribution, and is currently untested. We welcome pull requests to add + tests and additional support for this feature. + +Configuration +~~~~~~~~~~~~~ +Here is an example node configuration for PostgreSQL: + +.. sourcecode:: groovy + + dataSourceProperties = { + dataSourceClassName = "org.postgresql.ds.PGSimpleDataSource" + dataSource.url = "jdbc:postgresql://[HOST]:[PORT]/postgres" + dataSource.user = [USER] + dataSource.password = [PASSWORD] + } + database = { + transactionIsolationLevel = READ_COMMITTED + schema = [SCHEMA] + } + +Note that: + +* The ``database.schema`` property is optional +* The value of ``database.schema`` is not wrapped in double quotes and Postgres always treats it as a lower-case value + (e.g. ``AliceCorp`` becomes ``alicecorp``) \ No newline at end of file From cc84b34dcabd0c0dd24f7f32e53fac0e076cdc22 Mon Sep 17 00:00:00 2001 From: Maksymilian Pawlak <120831+m4ksio@users.noreply.github.com> Date: Mon, 26 Feb 2018 15:01:57 +0000 Subject: [PATCH 02/14] Close classloader to release associated file (#2635) [CORDA-1113] Close classloader to release associated file --- .../main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt index 774e30e2c0..49739303a1 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt @@ -26,6 +26,7 @@ fun scanJarForContracts(cordappJarPath: String): List { // Only keep instantiable contracts val classLoader = URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader) val concreteContracts = contracts.map(classLoader::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) } + classLoader.close() return concreteContracts.map { it.name } } From b91ef43e0fc0730f54887366a2559cd0fcd0e12c Mon Sep 17 00:00:00 2001 From: Maksymilian Pawlak <120831+m4ksio@users.noreply.github.com> Date: Mon, 26 Feb 2018 15:47:04 +0000 Subject: [PATCH 03/14] Remove the debug helper which should never make it into master (#2633) * Remove IRS Docker demo helpers --- .../irs-demo/src/system-test/kotlin/IRSDemoDockerTest.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/samples/irs-demo/src/system-test/kotlin/IRSDemoDockerTest.kt b/samples/irs-demo/src/system-test/kotlin/IRSDemoDockerTest.kt index b40e829c9f..cce25fa4f3 100644 --- a/samples/irs-demo/src/system-test/kotlin/IRSDemoDockerTest.kt +++ b/samples/irs-demo/src/system-test/kotlin/IRSDemoDockerTest.kt @@ -72,15 +72,9 @@ class IRSDemoDockerTest { //Wait for deals to appear in a rows table val dealsList = driverWait.until({ - makeScreenshot(driver, "second") it?.findElement(By.cssSelector("table#deal-list tbody tr")) }) assertNotNull(dealsList) } - - private fun makeScreenshot(driver: PhantomJSDriver, name: String) { - val screenshotAs = driver.getScreenshotAs(OutputType.FILE) - Files.copy(screenshotAs.toPath(), Paths.get("/Users", "maksymilianpawlak", "phantomjs", name + System.currentTimeMillis() + ".png"), StandardCopyOption.REPLACE_EXISTING) - } } From 2da28c574eda301a7db6a8234c84c39bba19ec7e Mon Sep 17 00:00:00 2001 From: Rick Parker Date: Mon, 26 Feb 2018 16:07:40 +0000 Subject: [PATCH 04/14] CORDA-1040 docs for FlowLogic.sleep (#2625) --- docs/source/api-flows.rst | 32 +++++++++++++++++++ .../cash/selection/AbstractCashSelection.kt | 7 +++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/source/api-flows.rst b/docs/source/api-flows.rst index 780c2572da..e3da7f5ad0 100644 --- a/docs/source/api-flows.rst +++ b/docs/source/api-flows.rst @@ -744,3 +744,35 @@ We then update the progress tracker's current step as we progress through the fl :start-after: DOCSTART 18 :end-before: DOCEND 18 :dedent: 12 + + +Concurrency, Locking and Waiting +-------------------------------- +This is an advanced topic. Because Corda is designed to: + +* run many flows in parallel, +* may persist flows to storage and resurrect those flows much later, +* (in the future) migrate flows between JVMs, + +flows should avoid use of locks and typically not even attempt to interact with objects shared between flows (except +``ServiceHub`` and other carefully crafted services such as Oracles. See :doc:`oracles`). +Locks will significantly reduce the scalability of the node, in the best case, and can cause the node to deadlock if they +remain locked across flow context switch boundaries (such as sending and receiving +from peers discussed above, and the sleep discussed below). + +If you need activities that are scheduled, you should investigate the use of ``SchedulableState``. +However, we appreciate that Corda support for some more advanced patterns is still in the future, and if there is a need +for brief pauses in flows then you should use ``FlowLogic.sleep`` in place of where you might have used ``Thread.sleep``. +Flows should expressly not use ``Thread.sleep``, since this will prevent the node from processing other flows +in the meantime, significantly impairing the performance of the node. +Even ``FlowLogic.sleep`` is not to be used to create long running flows, since the Corda ethos is for short lived flows +(otherwise upgrading nodes or CorDapps is much more complicated), or as a substitute to using the ``SchedulableState`` scheduler. + +Currently the ``finance`` package uses ``FlowLogic.sleep`` to make several attempts at coin selection, where necessary, +when many states are soft locked and we wish to wait for those, or other new states in their place, to become unlocked. + + .. literalinclude:: ../../finance/src/main/kotlin/net/corda/finance/contracts/asset/cash/selection/AbstractCashSelection.kt + :language: kotlin + :start-after: DOCSTART CASHSELECT 1 + :end-before: DOCEND CASHSELECT 1 + :dedent: 8 diff --git a/finance/src/main/kotlin/net/corda/finance/contracts/asset/cash/selection/AbstractCashSelection.kt b/finance/src/main/kotlin/net/corda/finance/contracts/asset/cash/selection/AbstractCashSelection.kt index eff93f6d3f..2fc04a2d46 100644 --- a/finance/src/main/kotlin/net/corda/finance/contracts/asset/cash/selection/AbstractCashSelection.kt +++ b/finance/src/main/kotlin/net/corda/finance/contracts/asset/cash/selection/AbstractCashSelection.kt @@ -12,7 +12,10 @@ import net.corda.core.node.ServiceHub import net.corda.core.node.services.StatesNotAvailableException import net.corda.core.utilities.* import net.corda.finance.contracts.asset.Cash -import java.sql.* +import java.sql.Connection +import java.sql.DatabaseMetaData +import java.sql.ResultSet +import java.sql.SQLException import java.util.* import java.util.concurrent.atomic.AtomicReference import java.util.concurrent.locks.ReentrantLock @@ -101,6 +104,7 @@ abstract class AbstractCashSelection { withIssuerRefs: Set = emptySet()): List> { val stateAndRefs = mutableListOf>() + // DOCSTART CASHSELECT 1 for (retryCount in 1..MAX_RETRIES) { if (!attemptSpend(services, amount, lockId, notary, onlyFromIssuerParties, withIssuerRefs, stateAndRefs)) { log.warn("Coin selection failed on attempt $retryCount") @@ -116,6 +120,7 @@ abstract class AbstractCashSelection { break } } + // DOCEND CASHSELECT 1 return stateAndRefs } From f1c3f584acc712e156e1f888069eb8665e118835 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Mon, 26 Feb 2018 16:47:53 +0000 Subject: [PATCH 05/14] Showcase link in Quickstart guide. Remove Utilities link (merged with Samples). --- docs/source/quickstart-index.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/source/quickstart-index.rst b/docs/source/quickstart-index.rst index fa884b229d..c0b655e7aa 100644 --- a/docs/source/quickstart-index.rst +++ b/docs/source/quickstart-index.rst @@ -1,10 +1,7 @@ Quickstart ========== -.. toctree:: - :maxdepth: 1 - - getting-set-up - tutorial-cordapp - Other CorDapps - Utilities \ No newline at end of file +* :doc:`Set up your machine for CorDapp development ` +* :doc:`Run the Example CorDapp ` +* `View CorDapps in Corda Explore `_ +* `Download sample CorDapps `_ \ No newline at end of file From 9da3a8ee055882108ca9f8238be16abd8365a0c4 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 26 Feb 2018 17:11:10 +0000 Subject: [PATCH 06/14] [ENT-1552] Delete temporary truststore once the test is complete. (#484) (#2634) --- .../kotlin/net/corda/node/ArgsParserTest.kt | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt index 8ddb9707e9..bb664bb4a7 100644 --- a/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt +++ b/node/src/test/kotlin/net/corda/node/ArgsParserTest.kt @@ -5,15 +5,29 @@ import net.corda.core.internal.div import net.corda.nodeapi.internal.crypto.X509KeyStore import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatExceptionOfType +import org.junit.BeforeClass import org.junit.Test import org.slf4j.event.Level +import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import kotlin.test.assertEquals import kotlin.test.assertNotNull class ArgsParserTest { private val parser = ArgsParser() - private val workingDirectory = Paths.get(".").normalize().toAbsolutePath() + + companion object { + private lateinit var workingDirectory: Path + private lateinit var buildDirectory: Path + + @BeforeClass + @JvmStatic + fun initDirectories() { + workingDirectory = Paths.get(".").normalize().toAbsolutePath() + buildDirectory = workingDirectory.resolve("build") + } + } @Test fun `no command line arguments`() { @@ -113,17 +127,21 @@ class ArgsParserTest { @Test fun `initial-registration`() { - val truststorePath = workingDirectory / "truststore" / "file.jks" + // Create this temporary file in the "build" directory so that "clean" can delete it. + val truststorePath = buildDirectory / "truststore" / "file.jks" assertThatExceptionOfType(IllegalArgumentException::class.java).isThrownBy { parser.parse("--initial-registration", "--network-root-truststore", "$truststorePath", "--network-root-truststore-password", "password-test") }.withMessageContaining("Network root trust store path").withMessageContaining("doesn't exist") X509KeyStore.fromFile(truststorePath, "dummy_password", createNew = true) - - val cmdLineOptions = parser.parse("--initial-registration", "--network-root-truststore", "$truststorePath", "--network-root-truststore-password", "password-test") - assertNotNull(cmdLineOptions.nodeRegistrationConfig) - assertEquals(truststorePath.toAbsolutePath(), cmdLineOptions.nodeRegistrationConfig?.networkRootTrustStorePath) - assertEquals("password-test", cmdLineOptions.nodeRegistrationConfig?.networkRootTrustStorePassword) + try { + val cmdLineOptions = parser.parse("--initial-registration", "--network-root-truststore", "$truststorePath", "--network-root-truststore-password", "password-test") + assertNotNull(cmdLineOptions.nodeRegistrationConfig) + assertEquals(truststorePath.toAbsolutePath(), cmdLineOptions.nodeRegistrationConfig?.networkRootTrustStorePath) + assertEquals("password-test", cmdLineOptions.nodeRegistrationConfig?.networkRootTrustStorePassword) + } finally { + Files.delete(truststorePath) + } } @Test From 3c8db47b5a6d928617dc8f5326e62fe6f4f803b3 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Mon, 26 Feb 2018 18:16:25 +0000 Subject: [PATCH 07/14] Updates shell example of starting flow to reflect cash API change. --- docs/source/shell.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/shell.rst b/docs/source/shell.rst index 86833f002b..90edacf5ff 100644 --- a/docs/source/shell.rst +++ b/docs/source/shell.rst @@ -184,7 +184,7 @@ Starting a flow We would start the ``CashIssue`` flow as follows: -``flow start CashIssue amount: $1000, issueRef: 1234, recipient: "O=Bank A,L=London,C=GB", notary: "O=Notary Service,OU=corda,L=London,C=GB"`` +``flow start CashIssueFlow amount: $1000, issuerBankPartyRef: 1234, notary: "O=Controller, L=London, C=GB"`` This breaks down as follows: From 0adc203d7ad771e628c14b10786891a91055a9ec Mon Sep 17 00:00:00 2001 From: Andrius Dagys Date: Mon, 26 Feb 2018 18:50:46 +0000 Subject: [PATCH 08/14] CORDA-696: Fix conflict resolution error, fix filename type --- ...iesForResolutionImpl.kt => ServicesForResolutionImpl.kt} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename node/src/main/kotlin/net/corda/node/internal/{ServiesForResolutionImpl.kt => ServicesForResolutionImpl.kt} (94%) diff --git a/node/src/main/kotlin/net/corda/node/internal/ServiesForResolutionImpl.kt b/node/src/main/kotlin/net/corda/node/internal/ServicesForResolutionImpl.kt similarity index 94% rename from node/src/main/kotlin/net/corda/node/internal/ServiesForResolutionImpl.kt rename to node/src/main/kotlin/net/corda/node/internal/ServicesForResolutionImpl.kt index c653a689b0..369bf89a33 100644 --- a/node/src/main/kotlin/net/corda/node/internal/ServiesForResolutionImpl.kt +++ b/node/src/main/kotlin/net/corda/node/internal/ServicesForResolutionImpl.kt @@ -23,10 +23,10 @@ data class ServicesForResolutionImpl( @Throws(TransactionResolutionException::class) override fun loadStates(stateRefs: Set): Set> { - return stateRefs.groupBy { it.txhash }.map { + return stateRefs.groupBy { it.txhash }.flatMap { val stx = validatedTransactions.getTransaction(it.key) ?: throw TransactionResolutionException(it.key) val baseTx = stx.resolveBaseTransaction(this) it.value.map { StateAndRef(baseTx.outputs[it.index], it) } - }.flatMap { it }.toSet() + }.toSet() } -} \ No newline at end of file +} From 147f46fbf48a1cccf0f373cc65868be960063acd Mon Sep 17 00:00:00 2001 From: Thomas Schroeter Date: Tue, 27 Feb 2018 10:00:30 +0000 Subject: [PATCH 09/14] [CORDA-1119] Sign in DB transaction (#2645) --- .../kotlin/net/corda/node/services/transactions/BFTSMaRt.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/transactions/BFTSMaRt.kt b/node/src/main/kotlin/net/corda/node/services/transactions/BFTSMaRt.kt index 899ecde450..aa3027559b 100644 --- a/node/src/main/kotlin/net/corda/node/services/transactions/BFTSMaRt.kt +++ b/node/src/main/kotlin/net/corda/node/services/transactions/BFTSMaRt.kt @@ -248,7 +248,7 @@ object BFTSMaRt { /** Generates a transaction signature over the specified transaction [txId]. */ protected fun sign(txId: SecureHash): TransactionSignature { val signableData = SignableData(txId, SignatureMetadata(services.myInfo.platformVersion, Crypto.findSignatureScheme(notaryIdentityKey).schemeNumberID)) - return services.keyManagementService.sign(signableData, notaryIdentityKey) + return services.database.transaction { services.keyManagementService.sign(signableData, notaryIdentityKey) } } // TODO: From f7c9f0d10e909f6e54db2356385f3fac3abe19e8 Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Tue, 27 Feb 2018 10:21:03 +0000 Subject: [PATCH 10/14] Wrong file name. --- docs/source/hello-world-template.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/hello-world-template.rst b/docs/source/hello-world-template.rst index cf34d8c892..616ad7bd78 100644 --- a/docs/source/hello-world-template.rst +++ b/docs/source/hello-world-template.rst @@ -67,7 +67,7 @@ To prevent build errors later on, we should delete the following files before we * Java: ``cordapp/src/main/java/com/template/TemplateClient.java`` -* Kotlin: ``cordapp/src/main/kotlin/com/template/TemplateClient.kt`` +* Kotlin: ``cordapp/src/main/kotlin/com/template/Client.kt`` Progress so far --------------- From 33918101017721039487cb9d99711e6b405e7b79 Mon Sep 17 00:00:00 2001 From: Andrzej Cichocki Date: Tue, 27 Feb 2018 10:22:30 +0000 Subject: [PATCH 11/14] CORDA-1004 Quasar-friendly ThreadLocal solution (#2594) * Use FastThreadLocalThread in fiber scheduler * Test that thread locals aren't serialized --- .../corda/core/flows/FastThreadLocalTest.kt | 165 ++++++++++++++++++ .../corda/node/utilities/AffinityExecutor.kt | 9 +- testing/test-utils/build.gradle | 1 + 3 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 core/src/test/kotlin/net/corda/core/flows/FastThreadLocalTest.kt diff --git a/core/src/test/kotlin/net/corda/core/flows/FastThreadLocalTest.kt b/core/src/test/kotlin/net/corda/core/flows/FastThreadLocalTest.kt new file mode 100644 index 0000000000..516d6babcb --- /dev/null +++ b/core/src/test/kotlin/net/corda/core/flows/FastThreadLocalTest.kt @@ -0,0 +1,165 @@ +package net.corda.core.flows + +import co.paralleluniverse.fibers.Fiber +import co.paralleluniverse.fibers.FiberExecutorScheduler +import co.paralleluniverse.fibers.Suspendable +import co.paralleluniverse.io.serialization.ByteArraySerializer +import co.paralleluniverse.strands.SuspendableCallable +import io.netty.util.concurrent.FastThreadLocal +import io.netty.util.concurrent.FastThreadLocalThread +import net.corda.core.internal.concurrent.OpenFuture +import net.corda.core.internal.concurrent.openFuture +import net.corda.core.internal.rootCause +import net.corda.core.utilities.getOrThrow +import org.assertj.core.api.Assertions.catchThrowable +import org.hamcrest.Matchers.lessThanOrEqualTo +import org.junit.After +import org.junit.Assert.assertThat +import org.junit.Test +import java.util.* +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class FastThreadLocalTest { + private inner class ExpensiveObj { + init { + expensiveObjCount.andIncrement + } + } + + private val expensiveObjCount = AtomicInteger() + private lateinit var pool: ExecutorService + private lateinit var scheduler: FiberExecutorScheduler + private fun init(threadCount: Int, threadImpl: (Runnable) -> Thread) { + pool = Executors.newFixedThreadPool(threadCount, threadImpl) + scheduler = FiberExecutorScheduler(null, pool) + } + + @After + fun poolShutdown() = try { + pool.shutdown() + } catch (e: UninitializedPropertyAccessException) { + // Do nothing. + } + + @After + fun schedulerShutdown() = try { + scheduler.shutdown() + } catch (e: UninitializedPropertyAccessException) { + // Do nothing. + } + + @Test + fun `ThreadLocal with plain old Thread is fiber-local`() { + init(3, ::Thread) + val threadLocal = object : ThreadLocal() { + override fun initialValue() = ExpensiveObj() + } + assertEquals(0, runFibers(100, threadLocal::get)) + assertEquals(100, expensiveObjCount.get()) + } + + @Test + fun `ThreadLocal with FastThreadLocalThread is fiber-local`() { + init(3, ::FastThreadLocalThread) + val threadLocal = object : ThreadLocal() { + override fun initialValue() = ExpensiveObj() + } + assertEquals(0, runFibers(100, threadLocal::get)) + assertEquals(100, expensiveObjCount.get()) + } + + @Test + fun `FastThreadLocal with plain old Thread is fiber-local`() { + init(3, ::Thread) + val threadLocal = object : FastThreadLocal() { + override fun initialValue() = ExpensiveObj() + } + assertEquals(0, runFibers(100, threadLocal::get)) + assertEquals(100, expensiveObjCount.get()) + } + + @Test + fun `FastThreadLocal with FastThreadLocalThread is not fiber-local`() { + init(3, ::FastThreadLocalThread) + val threadLocal = object : FastThreadLocal() { + override fun initialValue() = ExpensiveObj() + } + runFibers(100, threadLocal::get) // Return value could be anything. + assertThat(expensiveObjCount.get(), lessThanOrEqualTo(3)) + } + + /** @return the number of times a different expensive object was obtained post-suspend. */ + private fun runFibers(fiberCount: Int, threadLocalGet: () -> ExpensiveObj): Int { + val fibers = (0 until fiberCount).map { Fiber(scheduler, FiberTask(threadLocalGet)) } + val startedFibers = fibers.map { it.start() } + return startedFibers.map { it.get() }.count { it } + } + + private class FiberTask(private val threadLocalGet: () -> ExpensiveObj) : SuspendableCallable { + @Suspendable + override fun run(): Boolean { + val first = threadLocalGet() + Fiber.sleep(1) + return threadLocalGet() != first + } + } + + private class UnserializableObj { + @Suppress("unused") + private val fail: Nothing by lazy { throw UnsupportedOperationException("Nice try.") } + } + + @Test + fun `ThreadLocal content is not serialized`() { + contentIsNotSerialized(object : ThreadLocal() { + override fun initialValue() = UnserializableObj() + }::get) + } + + @Test + fun `FastThreadLocal content is not serialized`() { + contentIsNotSerialized(object : FastThreadLocal() { + override fun initialValue() = UnserializableObj() + }::get) + } + + private fun contentIsNotSerialized(threadLocalGet: () -> UnserializableObj) { + init(1, ::FastThreadLocalThread) + // Use false like AbstractKryoSerializationScheme, the default of true doesn't work at all: + val serializer = Fiber.getFiberSerializer(false) + val returnValue = UUID.randomUUID() + val deserializedFiber = serializer.read(openFuture().let { + Fiber(scheduler, FiberTask2(threadLocalGet, false, serializer, it, returnValue)).start() + it.getOrThrow() + }) as Fiber<*> + assertEquals(returnValue, Fiber.unparkDeserialized(deserializedFiber, scheduler).get()) + assertEquals("Nice try.", openFuture().let { + Fiber(scheduler, FiberTask2(threadLocalGet, true, serializer, it, returnValue)).start() + catchThrowable { it.getOrThrow() } + }.rootCause.message) + } + + private class FiberTask2( + @Transient private val threadLocalGet: () -> UnserializableObj, + private val retainObj: Boolean, + @Transient private val serializer: ByteArraySerializer, + @Transient private val bytesFuture: OpenFuture, + private val returnValue: UUID) : SuspendableCallable { + @Suspendable + override fun run(): UUID { + var obj: UnserializableObj? = threadLocalGet() + assertNotNull(obj) + if (!retainObj) { + @Suppress("UNUSED_VALUE") + obj = null + } + // In retainObj false case, check this doesn't attempt to serialize fields of currentThread: + Fiber.parkAndSerialize { fiber, _ -> bytesFuture.capture { serializer.write(fiber) } } + return returnValue + } + } +} diff --git a/node/src/main/kotlin/net/corda/node/utilities/AffinityExecutor.kt b/node/src/main/kotlin/net/corda/node/utilities/AffinityExecutor.kt index 096a958a22..1523e24444 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/AffinityExecutor.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/AffinityExecutor.kt @@ -1,6 +1,7 @@ package net.corda.node.utilities import com.google.common.util.concurrent.SettableFuture +import io.netty.util.concurrent.FastThreadLocalThread import java.util.* import java.util.concurrent.CompletableFuture import java.util.concurrent.Executor @@ -55,8 +56,8 @@ interface AffinityExecutor : Executor { private val threads = Collections.synchronizedSet(HashSet()) init { - setThreadFactory(fun(runnable: Runnable): Thread { - val thread = object : Thread() { + setThreadFactory { runnable -> + val thread = object : FastThreadLocalThread() { override fun run() { try { runnable.run() @@ -68,8 +69,8 @@ interface AffinityExecutor : Executor { thread.isDaemon = true thread.name = threadName threads += thread - return thread - }) + thread + } } override val isOnThread: Boolean get() = Thread.currentThread() in threads diff --git a/testing/test-utils/build.gradle b/testing/test-utils/build.gradle index b67dcbcb2f..6b9c9f55fa 100644 --- a/testing/test-utils/build.gradle +++ b/testing/test-utils/build.gradle @@ -20,6 +20,7 @@ dependencies { // Unit testing helpers. compile "junit:junit:$junit_version" + compile 'org.hamcrest:hamcrest-library:1.3' compile "com.nhaarman:mockito-kotlin:1.1.0" // Guava: Google test library (collections test suite) From cd569577d254ba177d5cf435f396538703fd824e Mon Sep 17 00:00:00 2001 From: Andras Slemmer Date: Tue, 27 Feb 2018 10:23:48 +0000 Subject: [PATCH 12/14] Fix rpc sender thread busy looping --- .../main/kotlin/net/corda/node/services/messaging/RPCServer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/RPCServer.kt b/node/src/main/kotlin/net/corda/node/services/messaging/RPCServer.kt index db3d90bef5..657de9535e 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/RPCServer.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/RPCServer.kt @@ -210,7 +210,7 @@ class RPCServer( return thread(name = "rpc-server-sender", isDaemon = true) { var deduplicationSequenceNumber = 0L while (true) { - val job = sendJobQueue.poll() + val job = sendJobQueue.take() when (job) { is RpcSendJob.Send -> handleSendJob(deduplicationSequenceNumber++, job) RpcSendJob.Stop -> return@thread From 3066926f0fa7fbc481ee16a89a626ce5aeaf2f5e Mon Sep 17 00:00:00 2001 From: Maksymilian Pawlak <120831+m4ksio@users.noreply.github.com> Date: Tue, 27 Feb 2018 11:15:23 +0000 Subject: [PATCH 13/14] Improved classloader closing (#2650) * Better handling of classloader closing --- .../kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt index 49739303a1..124cda119d 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt @@ -24,10 +24,9 @@ fun scanJarForContracts(cordappJarPath: String): List { val contracts = (scanResult.getNamesOfClassesImplementing(Contract::class.qualifiedName) ).distinct() // Only keep instantiable contracts - val classLoader = URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader) - val concreteContracts = contracts.map(classLoader::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) } - classLoader.close() - return concreteContracts.map { it.name } + return URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader).use { + contracts.map(it::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) } + }.map { it.name } } fun withContractsInJar(jarInputStream: InputStream, withContracts: (List, InputStream) -> T): T { From 854a40d87fed9b0b9a619856e62828c7948124d6 Mon Sep 17 00:00:00 2001 From: Maksymilian Pawlak <120831+m4ksio@users.noreply.github.com> Date: Tue, 27 Feb 2018 11:48:25 +0000 Subject: [PATCH 14/14] [Corda-1116] Classpath as env (#2652) * Move classpath to a system variable so it can overcome command limits on Windows, while keep working fine on other system. --- .../kotlin/net/corda/testing/node/internal/ProcessUtilities.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt index 6354cde0d0..3bf5e29f4b 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/ProcessUtilities.kt @@ -44,13 +44,12 @@ object ProcessUtilities { if (maximumHeapSize != null) add("-Xmx$maximumHeapSize") add("-XX:+UseG1GC") addAll(extraJvmArguments) - add("-cp") - add(classpath) add(className) addAll(arguments) } return ProcessBuilder(command).apply { inheritIO() + environment().put("CLASSPATH", classpath) if (workingDirectory != null) { redirectError((workingDirectory / "$className.stderr.log").toFile()) redirectOutput((workingDirectory / "$className.stdout.log").toFile())