From 60323cca1517b94ba751c628cb829e2142b926b9 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Tue, 17 Apr 2018 15:38:25 +0100 Subject: [PATCH 1/3] CORDA-1312: Network bootstrapper copies any CorDapp jars into each nodes' cordapps dir (#2974) --- constants.properties | 2 +- docs/source/changelog.rst | 6 ++- docs/source/setting-up-a-corda-network.rst | 5 +- .../nodeapi/internal/ClassloaderUtils.kt | 13 ++++-- .../internal/network/NetworkBootstrapper.kt | 46 ++++++++++--------- .../nodeapi/internal/network/NetworkMap.kt | 12 +++-- 6 files changed, 49 insertions(+), 35 deletions(-) diff --git a/constants.properties b/constants.properties index 1c67578704..7be4b2da09 100644 --- a/constants.properties +++ b/constants.properties @@ -1,4 +1,4 @@ -gradlePluginsVersion=4.0.12 +gradlePluginsVersion=4.0.13 kotlinVersion=1.2.20 platformVersion=4 guavaVersion=21.0 diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 679e4b0c0a..f20cd009da 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -19,8 +19,10 @@ Unreleased * Shell (embedded available only in dev mode or via SSH) connects to the node via RPC instead of using the ``CordaRPCOps`` object directly. To enable RPC connectivity ensure node’s ``rpcSettings.address`` and ``rpcSettings.adminAddress`` settings are present. -* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer - needs the whitelist.txt file. +* Changes to the network bootstrapper: + * The whitelist.txt file is no longer needed. The existing network parameters file is used to update the current contracts + whitelist. + * The CorDapp jars are also copied to each nodes' `cordapps` directory. * Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data. diff --git a/docs/source/setting-up-a-corda-network.rst b/docs/source/setting-up-a-corda-network.rst index d82c0fa9c7..7a227d45e3 100644 --- a/docs/source/setting-up-a-corda-network.rst +++ b/docs/source/setting-up-a-corda-network.rst @@ -91,7 +91,7 @@ Whitelisting Contracts If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`), you can pass in a list of CorDapp jars: -``java -jar network-bootstrapper.jar ..`` +``java -jar network-bootstrapper.jar <1st CorDapp jar> <2nd CorDapp jar> ..`` The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`). @@ -111,6 +111,9 @@ For example: net.corda.finance.contracts.asset.Cash net.corda.finance.contracts.asset.CommercialPaper +In addition to using the CorDapp jars to update the whitelist, the bootstrapper will also copy them to all the nodes' +``cordapps`` directory. + Starting the nodes ~~~~~~~~~~~~~~~~~~ diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt index 124cda119d..b7972dc2ec 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt @@ -6,11 +6,11 @@ import net.corda.core.contracts.ContractClassName import net.corda.core.internal.copyTo import net.corda.core.internal.deleteIfExists import net.corda.core.internal.read -import java.io.File import java.io.InputStream import java.lang.reflect.Modifier import java.net.URLClassLoader import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption @@ -18,13 +18,16 @@ import java.nio.file.StandardCopyOption * Scans the jar for contracts. * @returns: found contract class names or null if none found */ -fun scanJarForContracts(cordappJarPath: String): List { +fun scanJarForContracts(cordappJar: Path): List { val currentClassLoader = Contract::class.java.classLoader - val scanResult = FastClasspathScanner().addClassLoader(currentClassLoader).overrideClasspath(cordappJarPath, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()).toString()).scan() + val scanResult = FastClasspathScanner() + .addClassLoader(currentClassLoader) + .overrideClasspath(cordappJar, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI())) + .scan() val contracts = (scanResult.getNamesOfClassesImplementing(Contract::class.qualifiedName) ).distinct() // Only keep instantiable contracts - return URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader).use { + return URLClassLoader(arrayOf(cordappJar.toUri().toURL()), currentClassLoader).use { contracts.map(it::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) } }.map { it.name } } @@ -33,7 +36,7 @@ fun withContractsInJar(jarInputStream: InputStream, withContracts: (List) { val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" } - val cordapps = if (args.size > 1) args.toList().drop(1) else null - NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps) + val cordappJars = if (args.size > 1) args.asList().drop(1).map { Paths.get(it) } else emptyList() + NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordappJars) } } - fun bootstrap(directory: Path, cordapps: List?) { + fun bootstrap(directory: Path, cordappJars: List) { directory.createDirectories() println("Bootstrapping local network in $directory") - generateDirectoriesIfNeeded(directory) + generateDirectoriesIfNeeded(directory, cordappJars) val nodeDirs = directory.list { paths -> paths.filter { (it / "corda.jar").exists() }.toList() } require(nodeDirs.isNotEmpty()) { "No nodes found" } println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}") @@ -78,7 +79,7 @@ class NetworkBootstrapper { println("Gathering notary identities") val notaryInfos = gatherNotaryInfos(nodeInfoFiles) println("Generating contract implementations whitelist") - val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct()) + val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordappJars) val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs) println("${if (existingNetParams == null) "New" else "Updated"} $netParams") println("Bootstrapping complete!") @@ -88,11 +89,11 @@ class NetworkBootstrapper { } } - private fun generateDirectoriesIfNeeded(directory: Path) { + private fun generateDirectoriesIfNeeded(directory: Path, cordappJars: List) { val confFiles = directory.list { it.filter { it.toString().endsWith("_node.conf") }.toList() } val webServerConfFiles = directory.list { it.filter { it.toString().endsWith("_web-server.conf") }.toList() } if (confFiles.isEmpty()) return - println("Node config files found in the root directory - generating node directories") + println("Node config files found in the root directory - generating node directories and copying CorDapp jars into them") val cordaJar = extractCordaJarTo(directory) for (confFile in confFiles) { val nodeName = confFile.fileName.toString().removeSuffix("_node.conf") @@ -101,6 +102,8 @@ class NetworkBootstrapper { confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING) webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING) cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING) + val cordappsDir = (nodeDir / "cordapps").createDirectories() + cordappJars.forEach { it.copyToDirectory(cordappsDir) } } Files.delete(cordaJar) } @@ -108,7 +111,6 @@ class NetworkBootstrapper { private fun extractCordaJarTo(directory: Path): Path { val cordaJarPath = directory / "corda.jar" if (!cordaJarPath.exists()) { - println("No corda jar found in root directory. Extracting from jar") Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath) } return cordaJarPath @@ -134,7 +136,7 @@ class NetworkBootstrapper { check(process.waitFor() == 0) { "Node in ${nodeDir.fileName} exited with ${process.exitValue()} when generating its node-info - see logs in ${nodeDir / LOGS_DIR_NAME}" } - nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith("nodeInfo-") }.findFirst().get() } + nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() } } } @@ -182,7 +184,7 @@ class NetworkBootstrapper { } val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " + - "network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n") + "network parameters by copying the correct $NETWORK_PARAMS_FILE_NAME file across.\n\n") netParamsFilesGrouped.forEach { bytes, netParamsFiles -> netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ") @@ -211,6 +213,7 @@ class NetworkBootstrapper { epoch = existingNetParams.epoch + 1 ) } else { + // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize NetworkParameters( minimumPlatformVersion = 1, notaries = notaryInfos, @@ -221,28 +224,25 @@ class NetworkBootstrapper { epoch = 1 ) } - // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize val copier = NetworkParametersCopier(networkParameters, overwriteFile = true) - nodeDirs.forEach { copier.install(it) } + nodeDirs.forEach(copier::install) return networkParameters } private fun generateWhitelist(networkParameters: NetworkParameters?, excludeWhitelistFile: Path, - cordapps: List?): Map> { + cordappJars: List): Map> { val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap() - val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList() + val excludeContracts = readExcludeWhitelist(excludeWhitelistFile) if (excludeContracts.isNotEmpty()) { - println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}}") + println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}") } - val newWhiteList = cordapps?.flatMap { cordappJarPath -> - val jarHash = Paths.get(cordappJarPath).hash - scanJarForContracts(cordappJarPath).map { contract -> - contract to jarHash - } - }?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap() + val newWhiteList = cordappJars.flatMap { cordappJar -> + val jarHash = cordappJar.hash + scanJarForContracts(cordappJar).map { contract -> contract to jarHash } + }.filter { (contractClassName, _) -> contractClassName !in excludeContracts }.toMap() return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName -> val existing = existingWhitelist[contractClassName] ?: emptyList() @@ -251,7 +251,9 @@ class NetworkBootstrapper { }.toMap() } - private fun readExcludeWhitelist(file: Path): List = file.readAllLines().map(String::trim) + private fun readExcludeWhitelist(file: Path): List { + return if (file.exists()) file.readAllLines().map(String::trim) else emptyList() + } private fun NodeInfo.notaryIdentity(): Party { return when (legalIdentities.size) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt index e86d1e7da1..6ee2cbdd01 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt @@ -33,10 +33,14 @@ data class NetworkMap( val parametersUpdate: ParametersUpdate? ) { override fun toString(): String { - return """${NetworkMap::class.java.simpleName}(nodeInfoHashes= -${nodeInfoHashes.joinToString("\n")} -networkParameterHash=$networkParameterHash -parametersUpdate=$parametersUpdate)""" + return """NetworkMap { + nodeInfoHashes { + ${nodeInfoHashes.asSequence().take(10).joinToString("\n ")} + ${if (nodeInfoHashes.size > 10) "... ${nodeInfoHashes.size - 10} more" else ""} + } + networkParameterHash=$networkParameterHash + parametersUpdate=$parametersUpdate +}""" } } From 7db48de2b88ba6cb37b57f05b12f997ec2d83af7 Mon Sep 17 00:00:00 2001 From: Anthony Keenan Date: Wed, 18 Apr 2018 11:10:21 +0100 Subject: [PATCH 2/3] CORDA-1344 Fix query paging in scheduled flow tests (#2970) * Fix paging tests, move out of scheduled flow tests and provide java example * Fix a few issues with docs --- docs/source/api-vault-query.rst | 20 ++++++--- .../services/vault/VaultQueryJavaTests.java | 42 +++++++++++++++++++ .../services/events/ScheduledFlowTests.kt | 29 +++---------- .../node/services/vault/VaultQueryTests.kt | 31 ++++++++++++-- 4 files changed, 89 insertions(+), 33 deletions(-) diff --git a/docs/source/api-vault-query.rst b/docs/source/api-vault-query.rst index 1adb4631b9..a44ab01aa9 100644 --- a/docs/source/api-vault-query.rst +++ b/docs/source/api-vault-query.rst @@ -259,13 +259,13 @@ Query for all states with pagination specification (10 results per page): .. note:: The result set metadata field `totalStatesAvailable` allows you to further paginate accordingly as demonstrated in the following example. -Query for all states using pagination specification and iterate using `totalStatesAvailable` field until no further +Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available: -.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt +.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt :language: kotlin - :start-after: DOCSTART VaultQueryExamplePaging - :end-before: DOCEND VaultQueryExamplePaging + :start-after: DOCSTART VaultQueryExample24 + :end-before: DOCEND VaultQueryExample24 :dedent: 8 **LinearState and DealState queries using** ``LinearStateQueryCriteria``: @@ -426,6 +426,14 @@ Query for consumed deal states or linear ids, specify a paging specification and :end-before: DOCEND VaultJavaQueryExample2 :dedent: 12 +Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available: + +.. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java + :language: java + :start-after: DOCSTART VaultQueryExample24 + :end-before: DOCEND VaultQueryExample24 + :dedent: 8 + **Aggregate Function queries using** ``VaultCustomQueryCriteria``: Aggregations on cash using various functions: @@ -465,8 +473,8 @@ identifier): .. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java :language: java - :start-after: DOCSTART VaultJavaQueryExample4 - :end-before: DOCEND VaultJavaQueryExample4 + :start-after: DOCSTART VaultJavaQueryExample5 + :end-before: DOCEND VaultJavaQueryExample5 :dedent: 12 Troubleshooting diff --git a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java index 7cd4ce468e..ca4b25c6a8 100644 --- a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java +++ b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java @@ -485,4 +485,46 @@ public class VaultQueryJavaTests { return tx; }); } + + private List> queryStatesWithPaging(VaultService vaultService, int pageSize) { + // DOCSTART VaultQueryExample24 + int pageNumber = DEFAULT_PAGE_NUM; + List> states = new ArrayList<>(); + long totalResults; + do { + PageSpecification pageSpec = new PageSpecification(pageNumber, pageSize); + Vault.Page results = vaultService.queryBy(Cash.State.class, new VaultQueryCriteria(), pageSpec); + totalResults = results.getTotalStatesAvailable(); + List> newStates = results.getStates(); + System.out.println(newStates.size()); + states.addAll(results.getStates()); + pageNumber++; + } while ((pageSize * (pageNumber - 1) <= totalResults)); + // DOCEND VaultQueryExample24 + return states; + } + + @Test + public void testExampleOfQueryingStatesWithPagingWorksCorrectly() { + Amount dollars100 = new Amount<>(100, Currency.getInstance("USD")); + database.transaction(tx -> { + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 4, DUMMY_CASH_ISSUER); + return tx; + }); + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(4); + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER); + return tx; + }); + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(5); + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER); + return tx; + }); + + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(6); + return tx; + }); + } } diff --git a/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt b/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt index 5aea06bcfb..a878fb5a18 100644 --- a/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt @@ -11,8 +11,6 @@ import net.corda.core.flows.SchedulableFlow import net.corda.core.identity.Party import net.corda.core.node.services.VaultService import net.corda.core.node.services.queryBy -import net.corda.core.node.services.vault.DEFAULT_PAGE_NUM -import net.corda.core.node.services.vault.PageSpecification import net.corda.core.node.services.vault.QueryCriteria.VaultQueryCriteria import net.corda.core.node.services.vault.Sort import net.corda.core.node.services.vault.SortAttribute @@ -141,7 +139,7 @@ class ScheduledFlowTests { @Test fun `run a whole batch of scheduled flows`() { - val N = 100 + val N = 99 val futures = mutableListOf>() for (i in 0 until N) { futures.add(aliceNode.services.startFlow(InsertInitialStateFlow(bob, notary)).resultFuture) @@ -154,10 +152,10 @@ class ScheduledFlowTests { // Convert the states into maps to make error reporting easier val statesFromA: List> = aliceNode.database.transaction { - queryStatesWithPaging(aliceNode.services.vaultService) + queryStates(aliceNode.services.vaultService) } val statesFromB: List> = bobNode.database.transaction { - queryStatesWithPaging(bobNode.services.vaultService) + queryStates(bobNode.services.vaultService) } assertEquals("Expect all states to be present", 2 * N, statesFromA.count()) statesFromA.forEach { ref -> @@ -174,23 +172,6 @@ class ScheduledFlowTests { assertTrue("Expect all states have run the scheduled task", statesFromB.all { it.state.data.processed }) } - /** - * Query all states from the Vault, fetching results as a series of pages with ordered states in order to perform - * integration testing of that functionality. - * - * @return states ordered by the transaction ID. - */ - private fun queryStatesWithPaging(vaultService: VaultService): List> { - // DOCSTART VaultQueryExamplePaging - var pageNumber = DEFAULT_PAGE_NUM - val states = mutableListOf>() - do { - val pageSpec = PageSpecification(pageSize = PAGE_SIZE, pageNumber = pageNumber) - val results = vaultService.queryBy(VaultQueryCriteria(), pageSpec, SORTING) - states.addAll(results.states) - pageNumber++ - } while ((pageSpec.pageSize * (pageNumber)) <= results.totalStatesAvailable) - // DOCEND VaultQueryExamplePaging - return states.toList() - } + private fun queryStates(vaultService: VaultService): List> = + vaultService.queryBy(VaultQueryCriteria(), sorting = SORTING).states } diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index 92e33ab2e9..526b2ddda9 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -1001,14 +1001,39 @@ class VaultQueryTests { } } + // example of querying states with paging using totalStatesAvailable + private fun queryStatesWithPaging(vaultService: VaultService, pageSize: Int): List> { + // DOCSTART VaultQueryExample24 + var pageNumber = DEFAULT_PAGE_NUM + val states = mutableListOf>() + do { + val pageSpec = PageSpecification(pageNumber = pageNumber, pageSize = pageSize) + val results = vaultService.queryBy(VaultQueryCriteria(), pageSpec) + states.addAll(results.states) + pageNumber++ + } while ((pageSpec.pageSize * (pageNumber - 1)) <= results.totalStatesAvailable) + // DOCEND VaultQueryExample24 + return states.toList() + } + + // test paging query example works + @Test + fun `test example of querying states with paging works correctly`() { + database.transaction { + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 4, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(4) + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(5) + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(6) + } + } + // sorting @Test fun `sorting - all states sorted by contract type, state status, consumed time`() { - setUpDb(database) - database.transaction { - val sortCol1 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONTRACT_STATE_TYPE), Sort.Direction.DESC) val sortCol2 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.STATE_STATUS), Sort.Direction.ASC) val sortCol3 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONSUMED_TIME), Sort.Direction.DESC) From 39bb3912a555b657a5da048dccf13f6d7fb77e43 Mon Sep 17 00:00:00 2001 From: Anthony Keenan Date: Wed, 18 Apr 2018 11:37:38 +0100 Subject: [PATCH 3/3] Fix merge issue --- constants.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/constants.properties b/constants.properties index 5f1ded836f..6f82ef914b 100644 --- a/constants.properties +++ b/constants.properties @@ -1,4 +1,3 @@ -<<<<<<< HEAD # # R3 Proprietary and Confidential #