diff --git a/constants.properties b/constants.properties index bffdc995cc..6f82ef914b 100644 --- a/constants.properties +++ b/constants.properties @@ -8,7 +8,7 @@ # Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited. # -gradlePluginsVersion=4.0.12 +gradlePluginsVersion=4.0.13 kotlinVersion=1.2.20 platformVersion=4 guavaVersion=21.0 diff --git a/docs/source/api-vault-query.rst b/docs/source/api-vault-query.rst index 1adb4631b9..a44ab01aa9 100644 --- a/docs/source/api-vault-query.rst +++ b/docs/source/api-vault-query.rst @@ -259,13 +259,13 @@ Query for all states with pagination specification (10 results per page): .. note:: The result set metadata field `totalStatesAvailable` allows you to further paginate accordingly as demonstrated in the following example. -Query for all states using pagination specification and iterate using `totalStatesAvailable` field until no further +Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available: -.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt +.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt :language: kotlin - :start-after: DOCSTART VaultQueryExamplePaging - :end-before: DOCEND VaultQueryExamplePaging + :start-after: DOCSTART VaultQueryExample24 + :end-before: DOCEND VaultQueryExample24 :dedent: 8 **LinearState and DealState queries using** ``LinearStateQueryCriteria``: @@ -426,6 +426,14 @@ Query for consumed deal states or linear ids, specify a paging specification and :end-before: DOCEND VaultJavaQueryExample2 :dedent: 12 +Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available: + +.. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java + :language: java + :start-after: DOCSTART VaultQueryExample24 + :end-before: DOCEND VaultQueryExample24 + :dedent: 8 + **Aggregate Function queries using** ``VaultCustomQueryCriteria``: Aggregations on cash using various functions: @@ -465,8 +473,8 @@ identifier): .. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java :language: java - :start-after: DOCSTART VaultJavaQueryExample4 - :end-before: DOCEND VaultJavaQueryExample4 + :start-after: DOCSTART VaultJavaQueryExample5 + :end-before: DOCEND VaultJavaQueryExample5 :dedent: 12 Troubleshooting diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 17ec2f8f74..748c960dce 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -9,8 +9,10 @@ Unreleased * Upgraded H2 to v1.4.197. -* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer - needs the whitelist.txt file. +* Changes to the network bootstrapper: + * The whitelist.txt file is no longer needed. The existing network parameters file is used to update the current contracts + whitelist. + * The CorDapp jars are also copied to each nodes' `cordapps` directory. * Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data. diff --git a/docs/source/setting-up-a-corda-network.rst b/docs/source/setting-up-a-corda-network.rst index d82c0fa9c7..7a227d45e3 100644 --- a/docs/source/setting-up-a-corda-network.rst +++ b/docs/source/setting-up-a-corda-network.rst @@ -91,7 +91,7 @@ Whitelisting Contracts If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`), you can pass in a list of CorDapp jars: -``java -jar network-bootstrapper.jar ..`` +``java -jar network-bootstrapper.jar <1st CorDapp jar> <2nd CorDapp jar> ..`` The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`). @@ -111,6 +111,9 @@ For example: net.corda.finance.contracts.asset.Cash net.corda.finance.contracts.asset.CommercialPaper +In addition to using the CorDapp jars to update the whitelist, the bootstrapper will also copy them to all the nodes' +``cordapps`` directory. + Starting the nodes ~~~~~~~~~~~~~~~~~~ diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt index ea8d40d794..484bd1ce27 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt @@ -16,11 +16,11 @@ import net.corda.core.contracts.ContractClassName import net.corda.core.internal.copyTo import net.corda.core.internal.deleteIfExists import net.corda.core.internal.read -import java.io.File import java.io.InputStream import java.lang.reflect.Modifier import java.net.URLClassLoader import java.nio.file.Files +import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption @@ -28,13 +28,16 @@ import java.nio.file.StandardCopyOption * Scans the jar for contracts. * @returns: found contract class names or null if none found */ -fun scanJarForContracts(cordappJarPath: String): List { +fun scanJarForContracts(cordappJar: Path): List { val currentClassLoader = Contract::class.java.classLoader - val scanResult = FastClasspathScanner().addClassLoader(currentClassLoader).overrideClasspath(cordappJarPath, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()).toString()).scan() + val scanResult = FastClasspathScanner() + .addClassLoader(currentClassLoader) + .overrideClasspath(cordappJar, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI())) + .scan() val contracts = (scanResult.getNamesOfClassesImplementing(Contract::class.qualifiedName) ).distinct() // Only keep instantiable contracts - return URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader).use { + return URLClassLoader(arrayOf(cordappJar.toUri().toURL()), currentClassLoader).use { contracts.map(it::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) } }.map { it.name } } @@ -43,7 +46,7 @@ fun withContractsInJar(jarInputStream: InputStream, withContracts: (List) { val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" } - val cordapps = if (args.size > 1) args.toList().drop(1) else null - NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps) + val cordappJars = if (args.size > 1) args.asList().drop(1).map { Paths.get(it) } else emptyList() + NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordappJars) } } - fun bootstrap(directory: Path, cordapps: List?) { + fun bootstrap(directory: Path, cordappJars: List) { directory.createDirectories() println("Bootstrapping local network in $directory") - generateDirectoriesIfNeeded(directory) + generateDirectoriesIfNeeded(directory, cordappJars) val nodeDirs = directory.list { paths -> paths.filter { (it / "corda.jar").exists() }.toList() } require(nodeDirs.isNotEmpty()) { "No nodes found" } println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}") @@ -88,7 +89,7 @@ class NetworkBootstrapper { println("Gathering notary identities") val notaryInfos = gatherNotaryInfos(nodeInfoFiles) println("Generating contract implementations whitelist") - val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct()) + val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordappJars) val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs) println("${if (existingNetParams == null) "New" else "Updated"} $netParams") println("Bootstrapping complete!") @@ -98,11 +99,11 @@ class NetworkBootstrapper { } } - private fun generateDirectoriesIfNeeded(directory: Path) { + private fun generateDirectoriesIfNeeded(directory: Path, cordappJars: List) { val confFiles = directory.list { it.filter { it.toString().endsWith("_node.conf") }.toList() } val webServerConfFiles = directory.list { it.filter { it.toString().endsWith("_web-server.conf") }.toList() } if (confFiles.isEmpty()) return - println("Node config files found in the root directory - generating node directories") + println("Node config files found in the root directory - generating node directories and copying CorDapp jars into them") val cordaJar = extractCordaJarTo(directory) for (confFile in confFiles) { val nodeName = confFile.fileName.toString().removeSuffix("_node.conf") @@ -111,6 +112,8 @@ class NetworkBootstrapper { confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING) webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING) cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING) + val cordappsDir = (nodeDir / "cordapps").createDirectories() + cordappJars.forEach { it.copyToDirectory(cordappsDir) } } Files.delete(cordaJar) } @@ -118,7 +121,6 @@ class NetworkBootstrapper { private fun extractCordaJarTo(directory: Path): Path { val cordaJarPath = directory / "corda.jar" if (!cordaJarPath.exists()) { - println("No corda jar found in root directory. Extracting from jar") Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath) } return cordaJarPath @@ -144,7 +146,7 @@ class NetworkBootstrapper { check(process.waitFor() == 0) { "Node in ${nodeDir.fileName} exited with ${process.exitValue()} when generating its node-info - see logs in ${nodeDir / LOGS_DIR_NAME}" } - nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith("nodeInfo-") }.findFirst().get() } + nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() } } } @@ -192,7 +194,7 @@ class NetworkBootstrapper { } val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " + - "network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n") + "network parameters by copying the correct $NETWORK_PARAMS_FILE_NAME file across.\n\n") netParamsFilesGrouped.forEach { bytes, netParamsFiles -> netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ") @@ -221,6 +223,7 @@ class NetworkBootstrapper { epoch = existingNetParams.epoch + 1 ) } else { + // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize NetworkParameters( minimumPlatformVersion = 1, notaries = notaryInfos, @@ -231,28 +234,25 @@ class NetworkBootstrapper { epoch = 1 ) } - // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize val copier = NetworkParametersCopier(networkParameters, overwriteFile = true) - nodeDirs.forEach { copier.install(it) } + nodeDirs.forEach(copier::install) return networkParameters } private fun generateWhitelist(networkParameters: NetworkParameters?, excludeWhitelistFile: Path, - cordapps: List?): Map> { + cordappJars: List): Map> { val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap() - val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList() + val excludeContracts = readExcludeWhitelist(excludeWhitelistFile) if (excludeContracts.isNotEmpty()) { - println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}}") + println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}") } - val newWhiteList = cordapps?.flatMap { cordappJarPath -> - val jarHash = Paths.get(cordappJarPath).hash - scanJarForContracts(cordappJarPath).map { contract -> - contract to jarHash - } - }?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap() + val newWhiteList = cordappJars.flatMap { cordappJar -> + val jarHash = cordappJar.hash + scanJarForContracts(cordappJar).map { contract -> contract to jarHash } + }.filter { (contractClassName, _) -> contractClassName !in excludeContracts }.toMap() return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName -> val existing = existingWhitelist[contractClassName] ?: emptyList() @@ -261,7 +261,9 @@ class NetworkBootstrapper { }.toMap() } - private fun readExcludeWhitelist(file: Path): List = file.readAllLines().map(String::trim) + private fun readExcludeWhitelist(file: Path): List { + return if (file.exists()) file.readAllLines().map(String::trim) else emptyList() + } private fun NodeInfo.notaryIdentity(): Party { return when (legalIdentities.size) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt index fe2cc1ad34..bc49c77ad0 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt @@ -43,10 +43,14 @@ data class NetworkMap( val parametersUpdate: ParametersUpdate? ) { override fun toString(): String { - return """${NetworkMap::class.java.simpleName}(nodeInfoHashes= -${nodeInfoHashes.joinToString("\n")} -networkParameterHash=$networkParameterHash -parametersUpdate=$parametersUpdate)""" + return """NetworkMap { + nodeInfoHashes { + ${nodeInfoHashes.asSequence().take(10).joinToString("\n ")} + ${if (nodeInfoHashes.size > 10) "... ${nodeInfoHashes.size - 10} more" else ""} + } + networkParameterHash=$networkParameterHash + parametersUpdate=$parametersUpdate +}""" } } diff --git a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java index 172ed6baf4..569bfe45c4 100644 --- a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java +++ b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java @@ -499,4 +499,46 @@ public class VaultQueryJavaTests { return tx; }); } + + private List> queryStatesWithPaging(VaultService vaultService, int pageSize) { + // DOCSTART VaultQueryExample24 + int pageNumber = DEFAULT_PAGE_NUM; + List> states = new ArrayList<>(); + long totalResults; + do { + PageSpecification pageSpec = new PageSpecification(pageNumber, pageSize); + Vault.Page results = vaultService.queryBy(Cash.State.class, new VaultQueryCriteria(), pageSpec); + totalResults = results.getTotalStatesAvailable(); + List> newStates = results.getStates(); + System.out.println(newStates.size()); + states.addAll(results.getStates()); + pageNumber++; + } while ((pageSize * (pageNumber - 1) <= totalResults)); + // DOCEND VaultQueryExample24 + return states; + } + + @Test + public void testExampleOfQueryingStatesWithPagingWorksCorrectly() { + Amount dollars100 = new Amount<>(100, Currency.getInstance("USD")); + database.transaction(tx -> { + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 4, DUMMY_CASH_ISSUER); + return tx; + }); + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(4); + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER); + return tx; + }); + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(5); + vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER); + return tx; + }); + + database.transaction(tx -> { + assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(6); + return tx; + }); + } } diff --git a/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt b/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt index 6d53695474..3a58b5f348 100644 --- a/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt @@ -21,8 +21,6 @@ import net.corda.core.flows.SchedulableFlow import net.corda.core.identity.Party import net.corda.core.node.services.VaultService import net.corda.core.node.services.queryBy -import net.corda.core.node.services.vault.DEFAULT_PAGE_NUM -import net.corda.core.node.services.vault.PageSpecification import net.corda.core.node.services.vault.QueryCriteria.VaultQueryCriteria import net.corda.core.node.services.vault.Sort import net.corda.core.node.services.vault.SortAttribute @@ -151,7 +149,7 @@ class ScheduledFlowTests { @Test fun `run a whole batch of scheduled flows`() { - val N = 100 + val N = 99 val futures = mutableListOf>() for (i in 0 until N) { futures.add(aliceNode.services.startFlow(InsertInitialStateFlow(bob, notary)).resultFuture) @@ -164,10 +162,10 @@ class ScheduledFlowTests { // Convert the states into maps to make error reporting easier val statesFromA: List> = aliceNode.database.transaction { - queryStatesWithPaging(aliceNode.services.vaultService) + queryStates(aliceNode.services.vaultService) } val statesFromB: List> = bobNode.database.transaction { - queryStatesWithPaging(bobNode.services.vaultService) + queryStates(bobNode.services.vaultService) } assertEquals("Expect all states to be present", 2 * N, statesFromA.count()) statesFromA.forEach { ref -> @@ -184,23 +182,6 @@ class ScheduledFlowTests { assertTrue("Expect all states have run the scheduled task", statesFromB.all { it.state.data.processed }) } - /** - * Query all states from the Vault, fetching results as a series of pages with ordered states in order to perform - * integration testing of that functionality. - * - * @return states ordered by the transaction ID. - */ - private fun queryStatesWithPaging(vaultService: VaultService): List> { - // DOCSTART VaultQueryExamplePaging - var pageNumber = DEFAULT_PAGE_NUM - val states = mutableListOf>() - do { - val pageSpec = PageSpecification(pageSize = PAGE_SIZE, pageNumber = pageNumber) - val results = vaultService.queryBy(VaultQueryCriteria(), pageSpec, SORTING) - states.addAll(results.states) - pageNumber++ - } while ((pageSpec.pageSize * (pageNumber)) <= results.totalStatesAvailable) - // DOCEND VaultQueryExamplePaging - return states.toList() - } + private fun queryStates(vaultService: VaultService): List> = + vaultService.queryBy(VaultQueryCriteria(), sorting = SORTING).states } diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index 3f4a15c701..8704a7c871 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -1008,14 +1008,39 @@ open class VaultQueryTests { } } + // example of querying states with paging using totalStatesAvailable + private fun queryStatesWithPaging(vaultService: VaultService, pageSize: Int): List> { + // DOCSTART VaultQueryExample24 + var pageNumber = DEFAULT_PAGE_NUM + val states = mutableListOf>() + do { + val pageSpec = PageSpecification(pageNumber = pageNumber, pageSize = pageSize) + val results = vaultService.queryBy(VaultQueryCriteria(), pageSpec) + states.addAll(results.states) + pageNumber++ + } while ((pageSpec.pageSize * (pageNumber - 1)) <= results.totalStatesAvailable) + // DOCEND VaultQueryExample24 + return states.toList() + } + + // test paging query example works + @Test + fun `test example of querying states with paging works correctly`() { + database.transaction { + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 4, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(4) + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(5) + vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER) + assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(6) + } + } + // sorting @Test fun `sorting - all states sorted by contract type, state status, consumed time`() { - setUpDb(database) - database.transaction { - val sortCol1 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONTRACT_STATE_TYPE), Sort.Direction.DESC) val sortCol2 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.STATE_STATUS), Sort.Direction.ASC) val sortCol3 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONSUMED_TIME), Sort.Direction.DESC)