Merge pull request #750 from corda/anthony-os-merge-20180418

O/S Merge

constants.properties

@@ -8,7 +8,7 @@
 # Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited.
 #
 
-gradlePluginsVersion=4.0.12
+gradlePluginsVersion=4.0.13
 kotlinVersion=1.2.20
 platformVersion=4
 guavaVersion=21.0

docs/source/api-vault-query.rst

@@ -259,13 +259,13 @@ Query for all states with pagination specification (10 results per page):
 .. note:: The result set metadata field `totalStatesAvailable` allows you to further paginate accordingly as
    demonstrated in the following example.
 
-Query for all states using pagination specification and iterate using `totalStatesAvailable` field until no further
+Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further
 pages available:
 
-.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt
+.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt
     :language: kotlin
-    :start-after: DOCSTART VaultQueryExamplePaging
+    :start-after: DOCSTART VaultQueryExample24
-    :end-before: DOCEND VaultQueryExamplePaging
+    :end-before: DOCEND VaultQueryExample24
     :dedent: 8
 
 **LinearState and DealState queries using** ``LinearStateQueryCriteria``:
@@ -426,6 +426,14 @@ Query for consumed deal states or linear ids, specify a paging specification and
     :end-before: DOCEND VaultJavaQueryExample2
     :dedent: 12
 
+Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available:
+
+.. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java
+    :language: java
+    :start-after: DOCSTART VaultQueryExample24
+    :end-before: DOCEND VaultQueryExample24
+    :dedent: 8
+
 **Aggregate Function queries using** ``VaultCustomQueryCriteria``:
 
 Aggregations on cash using various functions:
@@ -465,8 +473,8 @@ identifier):
 
 .. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java
     :language: java
-    :start-after: DOCSTART VaultJavaQueryExample4
+    :start-after: DOCSTART VaultJavaQueryExample5
-    :end-before: DOCEND VaultJavaQueryExample4
+    :end-before: DOCEND VaultJavaQueryExample5
     :dedent: 12
 
 Troubleshooting

docs/source/changelog.rst

@@ -9,8 +9,10 @@ Unreleased
 
 * Upgraded H2 to v1.4.197.
 
-* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer
+* Changes to the network bootstrapper:
-  needs the whitelist.txt file.
+  * The whitelist.txt file is no longer needed. The existing network parameters file is used to update the current contracts
+    whitelist.
+  * The CorDapp jars are also copied to each nodes' `cordapps` directory.
 
 * Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data.
 

docs/source/setting-up-a-corda-network.rst

@@ -91,7 +91,7 @@ Whitelisting Contracts
 
 If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`), you can pass in a list of CorDapp jars:
 
-``java -jar network-bootstrapper.jar <nodes-root-dir> <path-to-first-corDapp> <path-to-second-corDapp> ..``
+``java -jar network-bootstrapper.jar <nodes-root-dir> <1st CorDapp jar> <2nd CorDapp jar> ..``
 
 The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part
 of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`).
@@ -111,6 +111,9 @@ For example:
     net.corda.finance.contracts.asset.Cash
     net.corda.finance.contracts.asset.CommercialPaper
 
+In addition to using the CorDapp jars to update the whitelist, the bootstrapper will also copy them to all the nodes'
+``cordapps`` directory.
+
 Starting the nodes
 ~~~~~~~~~~~~~~~~~~
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/ClassloaderUtils.kt

@@ -16,11 +16,11 @@ import net.corda.core.contracts.ContractClassName
 import net.corda.core.internal.copyTo
 import net.corda.core.internal.deleteIfExists
 import net.corda.core.internal.read
-import java.io.File
 import java.io.InputStream
 import java.lang.reflect.Modifier
 import java.net.URLClassLoader
 import java.nio.file.Files
+import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.StandardCopyOption
 
@@ -28,13 +28,16 @@ import java.nio.file.StandardCopyOption
  * Scans the jar for contracts.
  * @returns: found contract class names or null if none found
  */
-fun scanJarForContracts(cordappJarPath: String): List<ContractClassName> {
+fun scanJarForContracts(cordappJar: Path): List<ContractClassName> {
     val currentClassLoader = Contract::class.java.classLoader
-    val scanResult = FastClasspathScanner().addClassLoader(currentClassLoader).overrideClasspath(cordappJarPath, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()).toString()).scan()
+    val scanResult = FastClasspathScanner()
+            .addClassLoader(currentClassLoader)
+            .overrideClasspath(cordappJar, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()))
+            .scan()
     val contracts = (scanResult.getNamesOfClassesImplementing(Contract::class.qualifiedName) ).distinct()
 
     // Only keep instantiable contracts
-    return URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader).use {
+    return URLClassLoader(arrayOf(cordappJar.toUri().toURL()), currentClassLoader).use {
         contracts.map(it::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) }
     }.map { it.name }
 }
@@ -43,7 +46,7 @@ fun <T> withContractsInJar(jarInputStream: InputStream, withContracts: (List<Con
     val tempFile = Files.createTempFile("attachment", ".jar")
     try {
         jarInputStream.copyTo(tempFile, StandardCopyOption.REPLACE_EXISTING)
-        val contracts = scanJarForContracts(tempFile.toAbsolutePath().toString())
+        val contracts = scanJarForContracts(tempFile.toAbsolutePath())
         return tempFile.read { withContracts(contracts, it) }
     } finally {
         tempFile.deleteIfExists()

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt

@@ -28,6 +28,7 @@ import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.seconds
 import net.corda.nodeapi.internal.DEV_ROOT_CA
 import net.corda.nodeapi.internal.SignedNodeInfo
+import net.corda.nodeapi.internal.network.NodeInfoFilesCopier.Companion.NODE_INFO_FILE_NAME_PREFIX
 import net.corda.nodeapi.internal.scanJarForContracts
 import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
 import net.corda.nodeapi.internal.serialization.CordaSerializationMagic
@@ -63,15 +64,15 @@ class NetworkBootstrapper {
         @JvmStatic
         fun main(args: Array<String>) {
             val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" }
-            val cordapps = if (args.size > 1) args.toList().drop(1) else null
+            val cordappJars = if (args.size > 1) args.asList().drop(1).map { Paths.get(it) } else emptyList()
-            NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps)
+            NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordappJars)
         }
     }
 
-    fun bootstrap(directory: Path, cordapps: List<String>?) {
+    fun bootstrap(directory: Path, cordappJars: List<Path>) {
         directory.createDirectories()
         println("Bootstrapping local network in $directory")
-        generateDirectoriesIfNeeded(directory)
+        generateDirectoriesIfNeeded(directory, cordappJars)
         val nodeDirs = directory.list { paths -> paths.filter { (it / "corda.jar").exists() }.toList() }
         require(nodeDirs.isNotEmpty()) { "No nodes found" }
         println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}")
@@ -88,7 +89,7 @@ class NetworkBootstrapper {
             println("Gathering notary identities")
             val notaryInfos = gatherNotaryInfos(nodeInfoFiles)
             println("Generating contract implementations whitelist")
-            val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
+            val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordappJars)
             val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs)
             println("${if (existingNetParams == null) "New" else "Updated"} $netParams")
             println("Bootstrapping complete!")
@@ -98,11 +99,11 @@ class NetworkBootstrapper {
         }
     }
 
-    private fun generateDirectoriesIfNeeded(directory: Path) {
+    private fun generateDirectoriesIfNeeded(directory: Path, cordappJars: List<Path>) {
         val confFiles = directory.list { it.filter { it.toString().endsWith("_node.conf") }.toList() }
         val webServerConfFiles = directory.list { it.filter { it.toString().endsWith("_web-server.conf") }.toList() }
         if (confFiles.isEmpty()) return
-        println("Node config files found in the root directory - generating node directories")
+        println("Node config files found in the root directory - generating node directories and copying CorDapp jars into them")
         val cordaJar = extractCordaJarTo(directory)
         for (confFile in confFiles) {
             val nodeName = confFile.fileName.toString().removeSuffix("_node.conf")
@@ -111,6 +112,8 @@ class NetworkBootstrapper {
             confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING)
             webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING)
             cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING)
+            val cordappsDir = (nodeDir / "cordapps").createDirectories()
+            cordappJars.forEach { it.copyToDirectory(cordappsDir) }
         }
         Files.delete(cordaJar)
     }
@@ -118,7 +121,6 @@ class NetworkBootstrapper {
     private fun extractCordaJarTo(directory: Path): Path {
         val cordaJarPath = directory / "corda.jar"
         if (!cordaJarPath.exists()) {
-            println("No corda jar found in root directory. Extracting from jar")
             Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath)
         }
         return cordaJarPath
@@ -144,7 +146,7 @@ class NetworkBootstrapper {
                 check(process.waitFor() == 0) {
                     "Node in ${nodeDir.fileName} exited with ${process.exitValue()} when generating its node-info - see logs in ${nodeDir / LOGS_DIR_NAME}"
                 }
-                nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith("nodeInfo-") }.findFirst().get() }
+                nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() }
             }
         }
 
@@ -192,7 +194,7 @@ class NetworkBootstrapper {
         }
 
         val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " +
-                "network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n")
+                "network parameters by copying the correct $NETWORK_PARAMS_FILE_NAME file across.\n\n")
 
         netParamsFilesGrouped.forEach { bytes, netParamsFiles ->
             netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ")
@@ -221,6 +223,7 @@ class NetworkBootstrapper {
                     epoch = existingNetParams.epoch + 1
             )
         } else {
+            // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
             NetworkParameters(
                     minimumPlatformVersion = 1,
                     notaries = notaryInfos,
@@ -231,28 +234,25 @@ class NetworkBootstrapper {
                     epoch = 1
             )
         }
-        // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
         val copier = NetworkParametersCopier(networkParameters, overwriteFile = true)
-        nodeDirs.forEach { copier.install(it) }
+        nodeDirs.forEach(copier::install)
         return networkParameters
     }
 
     private fun generateWhitelist(networkParameters: NetworkParameters?,
                                   excludeWhitelistFile: Path,
-                                  cordapps: List<String>?): Map<String, List<AttachmentId>> {
+                                  cordappJars: List<Path>): Map<String, List<AttachmentId>> {
         val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap()
 
-        val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList()
+        val excludeContracts = readExcludeWhitelist(excludeWhitelistFile)
         if (excludeContracts.isNotEmpty()) {
-            println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}}")
+            println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}")
         }
 
-        val newWhiteList = cordapps?.flatMap { cordappJarPath ->
+        val newWhiteList = cordappJars.flatMap { cordappJar ->
-            val jarHash = Paths.get(cordappJarPath).hash
+            val jarHash = cordappJar.hash
-            scanJarForContracts(cordappJarPath).map { contract ->
+            scanJarForContracts(cordappJar).map { contract -> contract to jarHash }
-                contract to jarHash
+        }.filter { (contractClassName, _) -> contractClassName !in excludeContracts }.toMap()
-            }
-        }?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap()
 
         return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
             val existing = existingWhitelist[contractClassName] ?: emptyList()
@@ -261,7 +261,9 @@ class NetworkBootstrapper {
         }.toMap()
     }
 
-    private fun readExcludeWhitelist(file: Path): List<String> = file.readAllLines().map(String::trim)
+    private fun readExcludeWhitelist(file: Path): List<String> {
+        return if (file.exists()) file.readAllLines().map(String::trim) else emptyList()
+    }
 
     private fun NodeInfo.notaryIdentity(): Party {
         return when (legalIdentities.size) {

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkMap.kt

@@ -43,10 +43,14 @@ data class NetworkMap(
         val parametersUpdate: ParametersUpdate?
 ) {
     override fun toString(): String {
-        return """${NetworkMap::class.java.simpleName}(nodeInfoHashes=
+        return """NetworkMap {
-${nodeInfoHashes.joinToString("\n")}
+  nodeInfoHashes {
+    ${nodeInfoHashes.asSequence().take(10).joinToString("\n    ")}
+    ${if (nodeInfoHashes.size > 10) "... ${nodeInfoHashes.size - 10} more" else ""}
+  }
   networkParameterHash=$networkParameterHash
-parametersUpdate=$parametersUpdate)"""
+  parametersUpdate=$parametersUpdate
+}"""
     }
 }
 

node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java

@@ -499,4 +499,46 @@ public class VaultQueryJavaTests {
             return tx;
         });
     }
+
+    private List<StateAndRef<Cash.State>> queryStatesWithPaging(VaultService vaultService, int pageSize) {
+        // DOCSTART VaultQueryExample24
+        int pageNumber = DEFAULT_PAGE_NUM;
+        List<StateAndRef<Cash.State>> states = new ArrayList<>();
+        long totalResults;
+        do {
+            PageSpecification pageSpec = new PageSpecification(pageNumber, pageSize);
+            Vault.Page<Cash.State> results = vaultService.queryBy(Cash.State.class, new VaultQueryCriteria(), pageSpec);
+            totalResults = results.getTotalStatesAvailable();
+            List<StateAndRef<Cash.State>> newStates = results.getStates();
+            System.out.println(newStates.size());
+            states.addAll(results.getStates());
+            pageNumber++;
+        } while ((pageSize * (pageNumber - 1) <= totalResults));
+        // DOCEND VaultQueryExample24
+        return states;
+    }
+
+    @Test
+    public void testExampleOfQueryingStatesWithPagingWorksCorrectly() {
+        Amount<Currency> dollars100 = new Amount<>(100, Currency.getInstance("USD"));
+        database.transaction(tx -> {
+            vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 4, DUMMY_CASH_ISSUER);
+            return tx;
+        });
+        database.transaction(tx -> {
+            assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(4);
+            vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER);
+            return tx;
+        });
+        database.transaction(tx -> {
+            assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(5);
+            vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER);
+            return tx;
+        });
+
+        database.transaction(tx -> {
+            assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(6);
+            return tx;
+        });
+    }
 }

node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt

@@ -21,8 +21,6 @@ import net.corda.core.flows.SchedulableFlow
 import net.corda.core.identity.Party
 import net.corda.core.node.services.VaultService
 import net.corda.core.node.services.queryBy
-import net.corda.core.node.services.vault.DEFAULT_PAGE_NUM
-import net.corda.core.node.services.vault.PageSpecification
 import net.corda.core.node.services.vault.QueryCriteria.VaultQueryCriteria
 import net.corda.core.node.services.vault.Sort
 import net.corda.core.node.services.vault.SortAttribute
@@ -151,7 +149,7 @@ class ScheduledFlowTests {
 
     @Test
     fun `run a whole batch of scheduled flows`() {
-        val N = 100
+        val N = 99
         val futures = mutableListOf<CordaFuture<*>>()
         for (i in 0 until N) {
             futures.add(aliceNode.services.startFlow(InsertInitialStateFlow(bob, notary)).resultFuture)
@@ -164,10 +162,10 @@ class ScheduledFlowTests {
 
         // Convert the states into maps to make error reporting easier
         val statesFromA: List<StateAndRef<ScheduledState>> = aliceNode.database.transaction {
-            queryStatesWithPaging(aliceNode.services.vaultService)
+            queryStates(aliceNode.services.vaultService)
         }
         val statesFromB: List<StateAndRef<ScheduledState>> = bobNode.database.transaction {
-            queryStatesWithPaging(bobNode.services.vaultService)
+            queryStates(bobNode.services.vaultService)
         }
         assertEquals("Expect all states to be present", 2 * N, statesFromA.count())
         statesFromA.forEach { ref ->
@@ -184,23 +182,6 @@ class ScheduledFlowTests {
         assertTrue("Expect all states have run the scheduled task", statesFromB.all { it.state.data.processed })
     }
 
-    /**
+    private fun queryStates(vaultService: VaultService): List<StateAndRef<ScheduledState>> =
-     * Query all states from the Vault, fetching results as a series of pages with ordered states in order to perform
+        vaultService.queryBy<ScheduledState>(VaultQueryCriteria(), sorting = SORTING).states
-     * integration testing of that functionality.
-     *
-     * @return states ordered by the transaction ID.
-     */
-    private fun queryStatesWithPaging(vaultService: VaultService): List<StateAndRef<ScheduledState>> {
-        // DOCSTART VaultQueryExamplePaging
-        var pageNumber = DEFAULT_PAGE_NUM
-        val states = mutableListOf<StateAndRef<ScheduledState>>()
-        do {
-            val pageSpec = PageSpecification(pageSize = PAGE_SIZE, pageNumber = pageNumber)
-            val results = vaultService.queryBy<ScheduledState>(VaultQueryCriteria(), pageSpec, SORTING)
-            states.addAll(results.states)
-            pageNumber++
-        } while ((pageSpec.pageSize * (pageNumber)) <= results.totalStatesAvailable)
-        // DOCEND VaultQueryExamplePaging
-        return states.toList()
-    }
 }

node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt

@@ -1008,14 +1008,39 @@ open class VaultQueryTests {
         }
     }
 
+    // example of querying states with paging using totalStatesAvailable
+    private fun queryStatesWithPaging(vaultService: VaultService, pageSize: Int): List<StateAndRef<ContractState>> {
+        // DOCSTART VaultQueryExample24
+        var pageNumber = DEFAULT_PAGE_NUM
+        val states = mutableListOf<StateAndRef<ContractState>>()
+        do {
+            val pageSpec = PageSpecification(pageNumber = pageNumber, pageSize = pageSize)
+            val results = vaultService.queryBy<ContractState>(VaultQueryCriteria(), pageSpec)
+            states.addAll(results.states)
+            pageNumber++
+        } while ((pageSpec.pageSize * (pageNumber - 1)) <= results.totalStatesAvailable)
+        // DOCEND VaultQueryExample24
+        return states.toList()
+    }
+
+    // test paging query example works
+    @Test
+    fun `test example of querying states with paging works correctly`() {
+        database.transaction {
+            vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 4, DUMMY_CASH_ISSUER)
+            assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(4)
+            vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER)
+            assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(5)
+            vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER)
+            assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(6)
+        }
+    }
+
     // sorting
     @Test
     fun `sorting - all states sorted by contract type, state status, consumed time`() {
-
         setUpDb(database)
-
         database.transaction {
-
             val sortCol1 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONTRACT_STATE_TYPE), Sort.Direction.DESC)
             val sortCol2 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.STATE_STATUS), Sort.Direction.ASC)
             val sortCol3 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONSUMED_TIME), Sort.Direction.DESC)