Merge pull request #750 from corda/anthony-os-merge-20180418

O/S Merge
This commit is contained in:
Anthony Keenan 2018-04-18 13:02:56 +01:00 committed by GitHub
commit 5f0d157432
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 138 additions and 68 deletions

View File

@ -8,7 +8,7 @@
# Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited.
#
gradlePluginsVersion=4.0.12
gradlePluginsVersion=4.0.13
kotlinVersion=1.2.20
platformVersion=4
guavaVersion=21.0

View File

@ -259,13 +259,13 @@ Query for all states with pagination specification (10 results per page):
.. note:: The result set metadata field `totalStatesAvailable` allows you to further paginate accordingly as
demonstrated in the following example.
Query for all states using pagination specification and iterate using `totalStatesAvailable` field until no further
Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further
pages available:
.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/events/ScheduledFlowTests.kt
.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt
:language: kotlin
:start-after: DOCSTART VaultQueryExamplePaging
:end-before: DOCEND VaultQueryExamplePaging
:start-after: DOCSTART VaultQueryExample24
:end-before: DOCEND VaultQueryExample24
:dedent: 8
**LinearState and DealState queries using** ``LinearStateQueryCriteria``:
@ -426,6 +426,14 @@ Query for consumed deal states or linear ids, specify a paging specification and
:end-before: DOCEND VaultJavaQueryExample2
:dedent: 12
Query for all states using a pagination specification and iterate using the `totalStatesAvailable` field until no further pages available:
.. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java
:language: java
:start-after: DOCSTART VaultQueryExample24
:end-before: DOCEND VaultQueryExample24
:dedent: 8
**Aggregate Function queries using** ``VaultCustomQueryCriteria``:
Aggregations on cash using various functions:
@ -465,8 +473,8 @@ identifier):
.. literalinclude:: ../../node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java
:language: java
:start-after: DOCSTART VaultJavaQueryExample4
:end-before: DOCEND VaultJavaQueryExample4
:start-after: DOCSTART VaultJavaQueryExample5
:end-before: DOCEND VaultJavaQueryExample5
:dedent: 12
Troubleshooting

View File

@ -9,8 +9,10 @@ Unreleased
* Upgraded H2 to v1.4.197.
* The network bootstrapper uses the existing network parameters file to update the current contracts whitelist, and no longer
needs the whitelist.txt file.
* Changes to the network bootstrapper:
* The whitelist.txt file is no longer needed. The existing network parameters file is used to update the current contracts
whitelist.
* The CorDapp jars are also copied to each nodes' `cordapps` directory.
* Errors thrown by a Corda node will now reported to a calling RPC client with attention to serialization and obfuscation of internal data.

View File

@ -91,7 +91,7 @@ Whitelisting Contracts
If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`), you can pass in a list of CorDapp jars:
``java -jar network-bootstrapper.jar <nodes-root-dir> <path-to-first-corDapp> <path-to-second-corDapp> ..``
``java -jar network-bootstrapper.jar <nodes-root-dir> <1st CorDapp jar> <2nd CorDapp jar> ..``
The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part
of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`).
@ -111,6 +111,9 @@ For example:
net.corda.finance.contracts.asset.Cash
net.corda.finance.contracts.asset.CommercialPaper
In addition to using the CorDapp jars to update the whitelist, the bootstrapper will also copy them to all the nodes'
``cordapps`` directory.
Starting the nodes
~~~~~~~~~~~~~~~~~~

View File

@ -16,11 +16,11 @@ import net.corda.core.contracts.ContractClassName
import net.corda.core.internal.copyTo
import net.corda.core.internal.deleteIfExists
import net.corda.core.internal.read
import java.io.File
import java.io.InputStream
import java.lang.reflect.Modifier
import java.net.URLClassLoader
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.nio.file.StandardCopyOption
@ -28,13 +28,16 @@ import java.nio.file.StandardCopyOption
* Scans the jar for contracts.
* @returns: found contract class names or null if none found
*/
fun scanJarForContracts(cordappJarPath: String): List<ContractClassName> {
fun scanJarForContracts(cordappJar: Path): List<ContractClassName> {
val currentClassLoader = Contract::class.java.classLoader
val scanResult = FastClasspathScanner().addClassLoader(currentClassLoader).overrideClasspath(cordappJarPath, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()).toString()).scan()
val scanResult = FastClasspathScanner()
.addClassLoader(currentClassLoader)
.overrideClasspath(cordappJar, Paths.get(Contract::class.java.protectionDomain.codeSource.location.toURI()))
.scan()
val contracts = (scanResult.getNamesOfClassesImplementing(Contract::class.qualifiedName) ).distinct()
// Only keep instantiable contracts
return URLClassLoader(arrayOf(File(cordappJarPath).toURL()), currentClassLoader).use {
return URLClassLoader(arrayOf(cordappJar.toUri().toURL()), currentClassLoader).use {
contracts.map(it::loadClass).filter { !it.isInterface && !Modifier.isAbstract(it.modifiers) }
}.map { it.name }
}
@ -43,7 +46,7 @@ fun <T> withContractsInJar(jarInputStream: InputStream, withContracts: (List<Con
val tempFile = Files.createTempFile("attachment", ".jar")
try {
jarInputStream.copyTo(tempFile, StandardCopyOption.REPLACE_EXISTING)
val contracts = scanJarForContracts(tempFile.toAbsolutePath().toString())
val contracts = scanJarForContracts(tempFile.toAbsolutePath())
return tempFile.read { withContracts(contracts, it) }
} finally {
tempFile.deleteIfExists()

View File

@ -28,6 +28,7 @@ import net.corda.core.utilities.getOrThrow
import net.corda.core.utilities.seconds
import net.corda.nodeapi.internal.DEV_ROOT_CA
import net.corda.nodeapi.internal.SignedNodeInfo
import net.corda.nodeapi.internal.network.NodeInfoFilesCopier.Companion.NODE_INFO_FILE_NAME_PREFIX
import net.corda.nodeapi.internal.scanJarForContracts
import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
import net.corda.nodeapi.internal.serialization.CordaSerializationMagic
@ -63,15 +64,15 @@ class NetworkBootstrapper {
@JvmStatic
fun main(args: Array<String>) {
val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" }
val cordapps = if (args.size > 1) args.toList().drop(1) else null
NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordapps)
val cordappJars = if (args.size > 1) args.asList().drop(1).map { Paths.get(it) } else emptyList()
NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordappJars)
}
}
fun bootstrap(directory: Path, cordapps: List<String>?) {
fun bootstrap(directory: Path, cordappJars: List<Path>) {
directory.createDirectories()
println("Bootstrapping local network in $directory")
generateDirectoriesIfNeeded(directory)
generateDirectoriesIfNeeded(directory, cordappJars)
val nodeDirs = directory.list { paths -> paths.filter { (it / "corda.jar").exists() }.toList() }
require(nodeDirs.isNotEmpty()) { "No nodes found" }
println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}")
@ -88,7 +89,7 @@ class NetworkBootstrapper {
println("Gathering notary identities")
val notaryInfos = gatherNotaryInfos(nodeInfoFiles)
println("Generating contract implementations whitelist")
val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordapps?.distinct())
val newWhitelist = generateWhitelist(existingNetParams, directory / EXCLUDE_WHITELIST_FILE_NAME, cordappJars)
val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs)
println("${if (existingNetParams == null) "New" else "Updated"} $netParams")
println("Bootstrapping complete!")
@ -98,11 +99,11 @@ class NetworkBootstrapper {
}
}
private fun generateDirectoriesIfNeeded(directory: Path) {
private fun generateDirectoriesIfNeeded(directory: Path, cordappJars: List<Path>) {
val confFiles = directory.list { it.filter { it.toString().endsWith("_node.conf") }.toList() }
val webServerConfFiles = directory.list { it.filter { it.toString().endsWith("_web-server.conf") }.toList() }
if (confFiles.isEmpty()) return
println("Node config files found in the root directory - generating node directories")
println("Node config files found in the root directory - generating node directories and copying CorDapp jars into them")
val cordaJar = extractCordaJarTo(directory)
for (confFile in confFiles) {
val nodeName = confFile.fileName.toString().removeSuffix("_node.conf")
@ -111,6 +112,8 @@ class NetworkBootstrapper {
confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING)
webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING)
cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING)
val cordappsDir = (nodeDir / "cordapps").createDirectories()
cordappJars.forEach { it.copyToDirectory(cordappsDir) }
}
Files.delete(cordaJar)
}
@ -118,7 +121,6 @@ class NetworkBootstrapper {
private fun extractCordaJarTo(directory: Path): Path {
val cordaJarPath = directory / "corda.jar"
if (!cordaJarPath.exists()) {
println("No corda jar found in root directory. Extracting from jar")
Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").copyTo(cordaJarPath)
}
return cordaJarPath
@ -144,7 +146,7 @@ class NetworkBootstrapper {
check(process.waitFor() == 0) {
"Node in ${nodeDir.fileName} exited with ${process.exitValue()} when generating its node-info - see logs in ${nodeDir / LOGS_DIR_NAME}"
}
nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith("nodeInfo-") }.findFirst().get() }
nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() }
}
}
@ -192,7 +194,7 @@ class NetworkBootstrapper {
}
val msg = StringBuilder("Differing sets of network parameters were found. Make sure all the nodes have the same " +
"network parameters by copying over the correct $NETWORK_PARAMS_FILE_NAME file.\n\n")
"network parameters by copying the correct $NETWORK_PARAMS_FILE_NAME file across.\n\n")
netParamsFilesGrouped.forEach { bytes, netParamsFiles ->
netParamsFiles.map { it.parent.fileName }.joinTo(msg, ", ")
@ -221,6 +223,7 @@ class NetworkBootstrapper {
epoch = existingNetParams.epoch + 1
)
} else {
// TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
NetworkParameters(
minimumPlatformVersion = 1,
notaries = notaryInfos,
@ -231,28 +234,25 @@ class NetworkBootstrapper {
epoch = 1
)
}
// TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize
val copier = NetworkParametersCopier(networkParameters, overwriteFile = true)
nodeDirs.forEach { copier.install(it) }
nodeDirs.forEach(copier::install)
return networkParameters
}
private fun generateWhitelist(networkParameters: NetworkParameters?,
excludeWhitelistFile: Path,
cordapps: List<String>?): Map<String, List<AttachmentId>> {
cordappJars: List<Path>): Map<String, List<AttachmentId>> {
val existingWhitelist = networkParameters?.whitelistedContractImplementations ?: emptyMap()
val excludeContracts = if (excludeWhitelistFile.exists()) readExcludeWhitelist(excludeWhitelistFile) else emptyList()
val excludeContracts = readExcludeWhitelist(excludeWhitelistFile)
if (excludeContracts.isNotEmpty()) {
println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}}")
println("Exclude contracts from whitelist: ${excludeContracts.joinToString()}")
}
val newWhiteList = cordapps?.flatMap { cordappJarPath ->
val jarHash = Paths.get(cordappJarPath).hash
scanJarForContracts(cordappJarPath).map { contract ->
contract to jarHash
}
}?.filter { (contractClassName, _) -> contractClassName !in excludeContracts }?.toMap() ?: emptyMap()
val newWhiteList = cordappJars.flatMap { cordappJar ->
val jarHash = cordappJar.hash
scanJarForContracts(cordappJar).map { contract -> contract to jarHash }
}.filter { (contractClassName, _) -> contractClassName !in excludeContracts }.toMap()
return (newWhiteList.keys + existingWhitelist.keys).map { contractClassName ->
val existing = existingWhitelist[contractClassName] ?: emptyList()
@ -261,7 +261,9 @@ class NetworkBootstrapper {
}.toMap()
}
private fun readExcludeWhitelist(file: Path): List<String> = file.readAllLines().map(String::trim)
private fun readExcludeWhitelist(file: Path): List<String> {
return if (file.exists()) file.readAllLines().map(String::trim) else emptyList()
}
private fun NodeInfo.notaryIdentity(): Party {
return when (legalIdentities.size) {

View File

@ -43,10 +43,14 @@ data class NetworkMap(
val parametersUpdate: ParametersUpdate?
) {
override fun toString(): String {
return """${NetworkMap::class.java.simpleName}(nodeInfoHashes=
${nodeInfoHashes.joinToString("\n")}
networkParameterHash=$networkParameterHash
parametersUpdate=$parametersUpdate)"""
return """NetworkMap {
nodeInfoHashes {
${nodeInfoHashes.asSequence().take(10).joinToString("\n ")}
${if (nodeInfoHashes.size > 10) "... ${nodeInfoHashes.size - 10} more" else ""}
}
networkParameterHash=$networkParameterHash
parametersUpdate=$parametersUpdate
}"""
}
}

View File

@ -499,4 +499,46 @@ public class VaultQueryJavaTests {
return tx;
});
}
private List<StateAndRef<Cash.State>> queryStatesWithPaging(VaultService vaultService, int pageSize) {
// DOCSTART VaultQueryExample24
int pageNumber = DEFAULT_PAGE_NUM;
List<StateAndRef<Cash.State>> states = new ArrayList<>();
long totalResults;
do {
PageSpecification pageSpec = new PageSpecification(pageNumber, pageSize);
Vault.Page<Cash.State> results = vaultService.queryBy(Cash.State.class, new VaultQueryCriteria(), pageSpec);
totalResults = results.getTotalStatesAvailable();
List<StateAndRef<Cash.State>> newStates = results.getStates();
System.out.println(newStates.size());
states.addAll(results.getStates());
pageNumber++;
} while ((pageSize * (pageNumber - 1) <= totalResults));
// DOCEND VaultQueryExample24
return states;
}
@Test
public void testExampleOfQueryingStatesWithPagingWorksCorrectly() {
Amount<Currency> dollars100 = new Amount<>(100, Currency.getInstance("USD"));
database.transaction(tx -> {
vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 4, DUMMY_CASH_ISSUER);
return tx;
});
database.transaction(tx -> {
assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(4);
vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER);
return tx;
});
database.transaction(tx -> {
assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(5);
vaultFiller.fillWithSomeTestCash(dollars100, issuerServices, 1, DUMMY_CASH_ISSUER);
return tx;
});
database.transaction(tx -> {
assertThat(queryStatesWithPaging(vaultService, 5).size()).isEqualTo(6);
return tx;
});
}
}

View File

@ -21,8 +21,6 @@ import net.corda.core.flows.SchedulableFlow
import net.corda.core.identity.Party
import net.corda.core.node.services.VaultService
import net.corda.core.node.services.queryBy
import net.corda.core.node.services.vault.DEFAULT_PAGE_NUM
import net.corda.core.node.services.vault.PageSpecification
import net.corda.core.node.services.vault.QueryCriteria.VaultQueryCriteria
import net.corda.core.node.services.vault.Sort
import net.corda.core.node.services.vault.SortAttribute
@ -151,7 +149,7 @@ class ScheduledFlowTests {
@Test
fun `run a whole batch of scheduled flows`() {
val N = 100
val N = 99
val futures = mutableListOf<CordaFuture<*>>()
for (i in 0 until N) {
futures.add(aliceNode.services.startFlow(InsertInitialStateFlow(bob, notary)).resultFuture)
@ -164,10 +162,10 @@ class ScheduledFlowTests {
// Convert the states into maps to make error reporting easier
val statesFromA: List<StateAndRef<ScheduledState>> = aliceNode.database.transaction {
queryStatesWithPaging(aliceNode.services.vaultService)
queryStates(aliceNode.services.vaultService)
}
val statesFromB: List<StateAndRef<ScheduledState>> = bobNode.database.transaction {
queryStatesWithPaging(bobNode.services.vaultService)
queryStates(bobNode.services.vaultService)
}
assertEquals("Expect all states to be present", 2 * N, statesFromA.count())
statesFromA.forEach { ref ->
@ -184,23 +182,6 @@ class ScheduledFlowTests {
assertTrue("Expect all states have run the scheduled task", statesFromB.all { it.state.data.processed })
}
/**
* Query all states from the Vault, fetching results as a series of pages with ordered states in order to perform
* integration testing of that functionality.
*
* @return states ordered by the transaction ID.
*/
private fun queryStatesWithPaging(vaultService: VaultService): List<StateAndRef<ScheduledState>> {
// DOCSTART VaultQueryExamplePaging
var pageNumber = DEFAULT_PAGE_NUM
val states = mutableListOf<StateAndRef<ScheduledState>>()
do {
val pageSpec = PageSpecification(pageSize = PAGE_SIZE, pageNumber = pageNumber)
val results = vaultService.queryBy<ScheduledState>(VaultQueryCriteria(), pageSpec, SORTING)
states.addAll(results.states)
pageNumber++
} while ((pageSpec.pageSize * (pageNumber)) <= results.totalStatesAvailable)
// DOCEND VaultQueryExamplePaging
return states.toList()
}
private fun queryStates(vaultService: VaultService): List<StateAndRef<ScheduledState>> =
vaultService.queryBy<ScheduledState>(VaultQueryCriteria(), sorting = SORTING).states
}

View File

@ -1008,14 +1008,39 @@ open class VaultQueryTests {
}
}
// example of querying states with paging using totalStatesAvailable
private fun queryStatesWithPaging(vaultService: VaultService, pageSize: Int): List<StateAndRef<ContractState>> {
// DOCSTART VaultQueryExample24
var pageNumber = DEFAULT_PAGE_NUM
val states = mutableListOf<StateAndRef<ContractState>>()
do {
val pageSpec = PageSpecification(pageNumber = pageNumber, pageSize = pageSize)
val results = vaultService.queryBy<ContractState>(VaultQueryCriteria(), pageSpec)
states.addAll(results.states)
pageNumber++
} while ((pageSpec.pageSize * (pageNumber - 1)) <= results.totalStatesAvailable)
// DOCEND VaultQueryExample24
return states.toList()
}
// test paging query example works
@Test
fun `test example of querying states with paging works correctly`() {
database.transaction {
vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 4, DUMMY_CASH_ISSUER)
assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(4)
vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER)
assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(5)
vaultFiller.fillWithSomeTestCash(25.DOLLARS, notaryServices, 1, DUMMY_CASH_ISSUER)
assertThat(queryStatesWithPaging(vaultService, 5).count()).isEqualTo(6)
}
}
// sorting
@Test
fun `sorting - all states sorted by contract type, state status, consumed time`() {
setUpDb(database)
database.transaction {
val sortCol1 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONTRACT_STATE_TYPE), Sort.Direction.DESC)
val sortCol2 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.STATE_STATUS), Sort.Direction.ASC)
val sortCol3 = Sort.SortColumn(SortAttribute.Standard(Sort.VaultStateAttribute.CONSUMED_TIME), Sort.Direction.DESC)