diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 5dbfa353df..356d4cafaa 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -26,6 +26,8 @@ + + @@ -58,11 +60,15 @@ + + + + @@ -106,6 +112,10 @@ + + + + @@ -148,8 +158,13 @@ + + + + + @@ -175,6 +190,8 @@ + + @@ -182,6 +199,8 @@ + + diff --git a/build.gradle b/build.gradle index 07d610fb99..7ded190f54 100644 --- a/build.gradle +++ b/build.gradle @@ -81,6 +81,7 @@ buildscript { ext.snappy_version = '0.4' ext.fast_classpath_scanner_version = '2.12.3' ext.jcabi_manifests_version = '1.1' + ext.picocli_version = '3.0.0' ext.deterministic_rt_version = '1.0-SNAPSHOT' // Name of the IntelliJ SDK created for the deterministic Java rt.jar. diff --git a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt index 9eb005d3ce..45be94b6ff 100644 --- a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt @@ -14,6 +14,7 @@ import net.corda.core.flows.FlowLogic import net.corda.core.identity.CordaX500Name import net.corda.core.node.ServicesForResolution import net.corda.core.serialization.* +import net.corda.core.transactions.LedgerTransaction import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction @@ -386,12 +387,19 @@ fun uncheckedCast(obj: T) = obj as U fun Iterable>.toMultiMap(): Map> = this.groupBy({ it.first }) { it.second } /** Provide access to internal method for AttachmentClassLoaderTests */ -@DeleteForDJVM fun TransactionBuilder.toWireTransaction(services: ServicesForResolution, serializationContext: SerializationContext): WireTransaction { +@DeleteForDJVM +fun TransactionBuilder.toWireTransaction(services: ServicesForResolution, serializationContext: SerializationContext): WireTransaction { return toWireTransactionWithContext(services, serializationContext) } /** Provide access to internal method for AttachmentClassLoaderTests */ -@DeleteForDJVM fun TransactionBuilder.toLedgerTransaction(services: ServicesForResolution, serializationContext: SerializationContext) = toLedgerTransactionWithContext(services, serializationContext) +@DeleteForDJVM +fun TransactionBuilder.toLedgerTransaction(services: ServicesForResolution, serializationContext: SerializationContext): LedgerTransaction { + return toLedgerTransactionWithContext(services, serializationContext) +} + +/** Returns the location of this class. */ +val Class<*>.location: URL get() = protectionDomain.codeSource.location /** Convenience method to get the package name of a class literal. */ val KClass<*>.packageName: String get() = java.packageName diff --git a/core/src/main/kotlin/net/corda/core/internal/PathUtils.kt b/core/src/main/kotlin/net/corda/core/internal/PathUtils.kt index 7ef532e06b..50e05c06b6 100644 --- a/core/src/main/kotlin/net/corda/core/internal/PathUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/PathUtils.kt @@ -81,6 +81,9 @@ fun Path.lastModifiedTime(vararg options: LinkOption): FileTime = Files.getLastM /** @see Files.isDirectory */ fun Path.isDirectory(vararg options: LinkOption): Boolean = Files.isDirectory(this, *options) +/** @see Files.isSameFile */ +fun Path.isSameAs(other: Path): Boolean = Files.isSameFile(this, other) + /** * Same as [Files.list] except it also closes the [Stream]. * @return the output of [block] diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 40a796ebce..131b06a46e 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -243,7 +243,7 @@ Version 3.0 * Cordform (which is the ``deployNodes`` gradle task) does this copying automatically for the demos. The ``NetworkMap`` parameter is no longer needed. - * For test deployments we've introduced a bootstrapping tool (see :doc:`setting-up-a-corda-network`). + * For test deployments we've introduced a bootstrapping tool (see :doc:`network-bootstrapper`). * ``extraAdvertisedServiceIds``, ``notaryNodeAddress``, ``notaryClusterAddresses`` and ``bftSMaRt`` configs have been removed. The configuration of notaries has been simplified into a single ``notary`` config object. See diff --git a/docs/source/network-bootstrapper.rst b/docs/source/network-bootstrapper.rst new file mode 100644 index 0000000000..e0c3900927 --- /dev/null +++ b/docs/source/network-bootstrapper.rst @@ -0,0 +1,101 @@ +Network Bootstrapper +==================== + +Test deployments +~~~~~~~~~~~~~~~~ + +Nodes within a network see each other using the network map. This is a collection of statically signed node-info files, +one for each node. Most production deployments will use a highly available, secure distribution of the network map via HTTP. + +For test deployments where the nodes (at least initially) reside on the same filesystem, these node-info files can be +placed directly in the node's ``additional-node-infos`` directory from where the node will pick them up and store them +in its local network map cache. The node generates its own node-info file on startup. + +In addition to the network map, all the nodes must also use the same set of network parameters. These are a set of constants +which guarantee interoperability between the nodes. The HTTP network map distributes the network parameters which are downloaded +automatically by the nodes. In the absence of this the network parameters must be generated locally. + +For these reasons, test deployments can avail themselves of the network bootstrapper. This is a tool that scans all the +node configurations from a common directory to generate the network parameters file, which is then copied to all the nodes' +directories. It also copies each node's node-info file to every other node so that they can all be visible to each other. + +You can find out more about network maps and network parameters from :doc:`network-map`. + +Bootstrapping a test network +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The bootstrapper can be downloaded from https://downloads.corda.net/network-bootstrapper-VERSION.jar, where ``VERSION`` +is the Corda version. + +Create a directory containing a node config file, ending in "_node.conf", for each node you want to create. Then run the +following command: + +``java -jar network-bootstrapper-VERSION.jar --dir `` + +For example running the command on a directory containing these files: + +.. sourcecode:: none + + . + ├── notary_node.conf // The notary's node.conf file + ├── partya_node.conf // Party A's node.conf file + └── partyb_node.conf // Party B's node.conf file + +will generate directories containing three nodes: ``notary``, ``partya`` and ``partyb``. They will each use the ``corda.jar`` +that comes with the bootstrapper. If a different version of Corda is required then simply place that ``corda.jar`` file +alongside the configuration files in the directory. + +The directory can also contain CorDapp JARs which will be copied to each node's ``cordapps`` directory. + +You can also have the node directories containing their "node.conf" files already laid out. The previous example would be: + +.. sourcecode:: none + + . + ├── notary + │   └── node.conf + ├── partya + │   └── node.conf + └── partyb + └── node.conf + +Similarly, each node directory may contain its own ``corda.jar``, which the bootstrapper will use instead. + +Synchronisation +~~~~~~~~~~~~~~~ + +This tool only bootstraps a network. It cannot dynamically update if a new node needs to join the network or if an existing +one has changed something in their node-info, e.g. their P2P address. For this the new node-info file will need to be placed +in the other nodes' ``additional-node-infos`` directory. A simple way to do this is to use `rsync `_. +However, if it's known beforehand the set of nodes that will eventually form part of the network then all the node directories +can be pregenerated in the bootstrap and only started when needed. + +Running the bootstrapper again on the same network will allow a new node to be added or an existing one to have its updated +node-info re-distributed. However this comes at the expense of having to temporarily collect the node directories back +together again under a common parent directory. + +Whitelisting contracts +~~~~~~~~~~~~~~~~~~~~~~ + +The CorDapp JARs are also automatically used to create the *Zone whitelist* (see :doc:`api-contract-constraints`) for +the network. + +.. note:: If you only wish to whitelist the CorDapps but not copy them to each node then run with the ``--no-copy`` flag. + +The CorDapp JARs will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part +of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`). +If the network already has a set of network parameters defined (i.e. the node directories all contain the same network-parameters +file) then the new set of contracts will be appended to the current whitelist. + +.. note:: The whitelist can only ever be appended to. Once added a contract implementation can never be removed. + +By default the bootstrapper will whitelist all the contracts found in all the CorDapp JARs. To prevent certain +contracts from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``. These will instead +use the more restrictive ``HashAttachmentConstraint``. + +For example: + +.. sourcecode:: none + + net.corda.finance.contracts.asset.Cash + net.corda.finance.contracts.asset.CommercialPaper diff --git a/docs/source/network-map.rst b/docs/source/network-map.rst index c6f83ae8b4..b302a2d426 100644 --- a/docs/source/network-map.rst +++ b/docs/source/network-map.rst @@ -72,7 +72,7 @@ the network, along with the network parameters file and identity certificates. G online at once - an offline node that isn't being interacted with doesn't impact the network in any way. So a test cluster generated like this can be sized for the maximum size you may need, and then scaled up and down as necessary. -More information can be found in :doc:`setting-up-a-corda-network`. +More information can be found in :doc:`network-bootstrapper`. Network parameters ------------------ diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index 8f1e86ade2..c918aff957 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -169,7 +169,7 @@ Significant Changes in 3.0 .. important:: This replaces the Network Map service that was present in Corda 1.0 and Corda 2.0. - Further information can be found in the :doc:`changelog`, :doc:`network-map` and :doc:`setting-up-a-corda-network` documentation. + Further information can be found in the :doc:`changelog`, :doc:`network-map` and :doc:`network-bootstrapper` documentation. * **Contract Upgrade** diff --git a/docs/source/setting-up-a-corda-network.rst b/docs/source/setting-up-a-corda-network.rst index ce0d70a8b6..b8946a2841 100644 --- a/docs/source/setting-up-a-corda-network.rst +++ b/docs/source/setting-up-a-corda-network.rst @@ -46,81 +46,14 @@ The most important fields regarding network configuration are: and ``rpcAddress`` if they are on the same machine. * ``notary.serviceLegalName``: The name of the notary service, required to setup distributed notaries with the network-bootstrapper. -Bootstrapping the network -~~~~~~~~~~~~~~~~~~~~~~~~~ - -The nodes see each other using the network map. This is a collection of statically signed node-info files, one for each -node in the network. Most production deployments will use a highly available, secure distribution of the network map via HTTP. - -For test deployments where the nodes (at least initially) reside on the same filesystem, these node-info files can be -placed directly in the node's ``additional-node-infos`` directory from where the node will pick them up and store them -in its local network map cache. The node generates its own node-info file on startup. - -In addition to the network map, all the nodes on a network must use the same set of network parameters. These are a set -of constants which guarantee interoperability between nodes. The HTTP network map distributes the network parameters -which the node downloads automatically. In the absence of this the network parameters must be generated locally. This can -be done with the network bootstrapper. This is a tool that scans all the node configurations from a common directory to -generate the network parameters file which is copied to the nodes' directories. It also copies each node's node-info file -to every other node so that they can all transact with each other. - -The bootstrapper tool can be downloaded from https://downloads.corda.net/network-bootstrapper-corda-X.Y.jar, where ``X`` -is the major Corda version and ``Y`` is the minor Corda version. - -To use it, create a directory containing a node config file, ending in "_node.conf", for each node you want to create. -Then run the following command: - -``java -jar network-bootstrapper-corda-X.Y.jar `` - -For example running the command on a directory containing these files : - -.. sourcecode:: none - - . - ├── notary_node.conf // The notary's node.conf file - ├── partya_node.conf // Party A's node.conf file - └── partyb_node.conf // Party B's node.conf file - -Would generate directories containing three nodes: notary, partya and partyb. - -This tool only bootstraps a network. It cannot dynamically update if a new node needs to join the network or if an existing -one has changed something in their node-info, e.g. their P2P address. For this the new node-info file will need to be placed -in the other nodes' ``additional-node-infos`` directory. A simple way to do this is to use `rsync `_. -However, if it's known beforehand the set of nodes that will eventually the node folders can be pregenerated in the bootstrap -and only started when needed. - -Whitelisting Contracts -~~~~~~~~~~~~~~~~~~~~~~ - -If you want to create a *Zone whitelist* (see :doc:`api-contract-constraints`), you can pass in a list of CorDapp jars: - -``java -jar network-bootstrapper.jar <1st CorDapp jar> <2nd CorDapp jar> ..`` - -The CorDapp jars will be hashed and scanned for ``Contract`` classes. These contract class implementations will become part -of the whitelisted contracts in the network parameters (see ``NetworkParameters.whitelistedContractImplementations`` :doc:`network-map`). -If the network already has a set of network parameters defined (i.e. the node directories all contain the same network-parameters -file) then the new set of contracts will be appended to the current whitelist. - -.. note:: The whitelist can only ever be appended to. Once added a contract implementation can never be removed. - -By default the bootstrapper tool will whitelist all the contracts found in all the CorDapp jars. To prevent certain -contracts from being whitelisted, add their fully qualified class name in the ``exclude_whitelist.txt``. These will instead -use the more restrictive ``HashAttachmentConstraint``. - -For example: - -.. sourcecode:: none - - net.corda.finance.contracts.asset.Cash - net.corda.finance.contracts.asset.CommercialPaper - -In addition to using the CorDapp jars to update the whitelist, the bootstrapper will also copy them to all the nodes' -``cordapps`` directory. - Starting the nodes ~~~~~~~~~~~~~~~~~~ -You may now start the nodes in any order. You should see a banner, some log lines and eventually ``Node started up and registered``, -indicating that the node is fully started. +You will first need to create the local network by bootstrapping it with the bootstrapper. Details of how to do that can +be found in :doc:`network-bootstrapper`. + +Once that's done you may now start the nodes in any order. You should see a banner, some log lines and eventually +``Node started up and registered``, indicating that the node is fully started. .. TODO: Add a better way of polling for startup. A programmatic way of determining whether a node is up is to check whether it's ``webAddress`` is bound. diff --git a/docs/source/tools-index.rst b/docs/source/tools-index.rst index f2213c7496..2285b38d9d 100644 --- a/docs/source/tools-index.rst +++ b/docs/source/tools-index.rst @@ -4,6 +4,7 @@ Tools .. toctree:: :maxdepth: 1 + network-bootstrapper blob-inspector demobench node-explorer diff --git a/docs/source/tutorial-cordapp.rst b/docs/source/tutorial-cordapp.rst index 2c2447d448..961b17c6dd 100644 --- a/docs/source/tutorial-cordapp.rst +++ b/docs/source/tutorial-cordapp.rst @@ -418,7 +418,7 @@ be moved to another machine open its config file and change the Artemis messagin where the node will run (e.g. ``p2pAddress="10.18.0.166:10007"``). These changes require new node-info files to be distributed amongst the nodes. Use the network bootstrapper tool -(see :doc:`setting-up-a-corda-network` for more information on this and how to built it) to update the files and have +(see :doc:`network-bootstrapper` for more information on this and how to built it) to update the files and have them distributed locally. ``java -jar network-bootstrapper.jar kotlin-source/build/nodes`` diff --git a/docs/source/upgrade-notes.rst b/docs/source/upgrade-notes.rst index b9dd1a64bf..c7f90a8eca 100644 --- a/docs/source/upgrade-notes.rst +++ b/docs/source/upgrade-notes.rst @@ -117,7 +117,7 @@ With the re-designed network map service the following changes need to be made: * The network map is no longer provided by a node and thus the ``networkMapService`` config is ignored. Instead the network map is either provided by the compatibility zone (CZ) operator (who operates the doorman) and available using the ``compatibilityZoneURL`` config, or is provided using signed node info files which are copied locally. - See :doc:`network-map` for more details, and :doc:`setting-up-a-corda-network` on how to use the network + See :doc:`network-map` for more details, and :doc:`network-bootstrapper` on how to use the network bootstrapper for deploying a local network. * Configuration for a notary has been simplified. ``extraAdvertisedServiceIds``, ``notaryNodeAddress``, ``notaryClusterAddresses`` diff --git a/experimental/behave/prepare.sh b/experimental/behave/prepare.sh index bdd41c4a1b..1d78595971 100755 --- a/experimental/behave/prepare.sh +++ b/experimental/behave/prepare.sh @@ -31,5 +31,5 @@ curl "https://search.maven.org/remotecontent?filepath=com/h2database/h2/1.4.196/ curl -L "http://central.maven.org/maven2/org/postgresql/postgresql/42.1.4/postgresql-42.1.4.jar" > ${DRIVERS_DIR}/postgresql-42.1.4.jar # Build Network Bootstrapper -./gradlew buildBootstrapperJar +./gradlew tools:bootstrapper:jar cp -v $(ls tools/bootstrapper/build/libs/*.jar | tail -n1) ${CORDA_DIR}/network-bootstrapper.jar diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt index 5438953cf5..3dbc6589bf 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt @@ -27,8 +27,8 @@ import net.corda.serialization.internal.CordaSerializationMagic import net.corda.serialization.internal.SerializationFactoryImpl import net.corda.serialization.internal.amqp.AbstractAMQPSerializationScheme import net.corda.serialization.internal.amqp.amqpMagic +import java.io.InputStream import java.nio.file.Path -import java.nio.file.Paths import java.nio.file.StandardCopyOption.REPLACE_EXISTING import java.time.Instant import java.util.* @@ -43,7 +43,21 @@ import kotlin.streams.toList /** * Class to bootstrap a local network of Corda nodes on the same filesystem. */ -class NetworkBootstrapper { +// TODO Move this to tools:bootstrapper +class NetworkBootstrapper + @VisibleForTesting + internal constructor(private val initSerEnv: Boolean, + private val embeddedCordaJar: () -> InputStream, + private val nodeInfosGenerator: (List) -> List, + private val contractsJarConverter: (Path) -> ContractsJar) { + + constructor() : this( + initSerEnv = true, + embeddedCordaJar = Companion::extractEmbeddedCordaJar, + nodeInfosGenerator = Companion::generateNodeInfos, + contractsJarConverter = ::ContractsJarFile + ) + companion object { // TODO This will probably need to change once we start using a bundled JVM private val nodeInfoGenCmd = listOf( @@ -55,11 +69,42 @@ class NetworkBootstrapper { private const val LOGS_DIR_NAME = "logs" - @JvmStatic - fun main(args: Array) { - val baseNodeDirectory = requireNotNull(args.firstOrNull()) { "Expecting first argument which is the nodes' parent directory" } - val cordappJars = if (args.size > 1) args.asList().drop(1).map { Paths.get(it) } else emptyList() - NetworkBootstrapper().bootstrap(Paths.get(baseNodeDirectory).toAbsolutePath().normalize(), cordappJars) + private fun extractEmbeddedCordaJar(): InputStream { + return Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar") + } + + private fun generateNodeInfos(nodeDirs: List): List { + val numParallelProcesses = Runtime.getRuntime().availableProcessors() + val timePerNode = 40.seconds // On the test machine, generating the node info takes 7 seconds for a single node. + val tExpected = maxOf(timePerNode, timePerNode * nodeDirs.size.toLong() / numParallelProcesses.toLong()) + val warningTimer = Timer("WarnOnSlowMachines", false).schedule(tExpected.toMillis()) { + println("... still waiting. If this is taking longer than usual, check the node logs.") + } + val executor = Executors.newFixedThreadPool(numParallelProcesses) + return try { + nodeDirs.map { executor.fork { generateNodeInfo(it) } }.transpose().getOrThrow() + } finally { + warningTimer.cancel() + executor.shutdownNow() + } + } + + private fun generateNodeInfo(nodeDir: Path): Path { + val logsDir = (nodeDir / LOGS_DIR_NAME).createDirectories() + val process = ProcessBuilder(nodeInfoGenCmd) + .directory(nodeDir.toFile()) + .redirectErrorStream(true) + .redirectOutput((logsDir / "node-info-gen.log").toFile()) + .apply { environment()["CAPSULE_CACHE_DIR"] = "../.cache" } + .start() + if (!process.waitFor(3, TimeUnit.MINUTES)) { + process.destroyForcibly() + throw IllegalStateException("Error while generating node info file. Please check the logs in $logsDir.") + } + check(process.exitValue() == 0) { "Error while generating node info file. Please check the logs in $logsDir." } + return nodeDir.list { paths -> + paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() + } } } @@ -92,29 +137,61 @@ class NetworkBootstrapper { private fun generateServiceIdentitiesForNotaryClusters(configs: Map) { notaryClusters(configs).forEach { (cluster, directories) -> when (cluster) { - is NotaryCluster.BFT -> - DevIdentityGenerator.generateDistributedNotaryCompositeIdentity(directories, cluster.name, threshold = 1 + 2 * directories.size / 3) - is NotaryCluster.CFT -> - DevIdentityGenerator.generateDistributedNotarySingularIdentity(directories, cluster.name) + is NotaryCluster.BFT -> DevIdentityGenerator.generateDistributedNotaryCompositeIdentity( + directories, + cluster.name, + threshold = 1 + 2 * directories.size / 3 + ) + is NotaryCluster.CFT -> DevIdentityGenerator.generateDistributedNotarySingularIdentity(directories, cluster.name) } } } + /** Entry point for Cordform */ fun bootstrap(directory: Path, cordappJars: List) { + bootstrap(directory, cordappJars, copyCordapps = true, fromCordform = true) + } + + /** Entry point for the tool */ + fun bootstrap(directory: Path, copyCordapps: Boolean) { + // Don't accidently include the bootstrapper jar as a CorDapp! + val bootstrapperJar = javaClass.location.toPath() + val cordappJars = directory.list { paths -> + paths.filter { it.toString().endsWith(".jar") && !it.isSameAs(bootstrapperJar) && it.fileName.toString() != "corda.jar" }.toList() + } + bootstrap(directory, cordappJars, copyCordapps, fromCordform = false) + } + + private fun bootstrap(directory: Path, cordappJars: List, copyCordapps: Boolean, fromCordform: Boolean) { directory.createDirectories() - println("Bootstrapping local network in $directory") - generateDirectoriesIfNeeded(directory, cordappJars) - val nodeDirs = directory.list { paths -> paths.filter { (it / "corda.jar").exists() }.toList() } + println("Bootstrapping local test network in $directory") + if (!fromCordform) { + println("Found the following CorDapps: ${cordappJars.map { it.fileName }}") + } + createNodeDirectoriesIfNeeded(directory, fromCordform) + val nodeDirs = gatherNodeDirectories(directory) + require(nodeDirs.isNotEmpty()) { "No nodes found" } - println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}") + if (!fromCordform) { + println("Nodes found in the following sub-directories: ${nodeDirs.map { it.fileName }}") + } + val configs = nodeDirs.associateBy({ it }, { ConfigFactory.parseFile((it / "node.conf").toFile()) }) + checkForDuplicateLegalNames(configs.values) + if (copyCordapps && cordappJars.isNotEmpty()) { + println("Copying CorDapp JARs into node directories") + for (nodeDir in nodeDirs) { + val cordappsDir = (nodeDir / "cordapps").createDirectories() + cordappJars.forEach { it.copyToDirectory(cordappsDir) } + } + } generateServiceIdentitiesForNotaryClusters(configs) - initialiseSerialization() + if (initSerEnv) { + initialiseSerialization() + } try { println("Waiting for all nodes to generate their node-info files...") - val nodeInfoFiles = generateNodeInfos(nodeDirs) - println("Checking for duplicate nodes") - checkForDuplicateLegalNames(nodeInfoFiles) + val nodeInfoFiles = nodeInfosGenerator(nodeDirs) println("Distributing all node-info files to all nodes") distributeNodeInfos(nodeDirs, nodeInfoFiles) print("Loading existing network parameters... ") @@ -123,72 +200,70 @@ class NetworkBootstrapper { println("Gathering notary identities") val notaryInfos = gatherNotaryInfos(nodeInfoFiles, configs) println("Generating contract implementations whitelist") - val newWhitelist = generateWhitelist(existingNetParams, readExcludeWhitelist(directory), cordappJars.map(::ContractsJarFile)) - val netParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs) - println("${if (existingNetParams == null) "New" else "Updated"} $netParams") + val newWhitelist = generateWhitelist(existingNetParams, readExcludeWhitelist(directory), cordappJars.map(contractsJarConverter)) + val newNetParams = installNetworkParameters(notaryInfos, newWhitelist, existingNetParams, nodeDirs) + if (newNetParams != existingNetParams) { + println("${if (existingNetParams == null) "New" else "Updated"} $newNetParams") + } else { + println("Network parameters unchanged") + } println("Bootstrapping complete!") } finally { - _contextSerializationEnv.set(null) + if (initSerEnv) { + _contextSerializationEnv.set(null) + } } } - private fun generateNodeInfos(nodeDirs: List): List { - val numParallelProcesses = Runtime.getRuntime().availableProcessors() - val timePerNode = 40.seconds // On the test machine, generating the node info takes 7 seconds for a single node. - val tExpected = maxOf(timePerNode, timePerNode * nodeDirs.size.toLong() / numParallelProcesses.toLong()) - val warningTimer = Timer("WarnOnSlowMachines", false).schedule(tExpected.toMillis()) { - println("...still waiting. If this is taking longer than usual, check the node logs.") + private fun createNodeDirectoriesIfNeeded(directory: Path, fromCordform: Boolean) { + val cordaJar = directory / "corda.jar" + var usingEmbedded = false + if (!cordaJar.exists()) { + embeddedCordaJar().use { it.copyTo(cordaJar) } + usingEmbedded = true + } else if (!fromCordform) { + println("Using corda.jar in root directory") } - val executor = Executors.newFixedThreadPool(numParallelProcesses) - return try { - nodeDirs.map { executor.fork { generateNodeInfo(it) } }.transpose().getOrThrow() - } finally { - warningTimer.cancel() - executor.shutdownNow() - } - } - private fun generateNodeInfo(nodeDir: Path): Path { - val logsDir = (nodeDir / LOGS_DIR_NAME).createDirectories() - val process = ProcessBuilder(nodeInfoGenCmd) - .directory(nodeDir.toFile()) - .redirectErrorStream(true) - .redirectOutput((logsDir / "node-info-gen.log").toFile()) - .apply { environment()["CAPSULE_CACHE_DIR"] = "../.cache" } - .start() - if (!process.waitFor(3, TimeUnit.MINUTES)) { - process.destroyForcibly() - throw IllegalStateException("Error while generating node info file. Please check the logs in $logsDir.") - } - check(process.exitValue() == 0) { "Error while generating node info file. Please check the logs in $logsDir." } - return nodeDir.list { paths -> paths.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.findFirst().get() } - } - - private fun generateDirectoriesIfNeeded(directory: Path, cordappJars: List) { val confFiles = directory.list { it.filter { it.toString().endsWith("_node.conf") }.toList() } val webServerConfFiles = directory.list { it.filter { it.toString().endsWith("_web-server.conf") }.toList() } - if (confFiles.isEmpty()) return - println("Node config files found in the root directory - generating node directories and copying CorDapp jars into them") - val cordaJar = extractCordaJarTo(directory) + for (confFile in confFiles) { val nodeName = confFile.fileName.toString().removeSuffix("_node.conf") - println("Generating directory for $nodeName") + println("Generating node directory for $nodeName") val nodeDir = (directory / nodeName).createDirectories() - confFile.moveTo(nodeDir / "node.conf", REPLACE_EXISTING) - webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.moveTo(nodeDir / "web-server.conf", REPLACE_EXISTING) + confFile.copyTo(nodeDir / "node.conf", REPLACE_EXISTING) + webServerConfFiles.firstOrNull { directory.relativize(it).toString().removeSuffix("_web-server.conf") == nodeName }?.copyTo(nodeDir / "web-server.conf", REPLACE_EXISTING) cordaJar.copyToDirectory(nodeDir, REPLACE_EXISTING) - val cordappsDir = (nodeDir / "cordapps").createDirectories() - cordappJars.forEach { it.copyToDirectory(cordappsDir) } } - cordaJar.delete() + + directory.list { paths -> + paths.filter { (it / "node.conf").exists() && !(it / "corda.jar").exists() }.forEach { + println("Copying corda.jar into node directory ${it.fileName}") + cordaJar.copyToDirectory(it) + } + } + + if (fromCordform) { + confFiles.forEach(Path::delete) + webServerConfFiles.forEach(Path::delete) + } + + if (fromCordform || usingEmbedded) { + cordaJar.delete() + } } - private fun extractCordaJarTo(directory: Path): Path { - val cordaJarPath = directory / "corda.jar" - if (!cordaJarPath.exists()) { - Thread.currentThread().contextClassLoader.getResourceAsStream("corda.jar").use { it.copyTo(cordaJarPath) } + private fun gatherNodeDirectories(directory: Path): List { + return directory.list { paths -> + paths.filter { + val exists = (it / "corda.jar").exists() + if (exists) { + require((it / "node.conf").exists()) { "Missing node.conf in node directory ${it.fileName}" } + } + exists + }.toList() } - return cordaJarPath } private fun distributeNodeInfos(nodeDirs: List, nodeInfoFiles: List) { @@ -200,20 +275,13 @@ class NetworkBootstrapper { } } - /*the function checks for duplicate myLegalName in the all the *_node.conf files - All the myLegalName values are added to a HashSet - this helps detect duplicate values. - If a duplicate name is found the process is aborted with an error message - */ - private fun checkForDuplicateLegalNames(nodeInfoFiles: List) { - val legalNames = HashSet() - for (nodeInfoFile in nodeInfoFiles) { - val nodeConfig = ConfigFactory.parseFile((nodeInfoFile.parent / "node.conf").toFile()) - val legalName = nodeConfig.getString("myLegalName") - if(!legalNames.add(legalName)){ - println("Duplicate Node Found - ensure every node has a unique legal name"); - throw IllegalArgumentException("Duplicate Node Found - $legalName"); + private fun checkForDuplicateLegalNames(nodeConfigs: Collection) { + val duplicateLegalNames = nodeConfigs + .groupBy { it.getString("myLegalName") } + .mapNotNull { if (it.value.size > 1) it.key else null } + check(duplicateLegalNames.isEmpty()) { + "Nodes must have unique legal names. The following are used more than once: $duplicateLegalNames" } - } } private fun gatherNotaryInfos(nodeInfoFiles: List, configs: Map): List { @@ -264,15 +332,19 @@ class NetworkBootstrapper { whitelist: Map>, existingNetParams: NetworkParameters?, nodeDirs: List): NetworkParameters { - val networkParameters = if (existingNetParams != null) { - existingNetParams.copy( - notaries = notaryInfos, - modifiedTime = Instant.now(), - whitelistedContractImplementations = whitelist, - epoch = existingNetParams.epoch + 1 - ) + // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize + val netParams = if (existingNetParams != null) { + if (existingNetParams.whitelistedContractImplementations == whitelist && existingNetParams.notaries == notaryInfos) { + existingNetParams + } else { + existingNetParams.copy( + notaries = notaryInfos, + modifiedTime = Instant.now(), + whitelistedContractImplementations = whitelist, + epoch = existingNetParams.epoch + 1 + ) + } } else { - // TODO Add config for minimumPlatformVersion, maxMessageSize and maxTransactionSize NetworkParameters( minimumPlatformVersion = 1, notaries = notaryInfos, @@ -284,9 +356,9 @@ class NetworkBootstrapper { eventHorizon = 30.days ) } - val copier = NetworkParametersCopier(networkParameters, overwriteFile = true) + val copier = NetworkParametersCopier(netParams, overwriteFile = true) nodeDirs.forEach(copier::install) - return networkParameters + return netParams } private fun NodeInfo.notaryIdentity(): Party { @@ -301,7 +373,6 @@ class NetworkBootstrapper { } // We need to to set serialization env, because generation of parameters is run from Cordform. - // KryoServerSerializationScheme is not accessible from nodeapi. private fun initialiseSerialization() { _contextSerializationEnv.set(SerializationEnvironmentImpl( SerializationFactoryImpl().apply { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapperTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapperTest.kt index b57c177d61..04744c9376 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapperTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapperTest.kt @@ -1,141 +1,289 @@ package net.corda.nodeapi.internal.network -import net.corda.core.contracts.ContractClassName -import net.corda.core.crypto.SecureHash -import net.corda.core.node.services.AttachmentId -import net.corda.nodeapi.internal.ContractsJar -import net.corda.testing.common.internal.testNetworkParameters +import com.typesafe.config.ConfigFactory +import net.corda.cordform.CordformNode.NODE_INFO_DIRECTORY +import net.corda.core.crypto.secureRandomBytes +import net.corda.core.crypto.sha256 +import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.* +import net.corda.core.node.NetworkParameters +import net.corda.core.node.NodeInfo +import net.corda.core.serialization.serialize +import net.corda.node.services.config.NotaryConfig +import net.corda.nodeapi.internal.DEV_ROOT_CA +import net.corda.nodeapi.internal.SignedNodeInfo +import net.corda.nodeapi.internal.config.parseAs +import net.corda.nodeapi.internal.config.toConfig +import net.corda.nodeapi.internal.network.NodeInfoFilesCopier.Companion.NODE_INFO_FILE_NAME_PREFIX +import net.corda.testing.core.ALICE_NAME +import net.corda.testing.core.BOB_NAME +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.core.SerializationEnvironmentRule +import net.corda.testing.internal.createNodeInfoAndSigned import org.assertj.core.api.Assertions.assertThat -import org.assertj.core.api.Assertions.assertThatIllegalArgumentException +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.After +import org.junit.Rule import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.nio.file.Path +import kotlin.streams.toList class NetworkBootstrapperTest { - @Test - fun `no jars against empty whitelist`() { - val whitelist = generateWhitelist(emptyMap(), emptyList(), emptyList()) - assertThat(whitelist).isEmpty() - } + @Rule + @JvmField + val tempFolder = TemporaryFolder() - @Test - fun `no jars against single whitelist`() { - val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) - val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), emptyList()) - assertThat(newWhitelist).isEqualTo(existingWhitelist) - } + @Rule + @JvmField + val testSerialization = SerializationEnvironmentRule() - @Test - fun `empty jar against empty whitelist`() { - val whitelist = generateWhitelist(emptyMap(), emptyList(), listOf(TestContractsJar(contractClassNames = emptyList()))) - assertThat(whitelist).isEmpty() - } + private val fakeEmbeddedCordaJar = fakeFileBytes() - @Test - fun `empty jar against single whitelist`() { - val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) - val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(TestContractsJar(contractClassNames = emptyList()))) - assertThat(newWhitelist).isEqualTo(existingWhitelist) - } + private val contractsJars = HashMap() - @Test - fun `jar with single contract against empty whitelist`() { - val jar = TestContractsJar(contractClassNames = listOf("class1")) - val whitelist = generateWhitelist(emptyMap(), emptyList(), listOf(jar)) - assertThat(whitelist).isEqualTo(mapOf( - "class1" to listOf(jar.hash) - )) - } + private val bootstrapper = NetworkBootstrapper( + initSerEnv = false, + embeddedCordaJar = fakeEmbeddedCordaJar::inputStream, + nodeInfosGenerator = { nodeDirs -> + nodeDirs.map { nodeDir -> + val name = nodeDir.fakeNodeConfig.myLegalName + val file = nodeDir / "$NODE_INFO_FILE_NAME_PREFIX${name.serialize().hash}" + if (!file.exists()) { + createNodeInfoAndSigned(name).signed.serialize().open().copyTo(file) + } + file + } + }, + contractsJarConverter = { contractsJars[it]!! } + ) - @Test - fun `single contract jar against single whitelist of different contract`() { - val class1JarHash = SecureHash.randomSHA256() - val existingWhitelist = mapOf("class1" to listOf(class1JarHash)) - val jar = TestContractsJar(contractClassNames = listOf("class2")) - val whitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(jar)) - assertThat(whitelist).isEqualTo(mapOf( - "class1" to listOf(class1JarHash), - "class2" to listOf(jar.hash) - )) - } + private val aliceConfig = FakeNodeConfig(ALICE_NAME) + private val bobConfig = FakeNodeConfig(BOB_NAME) + private val notaryConfig = FakeNodeConfig(DUMMY_NOTARY_NAME, NotaryConfig(validating = true)) - @Test - fun `same jar with single contract`() { - val jarHash = SecureHash.randomSHA256() - val existingWhitelist = mapOf("class1" to listOf(jarHash)) - val jar = TestContractsJar(hash = jarHash, contractClassNames = listOf("class1")) - val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(jar)) - assertThat(newWhitelist).isEqualTo(existingWhitelist) - } + private var providedCordaJar: ByteArray? = null + private val configFiles = HashMap() - @Test - fun `jar with updated contract`() { - val previousJarHash = SecureHash.randomSHA256() - val existingWhitelist = mapOf("class1" to listOf(previousJarHash)) - val newContractsJar = TestContractsJar(contractClassNames = listOf("class1")) - val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(newContractsJar)) - assertThat(newWhitelist).isEqualTo(mapOf( - "class1" to listOf(previousJarHash, newContractsJar.hash) - )) - } - - @Test - fun `jar with one existing contract and one new one`() { - val previousJarHash = SecureHash.randomSHA256() - val existingWhitelist = mapOf("class1" to listOf(previousJarHash)) - val newContractsJar = TestContractsJar(contractClassNames = listOf("class1", "class2")) - val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(newContractsJar)) - assertThat(newWhitelist).isEqualTo(mapOf( - "class1" to listOf(previousJarHash, newContractsJar.hash), - "class2" to listOf(newContractsJar.hash) - )) - } - - @Test - fun `two versions of the same contract`() { - val version1Jar = TestContractsJar(contractClassNames = listOf("class1")) - val version2Jar = TestContractsJar(contractClassNames = listOf("class1")) - val newWhitelist = generateWhitelist(emptyMap(), emptyList(), listOf(version1Jar, version2Jar)) - assertThat(newWhitelist).isEqualTo(mapOf( - "class1" to listOf(version1Jar.hash, version2Jar.hash) - )) - } - - @Test - fun `jar with single new contract that's excluded`() { - val jar = TestContractsJar(contractClassNames = listOf("class1")) - val whitelist = generateWhitelist(emptyMap(), listOf("class1"), listOf(jar)) - assertThat(whitelist).isEmpty() - } - - @Test - fun `jar with two new contracts, one of which is excluded`() { - val jar = TestContractsJar(contractClassNames = listOf("class1", "class2")) - val whitelist = generateWhitelist(emptyMap(), listOf("class1"), listOf(jar)) - assertThat(whitelist).isEqualTo(mapOf( - "class2" to listOf(jar.hash) - )) - } - - @Test - fun `jar with updated contract but it's excluded`() { - val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) - val jar = TestContractsJar(contractClassNames = listOf("class1")) - assertThatIllegalArgumentException().isThrownBy { - generateWhitelist(existingWhitelist, listOf("class1"), listOf(jar)) + @After + fun `check config files are preserved`() { + configFiles.forEach { file, text -> + assertThat(file).hasContent(text) } } - private fun generateWhitelist(existingWhitelist: Map>, - excludeContracts: List, - contractJars: List): Map> { - return generateWhitelist( - testNetworkParameters(whitelistedContractImplementations = existingWhitelist), - excludeContracts, - contractJars - ) + @After + fun `check provided corda jar is preserved`() { + if (providedCordaJar == null) { + // Make sure we clean up if we used the embedded jar + assertThat(rootDir / "corda.jar").doesNotExist() + } else { + // Make sure we don't delete it if it was provided by the user + assertThat(rootDir / "corda.jar").hasBinaryContent(providedCordaJar) + } } - data class TestContractsJar(override val hash: SecureHash = SecureHash.randomSHA256(), - private val contractClassNames: List) : ContractsJar { - override fun scan(): List = contractClassNames + @Test + fun `empty dir`() { + assertThatThrownBy { + bootstrap() + }.hasMessage("No nodes found") } + + @Test + fun `single node conf file`() { + createNodeConfFile("node1", bobConfig) + bootstrap() + val networkParameters = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "node1" to bobConfig) + networkParameters.run { + assertThat(epoch).isEqualTo(1) + assertThat(notaries).isEmpty() + assertThat(whitelistedContractImplementations).isEmpty() + } + } + + @Test + fun `node conf file and corda jar`() { + createNodeConfFile("node1", bobConfig) + val fakeCordaJar = fakeFileBytes(rootDir / "corda.jar") + bootstrap() + assertBootstrappedNetwork(fakeCordaJar, "node1" to bobConfig) + } + + @Test + fun `single node directory with just node conf file`() { + createNodeDir("bob", bobConfig) + bootstrap() + assertBootstrappedNetwork(fakeEmbeddedCordaJar, "bob" to bobConfig) + } + + @Test + fun `single node directory with node conf file and corda jar`() { + val nodeDir = createNodeDir("bob", bobConfig) + val fakeCordaJar = fakeFileBytes(nodeDir / "corda.jar") + bootstrap() + assertBootstrappedNetwork(fakeCordaJar, "bob" to bobConfig) + } + + @Test + fun `single node directory with just corda jar`() { + val nodeCordaJar = (rootDir / "alice").createDirectories() / "corda.jar" + val fakeCordaJar = fakeFileBytes(nodeCordaJar) + assertThatThrownBy { + bootstrap() + }.hasMessageStartingWith("Missing node.conf in node directory alice") + assertThat(nodeCordaJar).hasBinaryContent(fakeCordaJar) // Make sure the corda.jar is left untouched + } + + @Test + fun `two node conf files, one of which is a notary`() { + createNodeConfFile("alice", aliceConfig) + createNodeConfFile("notary", notaryConfig) + bootstrap() + val networkParameters = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig, "notary" to notaryConfig) + networkParameters.assertContainsNotary("notary") + } + + @Test + fun `two node conf files with the same legal name`() { + createNodeConfFile("node1", aliceConfig) + createNodeConfFile("node2", aliceConfig) + assertThatThrownBy { + bootstrap() + }.hasMessageContaining("Nodes must have unique legal names") + } + + @Test + fun `one node directory and one node conf file`() { + createNodeConfFile("alice", aliceConfig) + createNodeDir("bob", bobConfig) + bootstrap() + assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig, "bob" to bobConfig) + } + + @Test + fun `node conf file and CorDapp jar`() { + createNodeConfFile("alice", aliceConfig) + val cordappBytes = createFakeCordappJar("sample-app", listOf("contract.class")) + bootstrap() + val networkParameters = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig) + assertThat(rootDir / "alice" / "cordapps" / "sample-app.jar").hasBinaryContent(cordappBytes) + assertThat(networkParameters.whitelistedContractImplementations).isEqualTo(mapOf( + "contract.class" to listOf(cordappBytes.sha256()) + )) + } + + @Test + fun `no copy CorDapps`() { + createNodeConfFile("alice", aliceConfig) + val cordappBytes = createFakeCordappJar("sample-app", listOf("contract.class")) + bootstrap(copyCordapps = false) + val networkParameters = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig) + assertThat(rootDir / "alice" / "cordapps" / "sample-app.jar").doesNotExist() + assertThat(networkParameters.whitelistedContractImplementations).isEqualTo(mapOf( + "contract.class" to listOf(cordappBytes.sha256()) + )) + } + + @Test + fun `add node to existing network`() { + createNodeConfFile("alice", aliceConfig) + bootstrap() + val networkParameters1 = (rootDir / "alice").networkParameters + createNodeConfFile("bob", bobConfig) + bootstrap() + val networkParameters2 = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig, "bob" to bobConfig) + assertThat(networkParameters1).isEqualTo(networkParameters2) + } + + @Test + fun `add notary to existing network`() { + createNodeConfFile("alice", aliceConfig) + bootstrap() + createNodeConfFile("notary", notaryConfig) + bootstrap() + val networkParameters = assertBootstrappedNetwork(fakeEmbeddedCordaJar, "alice" to aliceConfig, "notary" to notaryConfig) + networkParameters.assertContainsNotary("notary") + assertThat(networkParameters.epoch).isEqualTo(2) + } + + private val rootDir get() = tempFolder.root.toPath() + + private fun fakeFileBytes(writeToFile: Path? = null): ByteArray { + val bytes = secureRandomBytes(128) + writeToFile?.write(bytes) + return bytes + } + + private fun bootstrap(copyCordapps: Boolean = true) { + providedCordaJar = (rootDir / "corda.jar").let { if (it.exists()) it.readAll() else null } + bootstrapper.bootstrap(rootDir, copyCordapps) + } + + private fun createNodeConfFile(nodeDirName: String, config: FakeNodeConfig) { + writeNodeConfFile(rootDir / "${nodeDirName}_node.conf", config) + } + + private fun createNodeDir(nodeDirName: String, config: FakeNodeConfig): Path { + val nodeDir = (rootDir / nodeDirName).createDirectories() + writeNodeConfFile(nodeDir / "node.conf", config) + return nodeDir + } + + private fun writeNodeConfFile(file: Path, config: FakeNodeConfig) { + val configText = config.toConfig().root().render() + file.writeText(configText) + configFiles[file] = configText + } + + private fun createFakeCordappJar(cordappName: String, contractClassNames: List): ByteArray { + val cordappJarFile = rootDir / "$cordappName.jar" + val cordappBytes = fakeFileBytes(cordappJarFile) + contractsJars[cordappJarFile] = TestContractsJar(cordappBytes.sha256(), contractClassNames) + return cordappBytes + } + + private val Path.networkParameters: NetworkParameters get() { + return (this / NETWORK_PARAMS_FILE_NAME).readObject().verifiedNetworkMapCert(DEV_ROOT_CA.certificate) + } + + private val Path.nodeInfoFile: Path get() { + return list { it.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }.toList() }.single() + } + + private val Path.nodeInfo: NodeInfo get() = nodeInfoFile.readObject().verified() + + private val Path.fakeNodeConfig: FakeNodeConfig get() { + return ConfigFactory.parseFile((this / "node.conf").toFile()).parseAs(FakeNodeConfig::class) + } + + private fun assertBootstrappedNetwork(cordaJar: ByteArray, vararg nodes: Pair): NetworkParameters { + val networkParameters = (rootDir / nodes[0].first).networkParameters + val allNodeInfoFiles = nodes.map { (rootDir / it.first).nodeInfoFile }.associateBy({ it }, { it.readAll() }) + + for ((nodeDirName, config) in nodes) { + val nodeDir = rootDir / nodeDirName + assertThat(nodeDir / "corda.jar").hasBinaryContent(cordaJar) + assertThat(nodeDir.fakeNodeConfig).isEqualTo(config) + assertThat(nodeDir.networkParameters).isEqualTo(networkParameters) + // Make sure all the nodes have all of each others' node-info files + allNodeInfoFiles.forEach { nodeInfoFile, bytes -> + assertThat(nodeDir / NODE_INFO_DIRECTORY / nodeInfoFile.fileName.toString()).hasBinaryContent(bytes) + } + } + + return networkParameters + } + + private fun NetworkParameters.assertContainsNotary(dirName: String) { + val notaryParty = (rootDir / dirName).nodeInfo.legalIdentities.single() + assertThat(notaries).hasSize(1) + notaries[0].run { + assertThat(validating).isTrue() + assertThat(identity.name).isEqualTo(notaryParty.name) + assertThat(identity.owningKey).isEqualTo(notaryParty.owningKey) + } + } + + data class FakeNodeConfig(val myLegalName: CordaX500Name, val notary: NotaryConfig? = null) } diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/TestContractsJar.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/TestContractsJar.kt new file mode 100644 index 0000000000..ea1a4ffc88 --- /dev/null +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/TestContractsJar.kt @@ -0,0 +1,10 @@ +package net.corda.nodeapi.internal.network + +import net.corda.core.contracts.ContractClassName +import net.corda.core.crypto.SecureHash +import net.corda.nodeapi.internal.ContractsJar + +data class TestContractsJar(override val hash: SecureHash = SecureHash.randomSHA256(), + private val contractClassNames: List) : ContractsJar { + override fun scan(): List = contractClassNames +} diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/WhitelistGeneratorTest.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/WhitelistGeneratorTest.kt index 67f4bb9641..4bf6960c85 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/WhitelistGeneratorTest.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/network/WhitelistGeneratorTest.kt @@ -1,41 +1,135 @@ package net.corda.nodeapi.internal.network -import com.nhaarman.mockito_kotlin.mock -import com.nhaarman.mockito_kotlin.verify +import net.corda.core.contracts.ContractClassName import net.corda.core.crypto.SecureHash -import net.corda.nodeapi.internal.ContractsJar +import net.corda.core.node.services.AttachmentId +import net.corda.testing.common.internal.testNetworkParameters +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatIllegalArgumentException import org.junit.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue class WhitelistGeneratorTest { + @Test + fun `no jars against empty whitelist`() { + val whitelist = generateWhitelist(emptyMap(), emptyList(), emptyList()) + assertThat(whitelist).isEmpty() + } @Test - fun `whitelist generator builds the correct whitelist map`() { - // given - val jars = (0..9).map { - val index = it - mock { - val secureHash = SecureHash.randomSHA256() - on { scan() }.then { - listOf(index.toString()) - } - on { hash }.then { - secureHash - } - } - } + fun `no jars against single whitelist`() { + val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) + val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), emptyList()) + assertThat(newWhitelist).isEqualTo(existingWhitelist) + } - // when - val result = generateWhitelist(null, emptyList(), jars) + @Test + fun `empty jar against empty whitelist`() { + val whitelist = generateWhitelist(emptyMap(), emptyList(), listOf(TestContractsJar(contractClassNames = emptyList()))) + assertThat(whitelist).isEmpty() + } - // then - jars.forEachIndexed { index, item -> - verify(item).scan() - val attachmentIds = requireNotNull(result[index.toString()]) - assertEquals(1, attachmentIds.size) - assertTrue { attachmentIds.contains(item.hash) } + @Test + fun `empty jar against single whitelist`() { + val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) + val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(TestContractsJar(contractClassNames = emptyList()))) + assertThat(newWhitelist).isEqualTo(existingWhitelist) + } + + @Test + fun `jar with single contract against empty whitelist`() { + val jar = TestContractsJar(contractClassNames = listOf("class1")) + val whitelist = generateWhitelist(emptyMap(), emptyList(), listOf(jar)) + assertThat(whitelist).isEqualTo(mapOf( + "class1" to listOf(jar.hash) + )) + } + + @Test + fun `single contract jar against single whitelist of different contract`() { + val class1JarHash = SecureHash.randomSHA256() + val existingWhitelist = mapOf("class1" to listOf(class1JarHash)) + val jar = TestContractsJar(contractClassNames = listOf("class2")) + val whitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(jar)) + assertThat(whitelist).isEqualTo(mapOf( + "class1" to listOf(class1JarHash), + "class2" to listOf(jar.hash) + )) + } + + @Test + fun `same jar with single contract`() { + val jarHash = SecureHash.randomSHA256() + val existingWhitelist = mapOf("class1" to listOf(jarHash)) + val jar = TestContractsJar(hash = jarHash, contractClassNames = listOf("class1")) + val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(jar)) + assertThat(newWhitelist).isEqualTo(existingWhitelist) + } + + @Test + fun `jar with updated contract`() { + val previousJarHash = SecureHash.randomSHA256() + val existingWhitelist = mapOf("class1" to listOf(previousJarHash)) + val newContractsJar = TestContractsJar(contractClassNames = listOf("class1")) + val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(newContractsJar)) + assertThat(newWhitelist).isEqualTo(mapOf( + "class1" to listOf(previousJarHash, newContractsJar.hash) + )) + } + + @Test + fun `jar with one existing contract and one new one`() { + val previousJarHash = SecureHash.randomSHA256() + val existingWhitelist = mapOf("class1" to listOf(previousJarHash)) + val newContractsJar = TestContractsJar(contractClassNames = listOf("class1", "class2")) + val newWhitelist = generateWhitelist(existingWhitelist, emptyList(), listOf(newContractsJar)) + assertThat(newWhitelist).isEqualTo(mapOf( + "class1" to listOf(previousJarHash, newContractsJar.hash), + "class2" to listOf(newContractsJar.hash) + )) + } + + @Test + fun `two versions of the same contract`() { + val version1Jar = TestContractsJar(contractClassNames = listOf("class1")) + val version2Jar = TestContractsJar(contractClassNames = listOf("class1")) + val newWhitelist = generateWhitelist(emptyMap(), emptyList(), listOf(version1Jar, version2Jar)) + assertThat(newWhitelist).isEqualTo(mapOf( + "class1" to listOf(version1Jar.hash, version2Jar.hash) + )) + } + + @Test + fun `jar with single new contract that's excluded`() { + val jar = TestContractsJar(contractClassNames = listOf("class1")) + val whitelist = generateWhitelist(emptyMap(), listOf("class1"), listOf(jar)) + assertThat(whitelist).isEmpty() + } + + @Test + fun `jar with two new contracts, one of which is excluded`() { + val jar = TestContractsJar(contractClassNames = listOf("class1", "class2")) + val whitelist = generateWhitelist(emptyMap(), listOf("class1"), listOf(jar)) + assertThat(whitelist).isEqualTo(mapOf( + "class2" to listOf(jar.hash) + )) + } + + @Test + fun `jar with updated contract but it's excluded`() { + val existingWhitelist = mapOf("class1" to listOf(SecureHash.randomSHA256())) + val jar = TestContractsJar(contractClassNames = listOf("class1")) + assertThatIllegalArgumentException().isThrownBy { + generateWhitelist(existingWhitelist, listOf("class1"), listOf(jar)) } } -} \ No newline at end of file + private fun generateWhitelist(existingWhitelist: Map>, + excludeContracts: List, + contractJars: List): Map> { + return generateWhitelist( + testNetworkParameters(whitelistedContractImplementations = existingWhitelist), + excludeContracts, + contractJars + ) + } +} diff --git a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt index 9ede8f9af9..c2e326f048 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt @@ -272,7 +272,7 @@ open class NodeStartup(val args: Array) { logger.info("Revision: ${versionInfo.revision}") val info = ManagementFactory.getRuntimeMXBean() logger.info("PID: ${info.name.split("@").firstOrNull()}") // TODO Java 9 has better support for this - logger.info("Main class: ${NodeConfiguration::class.java.protectionDomain.codeSource.location.toURI().path}") + logger.info("Main class: ${NodeConfiguration::class.java.location.toURI().path}") logger.info("CommandLine Args: ${info.inputArguments.joinToString(" ")}") logger.info("Application Args: ${args.joinToString(" ")}") logger.info("bootclasspath: ${info.bootClassPath}") diff --git a/node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt b/node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt index 109063eba4..288eb74ddc 100644 --- a/node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt +++ b/node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt @@ -204,7 +204,7 @@ class CordappLoader private constructor(private val cordappJarPaths: List { return ServiceLoader.load(SerializationWhitelist::class.java, URLClassLoader(arrayOf(cordappJarPath.url), appClassLoader)).toList().filter { - it.javaClass.protectionDomain.codeSource.location == cordappJarPath.url && it.javaClass.name.startsWith(cordappJarPath.qualifiedNamePrefix) + it.javaClass.location == cordappJarPath.url && it.javaClass.name.startsWith(cordappJarPath.qualifiedNamePrefix) } + DefaultWhitelist // Always add the DefaultWhitelist to the whitelist for an app. } diff --git a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt index d650a8c565..b38ca474ab 100644 --- a/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt +++ b/node/src/main/kotlin/net/corda/node/services/statemachine/FlowStateMachineImpl.kt @@ -436,7 +436,7 @@ val Class>.flowVersionAndInitiatingClass: Pair>.appName: String get() { - val jarFile = protectionDomain.codeSource.location.toPath() + val jarFile = location.toPath() return if (jarFile.isRegularFile() && jarFile.toString().endsWith(".jar")) { jarFile.fileName.toString().removeSuffix(".jar") } else { diff --git a/tools/blobinspector/build.gradle b/tools/blobinspector/build.gradle index c20270831d..26a15d84dc 100644 --- a/tools/blobinspector/build.gradle +++ b/tools/blobinspector/build.gradle @@ -5,7 +5,7 @@ apply plugin: 'com.jfrog.artifactory' dependencies { compile project(':client:jackson') - compile 'info.picocli:picocli:3.0.0' + compile "info.picocli:picocli:$picocli_version" compile "org.slf4j:jul-to-slf4j:$slf4j_version" compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version" compile "com.jcabi:jcabi-manifests:$jcabi_manifests_version" diff --git a/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt b/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt index c65b02c6f8..31368eb228 100644 --- a/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt +++ b/tools/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt @@ -41,7 +41,7 @@ fun main(args: Array) { @Command( name = "Blob Inspector", - versionProvider = VersionProvider::class, + versionProvider = CordaVersionProvider::class, mixinStandardHelpOptions = true, // add --help and --version options, showDefaultValues = true, description = ["Inspect AMQP serialised binary blobs"] @@ -64,7 +64,9 @@ class Main : Runnable { var verbose: Boolean = false override fun run() { - System.setProperty("logLevel", if (verbose) "trace" else "off") + if (verbose) { + System.setProperty("logLevel", "trace") + } val bytes = source!!.readBytes().run { require(size > amqpMagic.size) { "Insufficient bytes for AMQP blob" } @@ -124,8 +126,13 @@ private class SourceConverter : ITypeConverter { } } -private class VersionProvider : IVersionProvider { - override fun getVersion(): Array = arrayOf(Manifests.read("Corda-Release-Version")) +private class CordaVersionProvider : IVersionProvider { + override fun getVersion(): Array { + return arrayOf( + "Version: ${Manifests.read("Corda-Release-Version")}", + "Revision: ${Manifests.read("Corda-Revision")}" + ) + } } private enum class FormatType { YAML, JSON } diff --git a/tools/blobinspector/src/main/resources/log4j2.xml b/tools/blobinspector/src/main/resources/log4j2.xml index a9885efca9..98b3648e6b 100644 --- a/tools/blobinspector/src/main/resources/log4j2.xml +++ b/tools/blobinspector/src/main/resources/log4j2.xml @@ -1,5 +1,8 @@ + + off + diff --git a/tools/bootstrapper/build.gradle b/tools/bootstrapper/build.gradle index 04962dbe2f..19ad5c3adf 100644 --- a/tools/bootstrapper/build.gradle +++ b/tools/bootstrapper/build.gradle @@ -1,51 +1,36 @@ -apply plugin: 'us.kirchmeier.capsule' +apply plugin: 'java' +apply plugin: 'kotlin' apply plugin: 'net.corda.plugins.publish-utils' apply plugin: 'com.jfrog.artifactory' description 'Network bootstrapper' -configurations { - runtimeArtifacts +dependencies { + compile project(':node-api') + compile "info.picocli:picocli:$picocli_version" + compile "org.slf4j:jul-to-slf4j:$slf4j_version" + compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version" + compile "com.jcabi:jcabi-manifests:$jcabi_manifests_version" } jar { - baseName "corda-tools-network-bootstrapper" -} - -dependencies { - compile "org.slf4j:slf4j-nop:$slf4j_version" -} - -task buildBootstrapperJar(type: FatCapsule, dependsOn: project(':node-api').compileJava) { - applicationClass 'net.corda.nodeapi.internal.network.NetworkBootstrapper' - archiveName "tools-network-bootstrapper-${corda_release_version}.jar" - capsuleManifest { - applicationVersion = corda_release_version - systemProperties['visualvm.display.name'] = 'Network Bootstrapper' - minJavaVersion = '1.8.0' - jvmArgs = ['-XX:+UseG1GC'] + from(configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }) { + exclude "META-INF/*.SF" + exclude "META-INF/*.DSA" + exclude "META-INF/*.RSA" } from(project(':node:capsule').tasks['buildCordaJAR']) { rename 'corda-(.*)', 'corda.jar' } - applicationSource = files( - project(':node-api').configurations.runtime, - project(':node-api').jar - ) -} - -artifacts { - runtimeArtifacts buildBootstrapperJar - publish buildBootstrapperJar { - classifier "" + archiveName = "network-bootstrapper-${corda_release_version}.jar" + manifest { + attributes( + 'Automatic-Module-Name': 'net.corda.bootstrapper', + 'Main-Class': 'net.corda.bootstrapper.MainKt' + ) } } -jar { - classifier "ignore" -} - publish { - disableDefaultJar = true name 'corda-tools-network-bootstrapper' } diff --git a/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt b/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt new file mode 100644 index 0000000000..f80699ee65 --- /dev/null +++ b/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt @@ -0,0 +1,65 @@ +package net.corda.bootstrapper + +import com.jcabi.manifests.Manifests +import net.corda.core.internal.rootMessage +import net.corda.nodeapi.internal.network.NetworkBootstrapper +import picocli.CommandLine +import picocli.CommandLine.* +import java.nio.file.Path +import java.nio.file.Paths +import kotlin.system.exitProcess + +fun main(args: Array) { + val main = Main() + try { + CommandLine.run(main, *args) + } catch (e: ExecutionException) { + val throwable = e.cause ?: e + if (main.verbose) { + throwable.printStackTrace() + } else { + System.err.println("*ERROR*: ${throwable.rootMessage ?: "Use --verbose for more details"}") + } + exitProcess(1) + } +} + +@Command( + name = "Network Bootstrapper", + versionProvider = CordaVersionProvider::class, + mixinStandardHelpOptions = true, + showDefaultValues = true, + description = [ "Bootstrap a local test Corda network using a set of node conf files and CorDapp JARs" ] +) +class Main : Runnable { + @Option( + names = ["--dir"], + description = [ + "Root directory containing the node conf files and CorDapp JARs that will form the test network.", + "It may also contain existing node directories." + ] + ) + private var dir: Path = Paths.get(".") + + @Option(names = ["--no-copy"], description = ["""Don't copy the CorDapp JARs into the nodes' "cordapps" directories."""]) + private var noCopy: Boolean = false + + @Option(names = ["--verbose"], description = ["Enable verbose output."]) + var verbose: Boolean = false + + override fun run() { + if (verbose) { + System.setProperty("logLevel", "trace") + } + NetworkBootstrapper().bootstrap(dir.toAbsolutePath().normalize(), copyCordapps = !noCopy) + } +} + +private class CordaVersionProvider : IVersionProvider { + override fun getVersion(): Array { + return arrayOf( + "Version: ${Manifests.read("Corda-Release-Version")}", + "Revision: ${Manifests.read("Corda-Revision")}" + ) + } +} diff --git a/tools/bootstrapper/src/main/resources/log4j2.xml b/tools/bootstrapper/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..98b3648e6b --- /dev/null +++ b/tools/bootstrapper/src/main/resources/log4j2.xml @@ -0,0 +1,16 @@ + + + + off + + + + + + + + + + + + \ No newline at end of file diff --git a/webserver/src/main/kotlin/net/corda/webserver/WebServer.kt b/webserver/src/main/kotlin/net/corda/webserver/WebServer.kt index 44dc37ad3a..3c0f2ec00a 100644 --- a/webserver/src/main/kotlin/net/corda/webserver/WebServer.kt +++ b/webserver/src/main/kotlin/net/corda/webserver/WebServer.kt @@ -4,6 +4,7 @@ package net.corda.webserver import com.typesafe.config.ConfigException import net.corda.core.internal.div +import net.corda.core.internal.location import net.corda.core.internal.rootCause import net.corda.webserver.internal.NodeWebServer import org.slf4j.LoggerFactory @@ -48,7 +49,7 @@ fun main(args: Array) { exitProcess(2) } - log.info("Main class: ${WebServerConfig::class.java.protectionDomain.codeSource.location.toURI().path}") + log.info("Main class: ${WebServerConfig::class.java.location.toURI().path}") val info = ManagementFactory.getRuntimeMXBean() log.info("CommandLine Args: ${info.inputArguments.joinToString(" ")}") log.info("Application Args: ${args.joinToString(" ")}")