From d5f43704430789eca9ada90d44ab39429ff55b55 Mon Sep 17 00:00:00 2001 From: Anthony Keenan Date: Wed, 1 Aug 2018 14:44:56 +0100 Subject: [PATCH 1/6] CORDA-1848 Add example alias and autocomplete for CLI tools (#3700) * Basic alias and autocomplete installation for bash in network bootstrapper * Address review comments * Update completion file if out of date * Refactoring * Some more minor tweaks * Use manifest revision rather than recalculating hash * Add zsh autocomplete compatibility * Actually write .zshrc file * Fix some descriptions * Only rewrite settings files if changes have been made, and make a backup if so. Some refactoring --- .../kotlin/net/corda/bootstrapper/Main.kt | 132 +++++++++++++++++- 1 file changed, 127 insertions(+), 5 deletions(-) diff --git a/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt b/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt index f80699ee65..f89750246f 100644 --- a/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt +++ b/tools/bootstrapper/src/main/kotlin/net/corda/bootstrapper/Main.kt @@ -1,12 +1,13 @@ package net.corda.bootstrapper import com.jcabi.manifests.Manifests -import net.corda.core.internal.rootMessage +import net.corda.core.internal.* import net.corda.nodeapi.internal.network.NetworkBootstrapper import picocli.CommandLine import picocli.CommandLine.* import java.nio.file.Path import java.nio.file.Paths +import java.nio.file.StandardCopyOption.REPLACE_EXISTING import kotlin.system.exitProcess fun main(args: Array) { @@ -29,13 +30,13 @@ fun main(args: Array) { versionProvider = CordaVersionProvider::class, mixinStandardHelpOptions = true, showDefaultValues = true, - description = [ "Bootstrap a local test Corda network using a set of node conf files and CorDapp JARs" ] + description = ["Bootstrap a local test Corda network using a set of node configuration files and CorDapp JARs"] ) class Main : Runnable { @Option( names = ["--dir"], description = [ - "Root directory containing the node conf files and CorDapp JARs that will form the test network.", + "Root directory containing the node configuration files and CorDapp JARs that will form the test network.", "It may also contain existing node directories." ] ) @@ -47,7 +48,123 @@ class Main : Runnable { @Option(names = ["--verbose"], description = ["Enable verbose output."]) var verbose: Boolean = false + @Option(names = ["--install-shell-extensions"], description = ["Install bootstrapper alias and autocompletion for bash and zsh"]) + var installShellExtensions: Boolean = false + + private class SettingsFile(val filePath: Path) { + private val lines: MutableList by lazy { getFileLines() } + var fileModified: Boolean = false + + // Return the lines in the file if it exists, else return an empty mutable list + private fun getFileLines(): MutableList { + return if (filePath.exists()) { + filePath.toFile().readLines().toMutableList() + } else { + emptyList().toMutableList() + } + } + + fun addOrReplaceIfStartsWith(startsWith: String, replaceWith: String) { + val index = lines.indexOfFirst { it.startsWith(startsWith) } + if (index >= 0) { + if (lines[index] != replaceWith) { + lines[index] = replaceWith + fileModified = true + } + } else { + lines.add(replaceWith) + fileModified = true + } + } + + fun addIfNotExists(line: String) { + if (!lines.contains(line)) { + lines.add(line) + fileModified = true + } + } + + fun updateAndBackupIfNecessary() { + if (fileModified) { + val backupFilePath = filePath.parent / "${filePath.fileName}.backup" + println("Updating settings in ${filePath.fileName} - existing settings file has been backed up to $backupFilePath") + if (filePath.exists()) filePath.copyTo(backupFilePath, REPLACE_EXISTING) + filePath.writeLines(lines) + } + } + } + + private val userHome: Path by lazy { Paths.get(System.getProperty("user.home")) } + private val jarLocation: Path by lazy { this.javaClass.location.toPath() } + + // If on Windows, Path.toString() returns a path with \ instead of /, but for bash Windows users we want to convert those back to /'s + private fun Path.toStringWithDeWindowsfication(): String = this.toAbsolutePath().toString().replace("\\", "/") + private fun jarVersion(alias: String) = "# $alias - Version: ${CordaVersionProvider.releaseVersion}, Revision: ${CordaVersionProvider.revision}" + private fun getAutoCompleteFileLocation(alias: String) = userHome / ".completion" / alias + + private fun generateAutoCompleteFile(alias: String) { + println("Generating $alias auto completion file") + val autoCompleteFile = getAutoCompleteFileLocation(alias) + autoCompleteFile.parent.createDirectories() + picocli.AutoComplete.main("-f", "-n", alias, this.javaClass.name, "-o", autoCompleteFile.toStringWithDeWindowsfication()) + + // Append hash of file to autocomplete file + autoCompleteFile.toFile().appendText(jarVersion(alias)) + } + + private fun installShellExtensions(alias: String) { + // Get jar location and generate alias command + val command = "alias $alias='java -jar \"${jarLocation.toStringWithDeWindowsfication()}\"'" + generateAutoCompleteFile(alias) + + // Get bash settings file + val bashSettingsFile = SettingsFile(userHome / ".bashrc") + // Replace any existing bootstrapper alias. There can be only one. + bashSettingsFile.addOrReplaceIfStartsWith("alias $alias", command) + val completionFileCommand = "for bcfile in ~/.completion/* ; do . \$bcfile; done" + bashSettingsFile.addIfNotExists(completionFileCommand) + bashSettingsFile.updateAndBackupIfNecessary() + + // Get zsh settings file + val zshSettingsFile = SettingsFile(userHome / ".zshrc") + zshSettingsFile.addIfNotExists("autoload -U +X compinit && compinit") + zshSettingsFile.addIfNotExists("autoload -U +X bashcompinit && bashcompinit") + zshSettingsFile.addOrReplaceIfStartsWith("alias $alias", command) + zshSettingsFile.addIfNotExists(completionFileCommand) + zshSettingsFile.updateAndBackupIfNecessary() + + println("Installation complete, $alias is available in bash with autocompletion. ") + println("Type `$alias ` from the commandline.") + println("Restart bash for this to take effect, or run `. ~/.bashrc` in bash or `. ~/.zshrc` in zsh to re-initialise your shell now") + } + + private fun checkForAutoCompleteUpdate(alias: String) { + val autoCompleteFile = getAutoCompleteFileLocation(alias) + + // If no autocomplete file, it hasn't been installed, so don't do anything + if (!autoCompleteFile.exists()) return + + var lastLine = "" + autoCompleteFile.toFile().forEachLine { lastLine = it } + + if (lastLine != jarVersion(alias)) { + println("Old auto completion file detected... regenerating") + generateAutoCompleteFile(alias) + println("Restart bash for this to take effect, or run `. ~/.bashrc` to re-initialise bash now") + } + } + + private fun installOrUpdateShellExtensions(alias: String) { + if (installShellExtensions) { + installShellExtensions(alias) + exitProcess(0) + } else { + checkForAutoCompleteUpdate(alias) + } + } + override fun run() { + installOrUpdateShellExtensions("bootstrapper") if (verbose) { System.setProperty("logLevel", "trace") } @@ -56,10 +173,15 @@ class Main : Runnable { } private class CordaVersionProvider : IVersionProvider { + companion object { + val releaseVersion: String by lazy { Manifests.read("Corda-Release-Version") } + val revision: String by lazy { Manifests.read("Corda-Revision") } + } + override fun getVersion(): Array { return arrayOf( - "Version: ${Manifests.read("Corda-Release-Version")}", - "Revision: ${Manifests.read("Corda-Revision")}" + "Version: $releaseVersion", + "Revision: $revision" ) } } From 2e03c7f8c021171672d1235a90ebea4955b2acae Mon Sep 17 00:00:00 2001 From: Joel Dudley Date: Wed, 1 Aug 2018 17:55:00 +0100 Subject: [PATCH 2/6] Clarifies Gradle wrapper usage. Clarifies one module = one CorDapp. (#3721) * Clarifies Gradle wrapper usage. Clarifies one module = one CorDapp. * Clean-up. Note on how to create nodes and CorDapps in prod. * Fixes based on rendering. * Addresses review comments. --- docs/source/cordapp-build-systems.rst | 33 +++++++++++++------- docs/source/tutorial-cordapp.rst | 44 +++++++++++++++++---------- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/docs/source/cordapp-build-systems.rst b/docs/source/cordapp-build-systems.rst index 48a1affbde..29b2a351ab 100644 --- a/docs/source/cordapp-build-systems.rst +++ b/docs/source/cordapp-build-systems.rst @@ -19,8 +19,18 @@ JAR will contain: Build tools ----------- -In the instructions that follow, we assume you are using ``gradle`` and the ``cordformation`` plugin to build your -CorDapp. You can find examples of building a CorDapp using these tools in the ``build.gradle`` file of the `Kotlin CorDapp Template `_ and the `Java CorDapp Template `_. +In the instructions that follow, we assume you are using Gradle and the ``cordformation`` plugin to build your +CorDapp. You can find examples of building a CorDapp using these tools in the +`Kotlin CorDapp Template `_ and the +`Java CorDapp Template `_. + +To ensure you are using the correct version of Gradle, you should use the provided Gradle Wrapper by copying across +the following folder and files from the `Kotlin CorDapp Template `_ or the +`Java CorDapp Template `_ to the root of your project: + +* ``gradle/`` +* ``gradlew`` +* ``gradlew.bat`` Setting your dependencies ------------------------- @@ -101,7 +111,10 @@ For further information about managing dependencies, see Example ^^^^^^^ -Below is a sample of what a CorDapp's Gradle dependencies block might look like. When building your own CorDapp, you should base yourself on the ``build.gradle`` file of the `Kotlin CorDapp Template `_ and the `Java CorDapp Template `_. +Below is a sample of what a CorDapp's Gradle dependencies block might look like. When building your own CorDapp, you should +base yourself on the ``build.gradle`` file of the +`Kotlin CorDapp Template `_ or the +`Java CorDapp Template `_. .. container:: codeset @@ -135,13 +148,14 @@ Below is a sample of what a CorDapp's Gradle dependencies block might look like. Creating the CorDapp JAR ------------------------ -Once your dependencies are set correctly, you can build your CorDapp JAR using the gradle ``jar`` task: +Once your dependencies are set correctly, you can build your CorDapp JAR(s) using the Gradle ``jar`` task * Unix/Mac OSX: ``./gradlew jar`` * Windows: ``gradlew.bat jar`` -The CorDapp JAR will be output to the ``build/libs`` folder. +Each of the project's modules will be compiled into its own CorDapp JAR. You can find these CorDapp JARs in the ``build/libs`` +folders of each of the project's modules. .. warning:: The hash of the generated CorDapp JAR is not deterministic, as it depends on variables such as the timestamp at creation. Nodes running the same CorDapp must therefore ensure they are using the exact same CorDapp @@ -158,9 +172,9 @@ Installing the CorDapp JAR .. note:: Before installing a CorDapp, you must create one or more nodes to install it on. For instructions, please see :doc:`generating-a-node`. -At start-up, nodes will load any CorDapps present in their ``cordapps`` folder. Therefore, in order to install a CorDapp on -a node, the CorDapp JAR must be added to the ``/cordapps/`` folder (where ``node_dir`` is the folder in which -the node's JAR and configuration files are stored) and the node restarted. +At start-up, nodes will load any CorDapps present in their ``cordapps`` folder. In order to install a CorDapp on a node, the +CorDapp JAR must be added to the ``/cordapps/`` folder (where ``node_dir`` is the folder in which the node's JAR +and configuration files are stored) and the node restarted. CorDapp configuration files --------------------------- @@ -175,6 +189,3 @@ CorDapp configuration can be accessed from ``CordappContext::config`` whenever a There is an example project that demonstrates in ``samples` called ``cordapp-configuration`` and API documentation in `_. - - - diff --git a/docs/source/tutorial-cordapp.rst b/docs/source/tutorial-cordapp.rst index a2b7dfe118..7840dc6e6a 100644 --- a/docs/source/tutorial-cordapp.rst +++ b/docs/source/tutorial-cordapp.rst @@ -165,33 +165,45 @@ Building the example CorDapp ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * Open a terminal window in the ``cordapp-example`` directory -* Build the test nodes with our CorDapp using the following command: +* Run the ``deployNodes`` Gradle task to build four nodes with our CorDapp already installed on them: * Unix/Mac OSX: ``./gradlew deployNodes`` * Windows: ``gradlew.bat deployNodes`` - This will automatically build four nodes with our CorDapp already installed - .. note:: CorDapps can be written in any language targeting the JVM. In our case, we've provided the example source in both Kotlin (``/kotlin-source/src``) and Java (``/java-source/src``). Since both sets of source files are functionally identical, we will refer to the Kotlin version throughout the documentation. -* After the build finishes, you will see the generated nodes in the ``kotlin-source/build/nodes`` folder +* After the build finishes, you will see the following output in the ``kotlin-source/build/nodes`` folder: - * There will be a folder for each generated node, plus a ``runnodes`` shell script (or batch file on Windows) to run - all the nodes simultaneously + * A folder for each generated node + * A ``runnodes`` shell script for running all the nodes simultaneously on osX + * A ``runnodes.bat`` batch file for running all the nodes simultaneously on Windows - * Each node in the ``nodes`` folder has the following structure: +* Each node in the ``nodes`` folder will have the following structure: - .. sourcecode:: none - - . nodeName - ├── corda.jar // The Corda node runtime. - ├── corda-webserver.jar // The node development webserver. - ├── node.conf // The node configuration file. - └── cordapps // The node's CorDapps. + .. sourcecode:: none + + . nodeName + ├── additional-node-infos // + ├── certificates + ├── corda.jar // The Corda node runtime + ├── corda-webserver.jar // The development node webserver runtime + ├── cordapps // The node's CorDapps + │   ├── corda-finance-3.2-corda.jar + │   └── cordapp-example-0.1.jar + ├── drivers + ├── logs + ├── network-parameters + ├── node.conf // The node's configuration file + ├── nodeInfo- // The hash will be different each time you generate a node + └── persistence.mv.db // The node's database +.. note:: ``deployNodes`` is a utility task to create an entirely new set of nodes for testing your CorDapp. In production, + you would instead create a single node as described in :doc:`generating-a-node` and build your CorDapp JARs as described + in :doc:`cordapp-build-systems`. + Running the example CorDapp ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Start the nodes by running the following command from the root of the ``cordapp-example`` folder: @@ -440,8 +452,8 @@ The nodes can be configured to communicate as a network even when distributed ac are distributed across machines. Otherwise, the nodes will not be able to communicate. .. note:: If you are using H2 and wish to use the same ``h2port`` value for two or more nodes, you must only assign them that - value after the nodes have been moved to their individual machines. The initial bootstrapping process requires access to the - nodes' databases and if two nodes share the same H2 port, the process will fail. + value after the nodes have been moved to their individual machines. The initial bootstrapping process requires access to + the nodes' databases and if two nodes share the same H2 port, the process will fail. Testing your CorDapp -------------------- From 6499a139515f59c590056864c63ef7bb73b3b7e8 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Wed, 1 Aug 2018 18:11:42 +0100 Subject: [PATCH 3/6] CORDA-1865 Fix race on inserting attachments (#3734) * CORDA-1865 Fix race on inserting attachments * CORDA-1865 Fix race on inserting attachments --- .../kotlin/net/corda/core/internal/FetchDataFlow.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt b/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt index 1feb959261..5fa61aff96 100644 --- a/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt +++ b/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt @@ -18,6 +18,7 @@ import net.corda.core.transactions.SignedTransaction import net.corda.core.utilities.NonEmptySet import net.corda.core.utilities.UntrustworthyData import net.corda.core.utilities.unwrap +import java.nio.file.FileAlreadyExistsException import java.util.* /** @@ -149,9 +150,15 @@ class FetchAttachmentsFlow(requests: Set, for (attachment in downloaded) { with(serviceHub.attachments) { if (!hasAttachment(attachment.id)) { - importAttachment(attachment.open(), "$P2P_UPLOADER:${otherSideSession.counterparty.name}", null) + try { + importAttachment(attachment.open(), "$P2P_UPLOADER:${otherSideSession.counterparty.name}", null) + } catch (e: FileAlreadyExistsException) { + // This can happen when another transaction will insert the same attachment during this transaction. + // The outcome is the same (the attachment is imported), so we can ignore this exception. + logger.debug("Attachment ${attachment.id} already inserted.") + } } else { - logger.info("Attachment ${attachment.id} already exists, skipping.") + logger.debug("Attachment ${attachment.id} already exists, skipping.") } } } From 56c00675407c4ebe9b32e0c93340d711c4b001cc Mon Sep 17 00:00:00 2001 From: Jonathan Sphar Date: Wed, 1 Aug 2018 18:06:43 -0400 Subject: [PATCH 4/6] Added jvmoverloads annotation to SignTxnFlow constructor (#3735) --- .../main/kotlin/net/corda/core/flows/CollectSignaturesFlow.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/net/corda/core/flows/CollectSignaturesFlow.kt b/core/src/main/kotlin/net/corda/core/flows/CollectSignaturesFlow.kt index 509f96409e..7e0d98efb4 100644 --- a/core/src/main/kotlin/net/corda/core/flows/CollectSignaturesFlow.kt +++ b/core/src/main/kotlin/net/corda/core/flows/CollectSignaturesFlow.kt @@ -195,7 +195,7 @@ class CollectSignatureFlow(val partiallySignedTx: SignedTransaction, val session * * @param otherSideSession The session which is providing you a transaction to sign. */ -abstract class SignTransactionFlow(val otherSideSession: FlowSession, +abstract class SignTransactionFlow @JvmOverloads constructor(val otherSideSession: FlowSession, override val progressTracker: ProgressTracker = SignTransactionFlow.tracker()) : FlowLogic() { companion object { From 055ba90e0d19c24e3623aa26a294598f94d7dc4a Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Thu, 2 Aug 2018 08:26:58 +0100 Subject: [PATCH 5/6] CORDA-1724 Add tx access right validity check during resolution (#3732) * CORDA-1724 Add tx access right validity check during resolution * CORDA-1724 Fix missing payload type * CORDA-1724 Fix test * CORDA-1724 Add test * CORDA-1865 Improve comments * CORDA-1724 Address code review comments --- .../corda/core/flows/SendTransactionFlow.kt | 60 +++++++++++++- .../net/corda/core/internal/FetchDataFlow.kt | 10 ++- .../core/internal/ResolveTransactionsFlow.kt | 2 +- .../net/corda/core/flows/AttachmentTests.kt | 2 +- ...ow.kt => TestNoSecurityDataVendingFlow.kt} | 2 +- .../internal/ResolveTransactionsFlowTest.kt | 82 ++++++++++++++++++- .../AttachmentSerializationTest.kt | 4 +- 7 files changed, 148 insertions(+), 14 deletions(-) rename core/src/test/kotlin/net/corda/core/flows/{TestDataVendingFlow.kt => TestNoSecurityDataVendingFlow.kt} (83%) diff --git a/core/src/main/kotlin/net/corda/core/flows/SendTransactionFlow.kt b/core/src/main/kotlin/net/corda/core/flows/SendTransactionFlow.kt index 46271ab3c3..d9a27cbaae 100644 --- a/core/src/main/kotlin/net/corda/core/flows/SendTransactionFlow.kt +++ b/core/src/main/kotlin/net/corda/core/flows/SendTransactionFlow.kt @@ -2,8 +2,10 @@ package net.corda.core.flows import co.paralleluniverse.fibers.Suspendable import net.corda.core.contracts.StateAndRef +import net.corda.core.crypto.SecureHash import net.corda.core.internal.FetchDataFlow import net.corda.core.internal.readFully +import net.corda.core.serialization.CordaSerializable import net.corda.core.transactions.SignedTransaction import net.corda.core.utilities.unwrap @@ -42,6 +44,25 @@ open class DataVendingFlow(val otherSideSession: FlowSession, val payload: Any) override fun call(): Void? { // The first payload will be the transaction data, subsequent payload will be the transaction/attachment data. var payload = payload + + // Depending on who called this flow, the type of the initial payload is different. + // The authorisation logic is to maintain a dynamic list of transactions that the caller is authorised to make based on the transactions that were made already. + // Each time an authorised transaction is requested, the input transactions are added to the list. + // Once a transaction has been requested, it will be removed from the authorised list. This means that it is a protocol violation to request a transaction twice. + val authorisedTransactions = when (payload) { + is NotarisationPayload -> TransactionAuthorisationFilter().addAuthorised(getInputTransactions(payload.signedTransaction)) + is SignedTransaction -> TransactionAuthorisationFilter().addAuthorised(getInputTransactions(payload)) + is RetrieveAnyTransactionPayload -> TransactionAuthorisationFilter(acceptAll = true) + is List<*> -> TransactionAuthorisationFilter().addAuthorised(payload.flatMap { stateAndRef -> + if (stateAndRef is StateAndRef<*>) { + getInputTransactions(serviceHub.validatedTransactions.getTransaction(stateAndRef.ref.txhash)!!) + stateAndRef.ref.txhash + } else { + throw Exception("Unknown payload type: ${stateAndRef!!::class.java} ?") + } + }.toSet()) + else -> throw Exception("Unknown payload type: ${payload::class.java} ?") + } + // This loop will receive [FetchDataFlow.Request] continuously until the `otherSideSession` has all the data they need // to resolve the transaction, a [FetchDataFlow.EndRequest] will be sent from the `otherSideSession` to indicate end of // data request. @@ -56,14 +77,47 @@ open class DataVendingFlow(val otherSideSession: FlowSession, val payload: Any) FetchDataFlow.Request.End -> return null } } + payload = when (dataRequest.dataType) { - FetchDataFlow.DataType.TRANSACTION -> dataRequest.hashes.map { - serviceHub.validatedTransactions.getTransaction(it) ?: throw FetchDataFlow.HashNotFound(it) + FetchDataFlow.DataType.TRANSACTION -> dataRequest.hashes.map { txId -> + if (!authorisedTransactions.isAuthorised(txId)) { + throw FetchDataFlow.IllegalTransactionRequest(txId) + } + val tx = serviceHub.validatedTransactions.getTransaction(txId) + ?: throw FetchDataFlow.HashNotFound(txId) + authorisedTransactions.removeAuthorised(tx.id) + authorisedTransactions.addAuthorised(getInputTransactions(tx)) + tx } FetchDataFlow.DataType.ATTACHMENT -> dataRequest.hashes.map { - serviceHub.attachments.openAttachment(it)?.open()?.readFully() ?: throw FetchDataFlow.HashNotFound(it) + serviceHub.attachments.openAttachment(it)?.open()?.readFully() + ?: throw FetchDataFlow.HashNotFound(it) } } } } + + @Suspendable + private fun getInputTransactions(tx: SignedTransaction): Set = tx.inputs.map { it.txhash }.toSet() + + private class TransactionAuthorisationFilter(private val authorisedTransactions: MutableSet = mutableSetOf(), val acceptAll: Boolean = false) { + fun isAuthorised(txId: SecureHash) = acceptAll || authorisedTransactions.contains(txId) + + fun addAuthorised(txs: Set): TransactionAuthorisationFilter { + authorisedTransactions.addAll(txs) + return this + } + + fun removeAuthorised(txId: SecureHash) { + authorisedTransactions.remove(txId) + } + } } + +/** + * This is a wildcard payload to be used by the invoker of the [DataVendingFlow] to allow unlimited access to its vault. + * + * Todo Fails with a serialization exception if it is not a list. Why? + */ +@CordaSerializable +object RetrieveAnyTransactionPayload : ArrayList() \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt b/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt index 5fa61aff96..4bb32757f5 100644 --- a/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt +++ b/core/src/main/kotlin/net/corda/core/internal/FetchDataFlow.kt @@ -50,6 +50,8 @@ sealed class FetchDataFlow( class HashNotFound(val requested: SecureHash) : FlowException() + class IllegalTransactionRequest(val requested: SecureHash) : FlowException("Illegal attempt to request a transaction (${requested}) that is not in the transitive dependency graph of the sent transaction.") + @CordaSerializable data class Result(val fromDisk: List, val downloaded: List) @@ -179,9 +181,11 @@ class FetchAttachmentsFlow(requests: Set, * Given a set of tx hashes (IDs), either loads them from local disk or asks the remote peer to provide them. * * A malicious response in which the data provided by the remote peer does not hash to the requested hash results in - * [FetchDataFlow.DownloadedVsRequestedDataMismatch] being thrown. If the remote peer doesn't have an entry, it - * results in a [FetchDataFlow.HashNotFound] exception. Note that returned transactions are not inserted into - * the database, because it's up to the caller to actually verify the transactions are valid. + * [FetchDataFlow.DownloadedVsRequestedDataMismatch] being thrown. + * If the remote peer doesn't have an entry, it results in a [FetchDataFlow.HashNotFound] exception. + * If the remote peer is not authorized to request this transaction, it results in a [FetchDataFlow.IllegalTransactionRequest] exception. + * Authorisation is accorded only on valid ancestors of the root transation. + * Note that returned transactions are not inserted into the database, because it's up to the caller to actually verify the transactions are valid. */ class FetchTransactionsFlow(requests: Set, otherSide: FlowSession) : FetchDataFlow(requests, otherSide, DataType.TRANSACTION) { diff --git a/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt b/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt index fa77543484..4786d1f998 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt @@ -72,7 +72,7 @@ class ResolveTransactionsFlow(txHashesArg: Set, } @Suspendable - @Throws(FetchDataFlow.HashNotFound::class) + @Throws(FetchDataFlow.HashNotFound::class, FetchDataFlow.IllegalTransactionRequest::class) override fun call() { val newTxns = ArrayList(txHashes.size) // Start fetching data. diff --git a/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt b/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt index 3a19824b7b..eaabb2d987 100644 --- a/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt +++ b/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt @@ -116,7 +116,7 @@ class AttachmentTests : WithMockNet { @InitiatedBy(InitiatingFetchAttachmentsFlow::class) private class FetchAttachmentsResponse(val otherSideSession: FlowSession) : FlowLogic() { @Suspendable - override fun call() = subFlow(TestDataVendingFlow(otherSideSession)) + override fun call() = subFlow(TestNoSecurityDataVendingFlow(otherSideSession)) } //region Generators diff --git a/core/src/test/kotlin/net/corda/core/flows/TestDataVendingFlow.kt b/core/src/test/kotlin/net/corda/core/flows/TestNoSecurityDataVendingFlow.kt similarity index 83% rename from core/src/test/kotlin/net/corda/core/flows/TestDataVendingFlow.kt rename to core/src/test/kotlin/net/corda/core/flows/TestNoSecurityDataVendingFlow.kt index aa2f698c89..23d61e1c01 100644 --- a/core/src/test/kotlin/net/corda/core/flows/TestDataVendingFlow.kt +++ b/core/src/test/kotlin/net/corda/core/flows/TestNoSecurityDataVendingFlow.kt @@ -5,7 +5,7 @@ import net.corda.core.internal.FetchDataFlow import net.corda.core.utilities.UntrustworthyData // Flow to start data vending without sending transaction. For testing only. -class TestDataVendingFlow(otherSideSession: FlowSession) : SendStateAndRefFlow(otherSideSession, emptyList()) { +class TestNoSecurityDataVendingFlow(otherSideSession: FlowSession) : DataVendingFlow(otherSideSession, RetrieveAnyTransactionPayload) { @Suspendable override fun sendPayloadAndReceiveDataRequest(otherSideSession: FlowSession, payload: Any): UntrustworthyData { return if (payload is List<*> && payload.isEmpty()) { diff --git a/core/src/test/kotlin/net/corda/core/internal/ResolveTransactionsFlowTest.kt b/core/src/test/kotlin/net/corda/core/internal/ResolveTransactionsFlowTest.kt index 8172bbc9be..291167e604 100644 --- a/core/src/test/kotlin/net/corda/core/internal/ResolveTransactionsFlowTest.kt +++ b/core/src/test/kotlin/net/corda/core/internal/ResolveTransactionsFlowTest.kt @@ -6,8 +6,10 @@ import net.corda.core.flows.* import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party import net.corda.core.transactions.SignedTransaction +import net.corda.core.utilities.NonEmptySet import net.corda.core.utilities.getOrThrow import net.corda.core.utilities.sequence +import net.corda.core.utilities.unwrap import net.corda.testing.contracts.DummyContract import net.corda.testing.core.singleIdentity import net.corda.testing.node.MockNetwork @@ -34,6 +36,8 @@ class ResolveTransactionsFlowTest { private lateinit var miniCorp: Party private lateinit var notary: Party + private lateinit var rootTx: SignedTransaction + @Before fun setup() { mockNet = MockNetwork(cordappPackages = listOf("net.corda.testing.contracts", "net.corda.core.internal")) @@ -160,6 +164,34 @@ class ResolveTransactionsFlowTest { } } + @Test + fun `Requesting a transaction while having the right to see it succeeds`() { + val (_, stx2) = makeTransactions() + val p = TestNoRightsVendingFlow(miniCorp, toVend = stx2, toRequest = stx2) + val future = megaCorpNode.startFlow(p) + mockNet.runNetwork() + future.getOrThrow() + } + + @Test + fun `Requesting a transaction without having the right to see it results in exception`() { + val (_, stx2) = makeTransactions() + val (_, stx3) = makeTransactions() + val p = TestNoRightsVendingFlow(miniCorp, toVend = stx2, toRequest = stx3) + val future = megaCorpNode.startFlow(p) + mockNet.runNetwork() + assertFailsWith { future.getOrThrow() } + } + + @Test + fun `Requesting a transaction twice results in exception`() { + val (_, stx2) = makeTransactions() + val p = TestResolveTwiceVendingFlow(miniCorp, stx2) + val future = megaCorpNode.startFlow(p) + mockNet.runNetwork() + assertFailsWith { future.getOrThrow() } + } + // DOCSTART 2 private fun makeTransactions(signFirstTX: Boolean = true, withAttachment: SecureHash? = null): Pair { // Make a chain of custody of dummy states and insert into node A. @@ -187,8 +219,9 @@ class ResolveTransactionsFlowTest { } // DOCEND 2 + @InitiatingFlow - private class TestFlow(val otherSide: Party, private val resolveTransactionsFlowFactory: (FlowSession) -> ResolveTransactionsFlow, private val txCountLimit: Int? = null) : FlowLogic() { + private open class TestFlow(val otherSide: Party, private val resolveTransactionsFlowFactory: (FlowSession) -> ResolveTransactionsFlow, private val txCountLimit: Int? = null) : FlowLogic() { constructor(txHashes: Set, otherSide: Party, txCountLimit: Int? = null) : this(otherSide, { ResolveTransactionsFlow(txHashes, it) }, txCountLimit = txCountLimit) constructor(stx: SignedTransaction, otherSide: Party) : this(otherSide, { ResolveTransactionsFlow(stx, it) }) @@ -200,11 +233,54 @@ class ResolveTransactionsFlowTest { subFlow(resolveTransactionsFlow) } } - @Suppress("unused") @InitiatedBy(TestFlow::class) private class TestResponseFlow(val otherSideSession: FlowSession) : FlowLogic() { @Suspendable - override fun call() = subFlow(TestDataVendingFlow(otherSideSession)) + override fun call() = subFlow(TestNoSecurityDataVendingFlow(otherSideSession)) + } + + // Used by the no-rights test + @InitiatingFlow + private class TestNoRightsVendingFlow(val otherSide: Party, val toVend: SignedTransaction, val toRequest: SignedTransaction) : FlowLogic() { + @Suspendable + override fun call() { + val session = initiateFlow(otherSide) + session.send(toRequest) + subFlow(DataVendingFlow(session, toVend)) + } + } + @Suppress("unused") + @InitiatedBy(TestNoRightsVendingFlow::class) + private open class TestResponseResolveNoRightsFlow(val otherSideSession: FlowSession) : FlowLogic() { + @Suspendable + override fun call() { + val noRightsTx = otherSideSession.receive().unwrap { it } + otherSideSession.receive().unwrap { it } + otherSideSession.sendAndReceive(FetchDataFlow.Request.Data(NonEmptySet.of(noRightsTx.inputs.first().txhash), FetchDataFlow.DataType.TRANSACTION)).unwrap { it } + otherSideSession.send(FetchDataFlow.Request.End) + } + } + + //Used by the resolve twice test + @InitiatingFlow + private class TestResolveTwiceVendingFlow(val otherSide: Party, val tx: SignedTransaction) : FlowLogic() { + @Suspendable + override fun call() { + val session = initiateFlow(otherSide) + subFlow(DataVendingFlow(session, tx)) + } + } + @Suppress("unused") + @InitiatedBy(TestResolveTwiceVendingFlow::class) + private open class TestResponseResolveTwiceFlow(val otherSideSession: FlowSession) : FlowLogic() { + @Suspendable + override fun call() { + val tx = otherSideSession.receive().unwrap { it } + val parent1 = tx.inputs.first().txhash + otherSideSession.sendAndReceive(FetchDataFlow.Request.Data(NonEmptySet.of(parent1), FetchDataFlow.DataType.TRANSACTION)).unwrap { it } + otherSideSession.sendAndReceive(FetchDataFlow.Request.Data(NonEmptySet.of(parent1), FetchDataFlow.DataType.TRANSACTION)).unwrap { it } + otherSideSession.send(FetchDataFlow.Request.End) + } } } diff --git a/core/src/test/kotlin/net/corda/core/serialization/AttachmentSerializationTest.kt b/core/src/test/kotlin/net/corda/core/serialization/AttachmentSerializationTest.kt index 92739f70c6..43c35ca0dd 100644 --- a/core/src/test/kotlin/net/corda/core/serialization/AttachmentSerializationTest.kt +++ b/core/src/test/kotlin/net/corda/core/serialization/AttachmentSerializationTest.kt @@ -6,7 +6,7 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.FlowLogic import net.corda.core.flows.FlowSession import net.corda.core.flows.InitiatingFlow -import net.corda.core.flows.TestDataVendingFlow +import net.corda.core.flows.TestNoSecurityDataVendingFlow import net.corda.core.identity.Party import net.corda.core.internal.FetchAttachmentsFlow import net.corda.core.internal.FetchDataFlow @@ -89,7 +89,7 @@ class AttachmentSerializationTest { @Suspendable override fun call() { if (sendData) { - subFlow(TestDataVendingFlow(clientSession)) + subFlow(TestNoSecurityDataVendingFlow(clientSession)) } clientSession.receive().unwrap { assertEquals("ping one", it) } clientSession.sendAndReceive("pong").unwrap { assertEquals("ping two", it) } From ff298e17e1ae66b892e36a102bf24ce7033fab04 Mon Sep 17 00:00:00 2001 From: Rick Parker Date: Thu, 2 Aug 2018 10:08:12 +0100 Subject: [PATCH 6/6] CORDA-1866 Avoid circular flushing in our hibernate column converters. (#3737) --- .../persistence/DatabaseTransaction.kt | 25 +++++++ .../keys/E2ETestKeyManagementService.kt | 1 + .../node/utilities/AppendOnlyPersistentMap.kt | 11 +-- .../HibernateColumnConverterTests.kt | 67 +++++++++++++++++++ 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 node/src/test/kotlin/net/corda/node/services/persistence/HibernateColumnConverterTests.kt diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt index 578479d39a..770477e173 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/DatabaseTransaction.kt @@ -1,6 +1,7 @@ package net.corda.nodeapi.internal.persistence import co.paralleluniverse.strands.Strand +import org.hibernate.BaseSessionEventListener import org.hibernate.Session import org.hibernate.Transaction import rx.subjects.PublishSubject @@ -21,6 +22,9 @@ class DatabaseTransaction( ) { val id: UUID = UUID.randomUUID() + val flushing: Boolean get() = _flushingCount > 0 + private var _flushingCount = 0 + val connection: Connection by lazy(LazyThreadSafetyMode.NONE) { database.dataSource.connection.apply { autoCommit = false @@ -30,6 +34,27 @@ class DatabaseTransaction( private val sessionDelegate = lazy { val session = database.entityManagerFactory.withOptions().connection(connection).openSession() + session.addEventListeners(object : BaseSessionEventListener() { + override fun flushStart() { + _flushingCount++ + super.flushStart() + } + + override fun flushEnd(numberOfEntities: Int, numberOfCollections: Int) { + super.flushEnd(numberOfEntities, numberOfCollections) + _flushingCount-- + } + + override fun partialFlushStart() { + _flushingCount++ + super.partialFlushStart() + } + + override fun partialFlushEnd(numberOfEntities: Int, numberOfCollections: Int) { + super.partialFlushEnd(numberOfEntities, numberOfCollections) + _flushingCount-- + } + }) hibernateTransaction = session.beginTransaction() session } diff --git a/node/src/main/kotlin/net/corda/node/services/keys/E2ETestKeyManagementService.kt b/node/src/main/kotlin/net/corda/node/services/keys/E2ETestKeyManagementService.kt index 5233041642..c99732a356 100644 --- a/node/src/main/kotlin/net/corda/node/services/keys/E2ETestKeyManagementService.kt +++ b/node/src/main/kotlin/net/corda/node/services/keys/E2ETestKeyManagementService.kt @@ -32,6 +32,7 @@ class E2ETestKeyManagementService(val identityService: IdentityService) : Single private val mutex = ThreadBox(InnerState()) // Accessing this map clones it. override val keys: Set get() = mutex.locked { keys.keys } + val keyPairs: Set get() = mutex.locked { keys.map { KeyPair(it.key, it.value) }.toSet() } override fun start(initialKeyPairs: Set) { mutex.locked { diff --git a/node/src/main/kotlin/net/corda/node/utilities/AppendOnlyPersistentMap.kt b/node/src/main/kotlin/net/corda/node/utilities/AppendOnlyPersistentMap.kt index 19db9ef18d..1fa769b77f 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/AppendOnlyPersistentMap.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/AppendOnlyPersistentMap.kt @@ -124,11 +124,14 @@ abstract class AppendOnlyPersistentMapBase( protected fun loadValue(key: K): V? { val session = currentDBSession() - // IMPORTANT: The flush is needed because detach() makes the queue of unflushed entries invalid w.r.t. Hibernate internal state if the found entity is unflushed. - // We want the detach() so that we rely on our cache memory management and don't retain strong references in the Hibernate session. - session.flush() + val flushing = contextTransaction.flushing + if (!flushing) { + // IMPORTANT: The flush is needed because detach() makes the queue of unflushed entries invalid w.r.t. Hibernate internal state if the found entity is unflushed. + // We want the detach() so that we rely on our cache memory management and don't retain strong references in the Hibernate session. + session.flush() + } val result = session.find(persistentEntityClass, toPersistentEntityKey(key)) - return result?.apply { session.detach(result) }?.let(fromPersistentEntity)?.second + return result?.apply { if (!flushing) session.detach(result) }?.let(fromPersistentEntity)?.second } operator fun contains(key: K) = get(key) != null diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/HibernateColumnConverterTests.kt b/node/src/test/kotlin/net/corda/node/services/persistence/HibernateColumnConverterTests.kt new file mode 100644 index 0000000000..4832474cac --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/services/persistence/HibernateColumnConverterTests.kt @@ -0,0 +1,67 @@ +package net.corda.node.services.persistence + +import net.corda.core.identity.Party +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.getOrThrow +import net.corda.finance.DOLLARS +import net.corda.finance.`issued by` +import net.corda.finance.contracts.asset.Cash +import net.corda.finance.flows.CashIssueFlow +import net.corda.node.services.identity.PersistentIdentityService +import net.corda.node.services.keys.E2ETestKeyManagementService +import net.corda.testing.core.BOC_NAME +import net.corda.testing.node.InMemoryMessagingNetwork +import net.corda.testing.node.MockNetwork +import net.corda.testing.node.StartedMockNode +import org.junit.After +import org.junit.Before +import org.junit.Test +import kotlin.test.assertEquals + +class HibernateColumnConverterTests { + private lateinit var mockNet: MockNetwork + private lateinit var bankOfCordaNode: StartedMockNode + private lateinit var bankOfCorda: Party + private lateinit var notary: Party + + @Before + fun start() { + mockNet = MockNetwork( + servicePeerAllocationStrategy = InMemoryMessagingNetwork.ServicePeerAllocationStrategy.RoundRobin(), + cordappPackages = listOf("net.corda.finance.contracts.asset", "net.corda.finance.schemas")) + bankOfCordaNode = mockNet.createPartyNode(BOC_NAME) + bankOfCorda = bankOfCordaNode.info.identityFromX500Name(BOC_NAME) + notary = mockNet.defaultNotaryIdentity + } + + @After + fun cleanUp() { + mockNet.stopNodes() + } + + // AbstractPartyToX500NameAsStringConverter could cause circular flush of Hibernate session because it is invoked during flush, and a + // cache miss was doing a flush. This also checks that loading during flush does actually work. + @Test + fun `issue some cash on a notary that exists only in the database to check cache loading works in our identity column converters during flush of vault update`() { + val expected = 500.DOLLARS + val ref = OpaqueBytes.of(0x01) + + // Create parallel set of key and identity services so that the values are not cached, forcing the node caches to do a lookup. + val identityService = PersistentIdentityService() + val originalIdentityService: PersistentIdentityService = bankOfCordaNode.services.identityService as PersistentIdentityService + identityService.database = originalIdentityService.database + identityService.start(originalIdentityService.trustRoot) + val keyService = E2ETestKeyManagementService(identityService) + keyService.start((bankOfCordaNode.services.keyManagementService as E2ETestKeyManagementService).keyPairs) + + // New identity for a notary (doesn't matter that it's for Bank Of Corda... since not going to use it as an actual notary etc). + val newKeyAndCert = keyService.freshKeyAndCert(bankOfCordaNode.info.legalIdentitiesAndCerts[0], false) + val randomNotary = Party(BOC_NAME, newKeyAndCert.owningKey) + + val future = bankOfCordaNode.startFlow(CashIssueFlow(expected, ref, randomNotary)) + mockNet.runNetwork() + val issueTx = future.getOrThrow().stx + val output = issueTx.tx.outputsOfType().single() + assertEquals(expected.`issued by`(bankOfCorda.ref(ref)), output.amount) + } +}