Merge branch 'master' of https://github.com/corda/corda into christians_checkpoint_checker_thread

.ci/dependency-checker/suppressedLibraries.xml

@@ -10,5 +10,22 @@
       <cpe>cpe:/a:apache:struts:2.0.0</cpe>
    </suppress>
     -->
-
+    <suppress>
+        <!-- Vulnerability when using SSLv2 Hello messages. Corda uses TLS1.2-->
+        <notes><![CDATA[file name: catalyst-netty-1.1.2.jar]]></notes>
+        <gav regex="true">^io\.atomix\.catalyst:catalyst-netty:.*$</gav>
+        <cve>CVE-2014-3488</cve>
+    </suppress>
+    <suppress>
+        <!-- Vulnerability to LDAP poisoning attacks. Corda doesn't use LDAP-->
+        <notes><![CDATA[file name: groovy-all-1.8.9.jar]]></notes>
+        <gav regex="true">^commons-cli:commons-cli:.*$</gav>
+        <cve>CVE-2016-6497</cve>
+    </suppress>
+    <suppress>
+        <!-- Java objects serialization disabled in Corda -->
+        <notes><![CDATA[file name: groovy-all-1.8.9.jar]]></notes>
+        <gav regex="true">^commons-cli:commons-cli:.*$</gav>
+        <cve>CVE-2015-3253</cve>
+    </suppress>
 </suppressions>

.gitignore

@@ -36,6 +36,7 @@ lib/quasar.jar
 .idea/dataSources
 .idea/markdown-navigator
 .idea/runConfigurations
+.idea/dictionaries
 /gradle-plugins/.idea/
 
 # Include the -parameters compiler option by default in IntelliJ required for serialization.

.idea/compiler.xml

@@ -42,6 +42,7 @@
       <module name="explorer-capsule_test" target="1.6" />
       <module name="explorer_main" target="1.8" />
       <module name="explorer_test" target="1.8" />
+      <module name="finance_integrationTest" target="1.8" />
       <module name="finance_main" target="1.8" />
       <module name="finance_test" target="1.8" />
       <module name="graphs_main" target="1.8" />

build.gradle

@@ -22,19 +22,19 @@ buildscript {
 
     ext.asm_version = '0.5.3'
     ext.artemis_version = '2.1.0'
-    ext.jackson_version = '2.8.5'
+    ext.jackson_version = '2.9.2'
-    ext.jetty_version = '9.3.9.v20160517'
+    ext.jetty_version = '9.4.7.v20170914'
     ext.jersey_version = '2.25'
     ext.jolokia_version = '2.0.0-M3'
-    ext.assertj_version = '3.6.1'
+    ext.assertj_version = '3.8.0'
     ext.slf4j_version = '1.7.25'
-    ext.log4j_version = '2.7'
+    ext.log4j_version = '2.9.1'
     ext.bouncycastle_version = constants.getProperty("bouncycastleVersion")
     ext.guava_version = constants.getProperty("guavaVersion")
     ext.okhttp_version = '3.5.0'
     ext.netty_version = '4.1.9.Final'
     ext.typesafe_config_version = constants.getProperty("typesafeConfigVersion")
-    ext.fileupload_version = '1.3.2'
+    ext.fileupload_version = '1.3.3'
     ext.junit_version = '4.12'
     ext.mockito_version = '2.10.0'
     ext.jopt_simple_version = '5.0.2'
@@ -46,6 +46,8 @@ buildscript {
     ext.dokka_version = '0.9.14'
     ext.eddsa_version = '0.2.0'
     ext.dependency_checker_version = '3.0.1'
+    ext.commons_collections_version = '4.1'
+    ext.beanutils_version = '1.9.3'
 
     // Update 121 is required for ObjectInputFilter and at time of writing 131 was latest:
     ext.java8_minUpdateVersion = '131'

client/jfx/build.gradle

@@ -37,6 +37,9 @@ dependencies {
     compile 'org.fxmisc.easybind:easybind:1.0.3'
 
     // Artemis Client: ability to connect to an Artemis broker and control it.
+    // TODO: remove the forced update of commons-collections and beanutils when artemis updates them
+    compile "org.apache.commons:commons-collections4:${commons_collections_version}"
+    compile "commons-beanutils:commons-beanutils:${beanutils_version}"
     compile "org.apache.activemq:artemis-core-client:${artemis_version}"
 
     // Unit testing helpers.

core/src/main/kotlin/net/corda/core/flows/InitiatedBy.kt

@@ -15,4 +15,5 @@ import kotlin.reflect.KClass
  * @see InitiatingFlow
  */
 @Target(CLASS)
+@MustBeDocumented
 annotation class InitiatedBy(val value: KClass<out FlowLogic<*>>)

core/src/main/kotlin/net/corda/core/flows/ReceiveTransactionFlow.kt

@@ -3,6 +3,7 @@ package net.corda.core.flows
 import co.paralleluniverse.fibers.Suspendable
 import net.corda.core.contracts.*
 import net.corda.core.internal.ResolveTransactionsFlow
+import net.corda.core.node.StatesToRecord
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.utilities.unwrap
 import java.security.SignatureException
@@ -14,25 +15,41 @@ import java.security.SignatureException
  * [SignedTransaction] and perform the resolution back-and-forth required to check the dependencies and download any missing
  * attachments. The flow will return the [SignedTransaction] after it is resolved and then verified using [SignedTransaction.verify].
  *
- * @param otherSideSession session to the other side which is calling [SendTransactionFlow].
+ * Please note that it will *not* store the transaction to the vault unless that is explicitly requested.
- * @param checkSufficientSignatures if true checks all required signatures are present. See [SignedTransaction.verify].
+ *
+ * @property otherSideSession session to the other side which is calling [SendTransactionFlow].
+ * @property checkSufficientSignatures if true checks all required signatures are present. See [SignedTransaction.verify].
+ * @property statesToRecord which transaction states should be recorded in the vault, if any.
  */
-class ReceiveTransactionFlow(private val otherSideSession: FlowSession,
+class ReceiveTransactionFlow @JvmOverloads constructor(private val otherSideSession: FlowSession,
-                             private val checkSufficientSignatures: Boolean) : FlowLogic<SignedTransaction>() {
+                                                       private val checkSufficientSignatures: Boolean = true,
-    /** Receives a [SignedTransaction] from [otherSideSession], verifies it and then records it in the vault. */
+                                                       private val statesToRecord: StatesToRecord = StatesToRecord.NONE) : FlowLogic<SignedTransaction>() {
-    constructor(otherSideSession: FlowSession) : this(otherSideSession, true)
+    @Suppress("KDocMissingDocumentation")
-
     @Suspendable
     @Throws(SignatureException::class,
             AttachmentResolutionException::class,
             TransactionResolutionException::class,
             TransactionVerificationException::class)
     override fun call(): SignedTransaction {
-        return otherSideSession.receive<SignedTransaction>().unwrap {
+        if (checkSufficientSignatures) {
+            logger.trace("Receiving a transaction from ${otherSideSession.counterparty}")
+        } else {
+            logger.trace("Receiving a transaction (but without checking the signatures) from ${otherSideSession.counterparty}")
+        }
+
+        val stx = otherSideSession.receive<SignedTransaction>().unwrap {
             subFlow(ResolveTransactionsFlow(it, otherSideSession))
             it.verify(serviceHub, checkSufficientSignatures)
             it
         }
+
+        if (checkSufficientSignatures) {
+            // We should only send a transaction to the vault for processing if we did in fact fully verify it, and
+            // there are no missing signatures. We don't want partly signed stuff in the vault.
+            logger.trace("Successfully received fully signed tx ${stx.id}, sending to the vault for processing")
+            serviceHub.recordTransactions(statesToRecord, setOf(stx))
+        }
+        return stx
     }
 }
 

core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt

@@ -5,6 +5,7 @@ import net.corda.core.crypto.SecureHash
 import net.corda.core.flows.FlowException
 import net.corda.core.flows.FlowLogic
 import net.corda.core.flows.FlowSession
+import net.corda.core.node.StatesToRecord
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.utilities.exactAdd
@@ -94,7 +95,7 @@ class ResolveTransactionsFlow(private val txHashes: Set<SecureHash>,
             // half way through, it's no big deal, although it might result in us attempting to re-download data
             // redundantly next time we attempt verification.
             it.verify(serviceHub)
-            serviceHub.recordTransactions(false, it)
+            serviceHub.recordTransactions(StatesToRecord.NONE, listOf(it))
         }
 
         return signedTransaction?.let {

core/src/main/kotlin/net/corda/core/node/ServiceHub.kt

@@ -50,6 +50,26 @@ interface ServicesForResolution : StateLoader {
     val cordappProvider: CordappProvider
 }
 
+/**
+ * Controls whether the transaction is sent to the vault at all, and if so whether states have to be relevant
+ * or not in order to be recorded. Used in [ServiceHub.recordTransactions]
+ */
+enum class StatesToRecord {
+    /** The received transaction is not sent to the vault at all. This is used within transaction resolution. */
+    NONE,
+    /**
+     * All states that can be seen in the transaction will be recorded by the vault, even if none of the identities
+     * on this node are a participant or owner.
+     */
+    ALL_VISIBLE,
+    /**
+     * Only states that involve one of our public keys will be stored in the vault. This is the default. A public
+     * key is involved (relevant) if it's in the [OwnableState.owner] field, or appears in the [ContractState.participants]
+     * collection. This is usually equivalent to "can I change the contents of this state by signing a transaction".
+     */
+    ONLY_RELEVANT
+}
+
 /**
  * A service hub is the starting point for most operations you can do inside the node. You are provided with one
  * when a class annotated with [CordaService] is constructed, and you have access to one inside flows. Most RPCs
@@ -132,7 +152,9 @@ interface ServiceHub : ServicesForResolution {
      * @param txs The transactions to record.
      * @param notifyVault indicate if the vault should be notified for the update.
      */
-    fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>)
+    fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+        recordTransactions(if (notifyVault) StatesToRecord.ONLY_RELEVANT else StatesToRecord.NONE, txs)
+    }
 
     /**
      * Stores the given [SignedTransaction]s in the local transaction storage and then sends them to the vault for
@@ -142,12 +164,22 @@ interface ServiceHub : ServicesForResolution {
         recordTransactions(notifyVault, listOf(first, *remaining))
     }
 
+    /**
+     * Stores the given [SignedTransaction]s in the local transaction storage and then sends them to the vault for
+     * further processing if [statesToRecord] is not [StatesToRecord.NONE].
+     * This is expected to be run within a database transaction.
+     *
+     * @param txs The transactions to record.
+     * @param statesToRecord how the vault should treat the output states of the transaction.
+     */
+    fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>)
+
     /**
      * Stores the given [SignedTransaction]s in the local transaction storage and then sends them to the vault for
      * further processing. This is expected to be run within a database transaction.
      */
     fun recordTransactions(first: SignedTransaction, vararg remaining: SignedTransaction) {
-        recordTransactions(true, first, *remaining)
+        recordTransactions(listOf(first, *remaining))
     }
 
     /**
@@ -155,7 +187,7 @@ interface ServiceHub : ServicesForResolution {
      * further processing. This is expected to be run within a database transaction.
      */
     fun recordTransactions(txs: Iterable<SignedTransaction>) {
-        recordTransactions(true, txs)
+        recordTransactions(StatesToRecord.ONLY_RELEVANT, txs)
     }
 
     /**

docs/source/changelog.rst

@@ -6,6 +6,7 @@ from the previous milestone release.
 
 UNRELEASED
 ----------
+
 * ``OpaqueBytes.bytes`` now returns a clone of its underlying ``ByteArray``, and has been redeclared as ``final``.
   This is a minor change to the public API, but is required to ensure that classes like ``SecureHash`` are immutable.
 
@@ -37,17 +38,17 @@ UNRELEASED
   ``notaryNodeAddress``, ``notaryClusterAddresses`` and ``bftSMaRt`` have also been removed and replaced by a single
   ``notary`` config object. See :doc:`corda-configuration-file` for more details.
 
-* Gradle task ``deployNodes`` can have an additional parameter `configFile` with the path to a properties file
+* Gradle task ``deployNodes`` can have an additional parameter ``configFile`` with the path to a properties file
   to be appended to node.conf.
 
-* Cordformation node building DSL can have an additional parameter `configFile` with the path to a properties file
+* Cordformation node building DSL can have an additional parameter ``configFile`` with the path to a properties file
   to be appended to node.conf.
 
 * ``FlowLogic`` now has a static method called ``sleep`` which can be used in certain circumstances to help with resolving
   contention over states in flows.  This should be used in place of any other sleep primitive since these are not compatible
   with flows and their use will be prevented at some point in the future.  Pay attention to the warnings and limitations
   described in the documentation for this method.  This helps resolve a bug in ``Cash`` coin selection.
-  A new static property `currentTopLevel` returns the top most `FlowLogic` instance, or null if not in a flow.
+  A new static property ``currentTopLevel`` returns the top most ``FlowLogic`` instance, or null if not in a flow.
 
 * ``CordaService`` annotated classes should be upgraded to take a constructor parameter of type ``AppServiceHub`` which
   allows services to start flows marked with the ``StartableByService`` annotation. For backwards compatability
@@ -65,7 +66,10 @@ UNRELEASED
 * A new function ``checkCommandVisibility(publicKey: PublicKey)`` has been added to ``FilteredTransaction`` to check
   if every command that a signer should receive (e.g. an Oracle) is indeed visible.
 
-* Change the AMQP serialiser to use the oficially assigned R3 identifier rather than a placeholder.
+* Changed the AMQP serialiser to use the oficially assigned R3 identifier rather than a placeholder.
+
+* The ``ReceiveTransactionFlow`` can now be told to record the transaction at the same time as receiving it. Using this
+  feature, better support for observer/regulator nodes has been added. See :doc:`tutorial-observer-nodes`.
 
 .. _changelog_v1:
 

docs/source/release-notes.rst

@@ -6,6 +6,9 @@ Here are release notes for each snapshot release from M9 onwards.
 Unreleased
 ----------
 
+Support for observer/regulator nodes has returned. Read :doc:`tutorial-observer-nodes` to learn more or examine the
+interest rate swaps demo.
+
 Release 1.0
 -----------
 Corda 1.0 is finally here!

docs/source/tutorial-observer-nodes.rst

@@ -0,0 +1,42 @@
+.. highlight:: kotlin
+.. raw:: html
+
+   <script type="text/javascript" src="_static/jquery.js"></script>
+   <script type="text/javascript" src="_static/codesets.js"></script>
+
+Observer nodes
+==============
+
+Posting transactions to an observer node is a common requirement in finance, where regulators often want
+to receive comprehensive reporting on all actions taken. By running their own node, regulators can receive a stream
+of digitally signed, de-duplicated reports useful for later processing.
+
+Adding support for observer nodes to your application is easy. The IRS (interest rate swap) demo shows to do it.
+
+Just define a new flow that wraps the SendTransactionFlow/ReceiveTransactionFlow, as follows:
+
+.. container:: codeset
+
+    .. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt
+        :language: kotlin
+        :start-after: DOCSTART 1
+        :end-before: DOCEND 1
+
+In this example, the ``AutoOfferFlow`` is the business logic, and we define two very short and simple flows to send
+the transaction to the regulator. There are two important aspects to note here:
+
+1. The ``ReportToRegulatorFlow`` is marked as an ``@InitiatingFlow`` because it will start a new conversation, context
+   free, with the regulator.
+2. The ``ReceiveRegulatoryReportFlow`` uses ``ReceiveTransactionFlow`` in a special way - it tells it to send the
+   transaction to the vault for processing, including all states even if not involving our public keys. This is required
+   because otherwise the vault will ignore states that don't list any of the node's public keys, but in this case,
+   we do want to passively observe states we can't change. So overriding this behaviour is required.
+
+If the states define a relational mapping (see :doc:`api-persistence`) then the regulator will be able to query the
+reports from their database and observe new transactions coming in via RPC.
+
+.. warning:: Nodes which act as both observers and which directly take part in the ledger are not supported at this
+   time. In particular, coin selection may return states which you do not have the private keys to be able to sign
+   for. Future versions of Corda may address this issue, but for now, if you wish to both participate in the ledger
+   and also observe transactions that you can't sign for you will need to run two nodes and have two separate
+   identities.

docs/source/tutorials-index.rst

@@ -19,4 +19,5 @@ Tutorials
    tutorial-custom-notary
    tutorial-tear-offs
    tutorial-attachments
-   event-scheduling
+   event-scheduling
+   tutorial-observer-nodes

gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordform.kt

@@ -11,6 +11,7 @@ import org.gradle.api.tasks.SourceSet.MAIN_SOURCE_SET_NAME
 import org.gradle.api.tasks.TaskAction
 import java.io.File
 import java.net.URLClassLoader
+import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.util.concurrent.TimeUnit
@@ -141,8 +142,6 @@ open class Cordform : DefaultTask() {
         }
     }
 
-    private fun fullNodePath(node: Node): Path = project.projectDir.toPath().resolve(node.nodeDir.toPath())
-
     private fun generateAndInstallNodeInfos() {
         generateNodeInfos()
         installNodeInfos()
@@ -150,35 +149,65 @@ open class Cordform : DefaultTask() {
 
     private fun generateNodeInfos() {
         project.logger.info("Generating node infos")
-        val generateTimeoutSeconds = 60L
+        var nodeProcesses = buildNodeProcesses()
-        val processes = nodes.map { node ->
-            project.logger.info("Generating node info for ${fullNodePath(node)}")
-            val logDir = File(fullNodePath(node).toFile(), "logs")
-            logDir.mkdirs() // Directory may not exist at this point
-            Pair(node, ProcessBuilder("java", "-jar", Node.nodeJarName, "--just-generate-node-info")
-                    .directory(fullNodePath(node).toFile())
-                    .redirectErrorStream(true)
-                    // InheritIO causes hangs on windows due the gradle buffer also not being flushed.
-                    // Must redirect to output or logger (node log is still written, this is just startup banner)
-                    .redirectOutput(File(logDir, "generate-info-log.txt"))
-                    .start())
-        }
         try {
-            processes.parallelStream().forEach { (node, process) ->
+            validateNodeProcessess(nodeProcesses)
-                if (!process.waitFor(generateTimeoutSeconds, TimeUnit.SECONDS)) {
-                    throw GradleException("Node took longer $generateTimeoutSeconds seconds than too to generate node info - see node log at ${fullNodePath(node)}/logs")
-                } else if (process.exitValue() != 0) {
-                    throw GradleException("Node exited with ${process.exitValue()} when generating node infos - see node log at ${fullNodePath(node)}/logs")
-                }
-            }
         } finally {
-            // This will be a no-op on success - abort remaining on failure
+            destroyNodeProcesses(nodeProcesses)
-            processes.forEach {
-                it.second.destroyForcibly()
-            }
         }
     }
 
+    private fun buildNodeProcesses(): Map<Node, Process> {
+        return nodes
+                .map { buildNodeProcess(it) }
+                .toMap()
+    }
+
+    private fun validateNodeProcessess(nodeProcesses: Map<Node, Process>) {
+        nodeProcesses.forEach { (node, process) ->
+            validateNodeProcess(node, process)
+        }
+    }
+
+    private fun destroyNodeProcesses(nodeProcesses: Map<Node, Process>) {
+        nodeProcesses.forEach { (_, process) ->
+            process.destroyForcibly()
+        }
+    }
+
+    private fun buildNodeProcess(node: Node): Pair<Node, Process> {
+        node.makeLogDirectory()
+        var process = ProcessBuilder(generateNodeInfoCommand())
+                .directory(node.fullPath().toFile())
+                .redirectErrorStream(true)
+                // InheritIO causes hangs on windows due the gradle buffer also not being flushed.
+                // Must redirect to output or logger (node log is still written, this is just startup banner)
+                .redirectOutput(node.logFile().toFile())
+                .addEnvironment("CAPSULE_CACHE_DIR", Node.capsuleCacheDir)
+                .start()
+        return Pair(node, process)
+    }
+
+    private fun generateNodeInfoCommand(): List<String> = listOf(
+            "java",
+            "-Dcapsule.log=verbose",
+            "-Dcapsule.dir=${Node.capsuleCacheDir}",
+            "-jar",
+            Node.nodeJarName,
+            "--just-generate-node-info"
+    )
+
+    private fun validateNodeProcess(node: Node, process: Process) {
+        val generateTimeoutSeconds = 60L
+        if (!process.waitFor(generateTimeoutSeconds, TimeUnit.SECONDS)) {
+            throw GradleException("Node took longer $generateTimeoutSeconds seconds than too to generate node info - see node log at ${node.fullPath()}/logs")
+        }
+        if (process.exitValue() != 0) {
+            throw GradleException("Node exited with ${process.exitValue()} when generating node infos - see node log at ${node.fullPath()}/logs")
+        }
+        project.logger.info("Generated node info for ${node.fullPath()}")
+    }
+
     private fun installNodeInfos() {
         project.logger.info("Installing node infos")
         for (source in nodes) {
@@ -186,13 +215,15 @@ open class Cordform : DefaultTask() {
                 if (source.nodeDir != destination.nodeDir) {
                     project.copy {
                         it.apply {
-                            from(fullNodePath(source).toString())
+                            from(source.fullPath().toString())
                             include("nodeInfo-*")
-                            into(fullNodePath(destination).resolve(CordformNode.NODE_INFO_DIRECTORY).toString())
+                            into(destination.fullPath().resolve(CordformNode.NODE_INFO_DIRECTORY).toString())
                         }
                     }
                 }
             }
         }
     }
+    private fun Node.logFile(): Path = this.logDirectory().resolve("generate-info.log")
+    private fun ProcessBuilder.addEnvironment(key: String, value: String) = this.apply { environment().put(key, value) }
 }

gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordformation.kt

@@ -18,10 +18,12 @@ class Cordformation : Plugin<Project> {
          * @return A file handle to the file in the JAR.
          */
         fun getPluginFile(project: Project, filePathInJar: String): File {
-            val archive: File? = project.rootProject.buildscript.configurations.single { it.name == "classpath" }.find {
+            val archive: File? = project.rootProject.buildscript.configurations
-                it.name.contains("cordformation")
+                    .single { it.name == "classpath" }
-            }
+                    .find { it.name.contains("cordformation") }
-            return project.rootProject.resources.text.fromArchiveEntry(archive, filePathInJar).asFile()
+            return project.rootProject.resources.text
+                    .fromArchiveEntry(archive, filePathInJar)
+                    .asFile()
         }
 
         val executableFileMode = "0755".toInt(8)

gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Node.kt

@@ -21,8 +21,13 @@ class Node(private val project: Project) : CordformNode() {
         @JvmStatic
         val webJarName = "corda-webserver.jar"
         private val configFileProperty = "configFile"
+        val capsuleCacheDir: String = "./cache"
     }
 
+    fun fullPath(): Path = project.projectDir.toPath().resolve(nodeDir.toPath())
+    fun logDirectory(): Path = fullPath().resolve("logs")
+    fun makeLogDirectory() = Files.createDirectories(logDirectory())
+
     /**
      * Set the list of CorDapps to install to the plugins directory. Each cordapp is a fully qualified Maven
      * dependency name, eg: com.example:product-name:0.1
@@ -128,7 +133,7 @@ class Node(private val project: Project) : CordformNode() {
             // Can't parse as an X500 name, use the full string
             name
         }
-        nodeDir = File(rootDir.toFile(), dirName.replace("\\s", ""))
+        nodeDir = File(rootDir.toFile(), dirName)
     }
 
     private fun configureProperties() {
@@ -201,7 +206,12 @@ class Node(private val project: Project) : CordformNode() {
      * Installs the configuration file to this node's directory and detokenises it.
      */
     private fun installConfig() {
-        val options = ConfigRenderOptions.defaults().setOriginComments(false).setComments(false).setFormatted(false).setJson(false)
+        val options = ConfigRenderOptions
+                .defaults()
+                .setOriginComments(false)
+                .setComments(false)
+                .setFormatted(true)
+                .setJson(false)
         val configFileText = config.root().render(options).split("\n").toList()
 
         // Need to write a temporary file first to use the project.copy, which resolves directories correctly.

gradle-plugins/cordformation/src/noderunner/kotlin/net/corda/plugins/NodeRunner.kt

@@ -104,7 +104,7 @@ private class TerminalWindowJavaCommand(jarName: String, dir: File, debugPort: I
     delay 0.5
     tell app "System Events" to tell process "Terminal" to keystroke "t" using command down
     delay 0.5
-    do script "bash -c 'cd $dir; ${command.joinToString(" ")} && exit'" in selected tab of the front window
+    do script "bash -c 'cd \"$dir\" ; \"${command.joinToString("""\" \"""")}\" && exit'" in selected tab of the front window
 end tell""")
         }
         OS.WINDOWS -> {

node-api/build.gradle

@@ -11,6 +11,10 @@ dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
     compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
     testCompile "org.jetbrains.kotlin:kotlin-test:$kotlin_version"
+
+    // TODO: remove the forced update of commons-collections and beanutils when artemis updates them
+    compile "org.apache.commons:commons-collections4:${commons_collections_version}"
+    compile "commons-beanutils:commons-beanutils:${beanutils_version}"
     compile "org.apache.activemq:artemis-core-client:${artemis_version}"
     compile "org.apache.activemq:artemis-commons:${artemis_version}"
 

node/build.gradle

@@ -95,6 +95,9 @@ dependencies {
     compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
 
     // Artemis: for reliable p2p message queues.
+    // TODO: remove the forced update of commons-collections and beanutils when artemis updates them
+    compile "org.apache.commons:commons-collections4:${commons_collections_version}"
+    compile "commons-beanutils:commons-beanutils:${beanutils_version}"
     compile "org.apache.activemq:artemis-server:${artemis_version}"
     compile "org.apache.activemq:artemis-core-client:${artemis_version}"
     runtime ("org.apache.activemq:artemis-amqp-protocol:${artemis_version}") {

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -16,10 +16,7 @@ import net.corda.core.internal.concurrent.doneFuture
 import net.corda.core.internal.concurrent.flatMap
 import net.corda.core.internal.concurrent.openFuture
 import net.corda.core.messaging.*
-import net.corda.core.node.AppServiceHub
+import net.corda.core.node.*
-import net.corda.core.node.NodeInfo
-import net.corda.core.node.ServiceHub
-import net.corda.core.node.StateLoader
 import net.corda.core.node.services.*
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
@@ -803,9 +800,9 @@ abstract class AbstractNode(config: NodeConfiguration,
             return flowFactories[initiatingFlowClass]
         }
 
-        override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+        override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
             database.transaction {
-                super.recordTransactions(notifyVault, txs)
+                super.recordTransactions(statesToRecord, txs)
             }
         }
 

node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt

@@ -274,6 +274,7 @@ class CordappLoader private constructor(private val cordappJarPaths: List<Restri
     /** @param rootPackageName only this package and subpackages may be extracted from [url], or null to allow all packages. */
     private class RestrictedURL(val url: URL, rootPackageName: String?) {
         val qualifiedNamePrefix = rootPackageName?.let { it + '.' } ?: ""
+        override fun toString() = url.toString()
     }
 
     private inner class RestrictedScanResult(private val scanResult: ScanResult, private val qualifiedNamePrefix: String) {

node/src/main/kotlin/net/corda/node/services/CoreFlowHandlers.kt

@@ -7,6 +7,7 @@ import net.corda.core.contracts.requireThat
 import net.corda.core.flows.*
 import net.corda.core.identity.Party
 import net.corda.core.internal.ContractUpgradeUtils
+import net.corda.core.node.StatesToRecord
 import net.corda.core.transactions.SignedTransaction
 
 // TODO: We should have a whitelist of contracts we're willing to accept at all, and reject if the transaction
@@ -16,8 +17,7 @@ import net.corda.core.transactions.SignedTransaction
 class FinalityHandler(private val sender: FlowSession) : FlowLogic<Unit>() {
     @Suspendable
     override fun call() {
-        val stx = subFlow(ReceiveTransactionFlow(sender))
+        subFlow(ReceiveTransactionFlow(sender, true, StatesToRecord.ONLY_RELEVANT))
-        serviceHub.recordTransactions(stx)
     }
 }
 

node/src/main/kotlin/net/corda/node/services/api/ServiceHubInternal.kt

@@ -15,6 +15,7 @@ import net.corda.core.messaging.SingleMessageRecipient
 import net.corda.core.messaging.StateMachineTransactionMapping
 import net.corda.core.node.NodeInfo
 import net.corda.core.node.ServiceHub
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.NetworkMapCache
 import net.corda.core.node.services.NetworkMapCacheBase
 import net.corda.core.node.services.TransactionStorage
@@ -93,7 +94,7 @@ interface ServiceHubInternal : ServiceHub {
     val database: CordaPersistence
     val configuration: NodeConfiguration
     override val cordappProvider: CordappProviderInternal
-    override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+    override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
         require(txs.any()) { "No transactions passed in for recording" }
         val recordedTransactions = txs.filter { validatedTransactions.addTransaction(it) }
         val stateMachineRunId = FlowStateMachineImpl.currentStateMachine()?.id
@@ -105,9 +106,43 @@ interface ServiceHubInternal : ServiceHub {
             log.warn("Transactions recorded from outside of a state machine")
         }
 
-        if (notifyVault) {
+        if (statesToRecord != StatesToRecord.NONE) {
             val toNotify = recordedTransactions.map { if (it.isNotaryChangeTransaction()) it.notaryChangeTx else it.tx }
-            vaultService.notifyAll(toNotify)
+            // When the user has requested StatesToRecord.ALL we may end up recording and relationally mapping states
+            // that do not involve us and that we cannot sign for. This will break coin selection and thus a warning
+            // is present in the documentation for this feature (see the "Observer nodes" tutorial on docs.corda.net).
+            //
+            // The reason for this is three-fold:
+            //
+            // 1) We are putting in place the observer mode feature relatively quickly to meet specific customer
+            //    launch target dates.
+            //
+            // 2) The right design for vaults which mix observations and relevant states isn't entirely clear yet.
+            //
+            // 3) If we get the design wrong it could create security problems and business confusions.
+            //
+            // Back in the bitcoinj days I did add support for "watching addresses" to the wallet code, which is the
+            // Bitcoin equivalent of observer nodes:
+            //
+            //   https://bitcoinj.github.io/working-with-the-wallet#watching-wallets
+            //
+            // The ability to have a wallet containing both irrelevant and relevant states complicated everything quite
+            // dramatically, even methods as basic as the getBalance() API which required additional modes to let you
+            // query "balance I can spend" vs "balance I am observing". In the end it might have been better to just
+            // require the user to create an entirely separate wallet for observing with.
+            //
+            // In Corda we don't support a single node having multiple vaults (at the time of writing), and it's not
+            // clear that's the right way to go: perhaps adding an "origin" column to the VAULT_STATES table is a better
+            // solution. Then you could select subsets of states depending on where the report came from.
+            //
+            // The risk of doing this is that apps/developers may use 'canned SQL queries' not written by us that forget
+            // to add a WHERE clause for the origin column. Those queries will seem to work most of the time until
+            // they're run on an observer node and mix in irrelevant data. In the worst case this may result in
+            // erroneous data being reported to the user, which could cause security problems.
+            //
+            // Because the primary use case for recording irrelevant states is observer/regulator nodes, who are unlikely
+            // to make writes to the ledger very often or at all, we choose to punt this issue for the time being.
+            vaultService.notifyAll(statesToRecord, toNotify)
         }
     }
 

node/src/main/kotlin/net/corda/node/services/api/VaultServiceInternal.kt

@@ -1,5 +1,6 @@
 package net.corda.node.services.api
 
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.VaultService
 import net.corda.core.transactions.CoreTransaction
 import net.corda.core.transactions.NotaryChangeWireTransaction
@@ -12,8 +13,8 @@ interface VaultServiceInternal : VaultService {
      * indicate whether an update consists entirely of regular or notary change transactions, which may require
      * different processing logic.
      */
-    fun notifyAll(txns: Iterable<CoreTransaction>)
+    fun notifyAll(statesToRecord: StatesToRecord, txns: Iterable<CoreTransaction>)
 
     /** Same as notifyAll but with a single transaction. */
-    fun notify(tx: CoreTransaction) = notifyAll(listOf(tx))
+    fun notify(statesToRecord: StatesToRecord, tx: CoreTransaction) = notifyAll(statesToRecord, listOf(tx))
 }

node/src/main/kotlin/net/corda/node/services/statemachine/FlowSessionImpl.kt

@@ -8,10 +8,7 @@ import net.corda.core.identity.Party
 import net.corda.core.internal.FlowStateMachine
 import net.corda.core.utilities.UntrustworthyData
 
-class FlowSessionImpl(
+class FlowSessionImpl(override val counterparty: Party) : FlowSession() {
-        override val counterparty: Party
-) : FlowSession() {
-
     internal lateinit var stateMachine: FlowStateMachine<*>
     internal lateinit var sessionFlow: FlowLogic<*>
 
@@ -57,5 +54,7 @@ class FlowSessionImpl(
 
     @Suspendable
     override fun send(payload: Any) = send(payload, maySkipCheckpoint = false)
+
+    override fun toString() = "Flow session with $counterparty"
 }
 

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -13,6 +13,7 @@ import net.corda.core.node.services.vault.QueryCriteria
 import net.corda.core.node.services.vault.Sort
 import net.corda.core.node.services.vault.SortAttribute
 import net.corda.core.messaging.DataFeed
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.VaultQueryException
 import net.corda.core.node.services.vault.*
 import net.corda.core.schemas.PersistentStateRef
@@ -50,8 +51,7 @@ private fun CriteriaBuilder.executeUpdate(session: Session, configure: Root<*>.(
 }
 
 /**
- * Currently, the node vault service is a very simple RDBMS backed implementation.  It will change significantly when
+ * The vault service handles storage, retrieval and querying of states.
- * we add further functionality as the design for the vault and vault service matures.
  *
  * This class needs database transactions to be in-flight during method calls and init, and will throw exceptions if
  * this is not the case.
@@ -59,8 +59,12 @@ private fun CriteriaBuilder.executeUpdate(session: Session, configure: Root<*>.(
  * TODO: keep an audit trail with time stamps of previously unconsumed states "as of" a particular point in time.
  * TODO: have transaction storage do some caching.
  */
-class NodeVaultService(private val clock: Clock, private val keyManagementService: KeyManagementService, private val stateLoader: StateLoader, hibernateConfig: HibernateConfiguration) : SingletonSerializeAsToken(), VaultServiceInternal {
+class NodeVaultService(
-
+        private val clock: Clock,
+        private val keyManagementService: KeyManagementService,
+        private val stateLoader: StateLoader,
+        hibernateConfig: HibernateConfiguration
+) : SingletonSerializeAsToken(), VaultServiceInternal {
     private companion object {
         val log = loggerFor<NodeVaultService>()
     }
@@ -118,7 +122,10 @@ class NodeVaultService(private val clock: Clock, private val keyManagementServic
     override val updates: Observable<Vault.Update<ContractState>>
         get() = mutex.locked { _updatesInDbTx }
 
-    override fun notifyAll(txns: Iterable<CoreTransaction>) {
+    override fun notifyAll(statesToRecord: StatesToRecord, txns: Iterable<CoreTransaction>) {
+        if (statesToRecord == StatesToRecord.NONE)
+            return
+
         // It'd be easier to just group by type, but then we'd lose ordering.
         val regularTxns = mutableListOf<WireTransaction>()
         val notaryChangeTxns = mutableListOf<NotaryChangeWireTransaction>()
@@ -128,30 +135,33 @@ class NodeVaultService(private val clock: Clock, private val keyManagementServic
                 is WireTransaction -> {
                     regularTxns.add(tx)
                     if (notaryChangeTxns.isNotEmpty()) {
-                        notifyNotaryChange(notaryChangeTxns.toList())
+                        notifyNotaryChange(notaryChangeTxns.toList(), statesToRecord)
                         notaryChangeTxns.clear()
                     }
                 }
                 is NotaryChangeWireTransaction -> {
                     notaryChangeTxns.add(tx)
                     if (regularTxns.isNotEmpty()) {
-                        notifyRegular(regularTxns.toList())
+                        notifyRegular(regularTxns.toList(), statesToRecord)
                         regularTxns.clear()
                     }
                 }
             }
         }
 
-        if (regularTxns.isNotEmpty()) notifyRegular(regularTxns.toList())
+        if (regularTxns.isNotEmpty()) notifyRegular(regularTxns.toList(), statesToRecord)
-        if (notaryChangeTxns.isNotEmpty()) notifyNotaryChange(notaryChangeTxns.toList())
+        if (notaryChangeTxns.isNotEmpty()) notifyNotaryChange(notaryChangeTxns.toList(), statesToRecord)
     }
 
-    private fun notifyRegular(txns: Iterable<WireTransaction>) {
+    private fun notifyRegular(txns: Iterable<WireTransaction>, statesToRecord: StatesToRecord) {
         fun makeUpdate(tx: WireTransaction): Vault.Update<ContractState> {
             val myKeys = keyManagementService.filterMyKeys(tx.outputs.flatMap { it.data.participants.map { it.owningKey } })
-            val ourNewStates = tx.outputs.
+
-                    filter { isRelevant(it.data, myKeys.toSet()) }.
+            val ourNewStates = when (statesToRecord) {
-                    map { tx.outRef<ContractState>(it.data) }
+                StatesToRecord.NONE -> throw AssertionError("Should not reach here")
+                StatesToRecord.ONLY_RELEVANT -> tx.outputs.filter { isRelevant(it.data, myKeys.toSet()) }
+                StatesToRecord.ALL_VISIBLE -> tx.outputs
+            }.map { tx.outRef<ContractState>(it.data) }
 
             // Retrieve all unconsumed states for this transaction's inputs
             val consumedStates = loadStates(tx.inputs)
@@ -169,7 +179,7 @@ class NodeVaultService(private val clock: Clock, private val keyManagementServic
         processAndNotify(netDelta)
     }
 
-    private fun notifyNotaryChange(txns: Iterable<NotaryChangeWireTransaction>) {
+    private fun notifyNotaryChange(txns: Iterable<NotaryChangeWireTransaction>, statesToRecord: StatesToRecord) {
         fun makeUpdate(tx: NotaryChangeWireTransaction): Vault.Update<ContractState> {
             // We need to resolve the full transaction here because outputs are calculated from inputs
             // We also can't do filtering beforehand, since output encumbrance pointers get recalculated based on
@@ -178,7 +188,12 @@ class NodeVaultService(private val clock: Clock, private val keyManagementServic
             val myKeys = keyManagementService.filterMyKeys(ltx.outputs.flatMap { it.data.participants.map { it.owningKey } })
             val (consumedStateAndRefs, producedStates) = ltx.inputs.
                     zip(ltx.outputs).
-                    filter { (_, output) -> isRelevant(output.data, myKeys.toSet()) }.
+                    filter { (_, output) ->
+                        if (statesToRecord == StatesToRecord.ONLY_RELEVANT)
+                            isRelevant(output.data, myKeys.toSet())
+                        else
+                            true
+                    }.
                     unzip()
 
             val producedStateAndRefs = producedStates.map { ltx.outRef<ContractState>(it.data) }

node/src/test/kotlin/net/corda/node/services/events/NodeSchedulerServiceTest.kt

@@ -12,6 +12,7 @@ import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.node.NodeInfo
 import net.corda.core.node.ServiceHub
+import net.corda.core.node.StatesToRecord
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.TransactionBuilder
@@ -107,11 +108,12 @@ class NodeSchedulerServiceTest : SingletonSerializeAsToken() {
                 doReturn(myInfo).whenever(it).myInfo
                 doReturn(kms).whenever(it).keyManagementService
                 doReturn(CordappProviderImpl(CordappLoader.createWithTestPackages(listOf("net.corda.testing.contracts")), MockAttachmentStorage())).whenever(it).cordappProvider
-                doCallRealMethod().whenever(it).recordTransactions(any<SignedTransaction>())
+                doCallRealMethod().whenever(it).recordTransactions(any<StatesToRecord>(), any<Iterable<SignedTransaction>>())
-                doCallRealMethod().whenever(it).recordTransactions(any<Boolean>(), any<SignedTransaction>())
+                doCallRealMethod().whenever(it).recordTransactions(any<Iterable<SignedTransaction>>())
-                doCallRealMethod().whenever(it).recordTransactions(any(), any<Iterable<SignedTransaction>>())
+                doCallRealMethod().whenever(it).recordTransactions(any<SignedTransaction>(), anyVararg<SignedTransaction>())
                 doReturn(NodeVaultService(testClock, kms, stateLoader, database.hibernateConfig)).whenever(it).vaultService
                 doReturn(this@NodeSchedulerServiceTest).whenever(it).testReference
+
             }
             smmExecutor = AffinityExecutor.ServiceAffinityExecutor("test", 1)
             mockSMM = StateMachineManagerImpl(services, DBCheckpointStorage(), smmExecutor, database)

node/src/test/kotlin/net/corda/node/services/persistence/DBTransactionStorageTests.kt

@@ -5,13 +5,13 @@ import net.corda.core.crypto.Crypto
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.SignatureMetadata
 import net.corda.core.crypto.TransactionSignature
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.VaultService
 import net.corda.core.toFuture
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.WireTransaction
 import net.corda.node.services.api.VaultServiceInternal
 import net.corda.node.services.schema.HibernateObserver
-import net.corda.node.services.schema.NodeSchemaService
 import net.corda.node.services.transactions.PersistentUniquenessProvider
 import net.corda.node.services.vault.NodeVaultService
 import net.corda.node.utilities.CordaPersistence
@@ -40,7 +40,6 @@ class DBTransactionStorageTests : TestDependencyInjectionBase() {
         val dataSourceProps = makeTestDataSourceProperties()
         database = configureDatabase(dataSourceProps, makeTestDatabaseProperties(), ::makeTestIdentityService)
         database.transaction {
-
             services = object : MockServices(BOB_KEY) {
                 override val vaultService: VaultServiceInternal
                     get() {
@@ -54,7 +53,7 @@ class DBTransactionStorageTests : TestDependencyInjectionBase() {
                         validatedTransactions.addTransaction(stx)
                     }
                     // Refactored to use notifyAll() as we have no other unit test for that method with multiple transactions.
-                    vaultService.notifyAll(txs.map { it.tx })
+                    vaultService.notifyAll(StatesToRecord.ONLY_RELEVANT, txs.map { it.tx })
                 }
             }
         }

node/src/test/kotlin/net/corda/node/services/persistence/HibernateConfigurationTest.kt

@@ -5,6 +5,7 @@ import net.corda.core.contracts.StateAndRef
 import net.corda.core.contracts.StateRef
 import net.corda.core.contracts.TransactionState
 import net.corda.core.crypto.SecureHash
+import net.corda.core.node.StatesToRecord
 import net.corda.core.utilities.toBase58String
 import net.corda.core.node.services.Vault
 import net.corda.core.node.services.VaultService
@@ -82,12 +83,12 @@ class HibernateConfigurationTest : TestDependencyInjectionBase() {
             hibernateConfig = database.hibernateConfig
             services = object : MockServices(cordappPackages, BOB_KEY, BOC_KEY, DUMMY_NOTARY_KEY) {
                 override val vaultService = makeVaultService(database.hibernateConfig)
-                override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+                override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
                     for (stx in txs) {
                         validatedTransactions.addTransaction(stx)
                     }
                     // Refactored to use notifyAll() as we have no other unit test for that method with multiple transactions.
-                    vaultService.notifyAll(txs.map { it.tx })
+                    vaultService.notifyAll(statesToRecord, txs.map { it.tx })
                 }
 
                 override fun jdbcSession() = database.createSession()

node/src/test/kotlin/net/corda/node/services/statemachine/FlowFrameworkTests.kt

@@ -145,17 +145,7 @@ class FlowFrameworkTests {
         val restoredFlow = bobNode.restartAndGetRestoredFlow<InitiatedReceiveFlow>()
         assertThat(restoredFlow.receivedPayloads[0]).isEqualTo("Hello")
     }
-
+    
-    @Test
-    fun `flow added before network map does run after init`() {
-        val charlieNode = mockNet.createNode() //create vanilla node
-        val flow = NoOpFlow()
-        charlieNode.services.startFlow(flow)
-        assertEquals(false, flow.flowStarted) // Not started yet as no network activity has been allowed yet
-        mockNet.runNetwork() // Allow network map messages to flow
-        assertEquals(true, flow.flowStarted) // Now we should have run the flow
-    }
-
     @Test
     fun `flow loaded from checkpoint will respond to messages from before start`() {
         aliceNode.registerFlowFactory(ReceiveFlow::class) { InitiatedSendFlow("Hello", it) }

node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt

@@ -1,16 +1,15 @@
 package net.corda.node.services.vault
 
 import co.paralleluniverse.fibers.Suspendable
-import net.corda.core.contracts.Amount
+import net.corda.core.contracts.*
-import net.corda.core.contracts.Issued
-import net.corda.core.contracts.StateAndRef
-import net.corda.core.contracts.StateRef
 import net.corda.core.crypto.generateKeyPair
 import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.AnonymousParty
 import net.corda.core.identity.Party
 import net.corda.core.internal.packageName
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.*
+import net.corda.core.node.services.vault.PageSpecification
 import net.corda.core.node.services.vault.QueryCriteria
 import net.corda.core.node.services.vault.QueryCriteria.*
 import net.corda.core.transactions.NotaryChangeWireTransaction
@@ -30,7 +29,6 @@ import net.corda.node.utilities.CordaPersistence
 import net.corda.testing.*
 import net.corda.testing.contracts.fillWithSomeTestCash
 import net.corda.testing.node.MockServices
-import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndMockServices
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatExceptionOfType
 import org.junit.After
@@ -58,8 +56,10 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
     @Before
     fun setUp() {
         LogHelper.setLevel(NodeVaultService::class)
-        val databaseAndServices = makeTestDatabaseAndMockServices(keys = listOf(BOC_KEY, DUMMY_CASH_ISSUER_KEY),
+        val databaseAndServices = MockServices.makeTestDatabaseAndMockServices(
-                cordappPackages = cordappPackages)
+                keys = listOf(BOC_KEY, DUMMY_CASH_ISSUER_KEY),
+                cordappPackages = cordappPackages
+        )
         database = databaseAndServices.first
         services = databaseAndServices.second
         issuerServices = MockServices(cordappPackages, DUMMY_CASH_ISSUER_KEY, BOC_KEY)
@@ -102,10 +102,10 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
             val originalVault = vaultService
             val services2 = object : MockServices() {
                 override val vaultService: NodeVaultService get() = originalVault
-                override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+                override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
                     for (stx in txs) {
                         validatedTransactions.addTransaction(stx)
-                        vaultService.notify(stx.tx)
+                        vaultService.notify(statesToRecord, stx.tx)
                     }
                 }
             }
@@ -512,14 +512,14 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
         }.toWireTransaction(services)
         val cashState = StateAndRef(issueTx.outputs.single(), StateRef(issueTx.id, 0))
 
-        database.transaction { service.notify(issueTx) }
+        database.transaction { service.notify(StatesToRecord.ONLY_RELEVANT, issueTx) }
         val expectedIssueUpdate = Vault.Update(emptySet(), setOf(cashState), null)
 
         database.transaction {
             val moveTx = TransactionBuilder(services.myInfo.chooseIdentity()).apply {
                 Cash.generateSpend(services, this, Amount(1000, GBP), thirdPartyIdentity)
             }.toWireTransaction(services)
-            service.notify(moveTx)
+            service.notify(StatesToRecord.ONLY_RELEVANT, moveTx)
         }
         val expectedMoveUpdate = Vault.Update(setOf(cashState), emptySet(), null)
 
@@ -556,7 +556,7 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
         val cashStateWithNewNotary = StateAndRef(initialCashState.state.copy(notary = newNotary), StateRef(changeNotaryTx.id, 0))
 
         database.transaction {
-            service.notifyAll(listOf(issueStx.tx, changeNotaryTx))
+            service.notifyAll(StatesToRecord.ONLY_RELEVANT, listOf(issueStx.tx, changeNotaryTx))
         }
 
         // Move cash
@@ -567,7 +567,7 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
         }
 
         database.transaction {
-            service.notify(moveTx)
+            service.notify(StatesToRecord.ONLY_RELEVANT, moveTx)
         }
 
         val expectedIssueUpdate = Vault.Update(emptySet(), setOf(initialCashState), null)
@@ -577,4 +577,32 @@ class NodeVaultServiceTest : TestDependencyInjectionBase() {
         val observedUpdates = vaultSubscriber.onNextEvents
         assertEquals(observedUpdates, listOf(expectedIssueUpdate, expectedNotaryChangeUpdate, expectedMoveUpdate))
     }
+
+    @Test
+    fun observerMode() {
+        fun countCash(): Long {
+            return database.transaction {
+                vaultService.queryBy(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(), PageSpecification(1)).totalStatesAvailable
+            }
+        }
+        val currentCashStates = countCash()
+
+        // Send some minimalist dummy transaction.
+        val txb = TransactionBuilder(DUMMY_NOTARY)
+        txb.addOutputState(Cash.State(MEGA_CORP.ref(0), 100.DOLLARS, MINI_CORP), Cash::class.java.name)
+        txb.addCommand(Cash.Commands.Move(), MEGA_CORP_PUBKEY)
+        val wtx = txb.toWireTransaction(services)
+        database.transaction {
+            vaultService.notify(StatesToRecord.ONLY_RELEVANT, wtx)
+        }
+
+        // Check that it was ignored as irrelevant.
+        assertEquals(currentCashStates, countCash())
+
+        // Now try again and check it was accepted.
+        database.transaction {
+            vaultService.notify(StatesToRecord.ALL_VISIBLE, wtx)
+        }
+        assertEquals(currentCashStates + 1, countCash())
+    }
 }

samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/AutoOfferFlow.kt

@@ -2,8 +2,10 @@ package net.corda.irs.flows
 
 import co.paralleluniverse.fibers.Suspendable
 import net.corda.core.flows.*
+import net.corda.core.identity.Party
 import net.corda.core.identity.excludeHostNode
 import net.corda.core.identity.groupAbstractPartyByWellKnownParty
+import net.corda.core.node.StatesToRecord
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.utilities.ProgressTracker
 import net.corda.finance.contracts.DealState
@@ -23,7 +25,6 @@ object AutoOfferFlow {
     @InitiatingFlow
     @StartableByRPC
     class Requester(val dealToBeOffered: DealState) : FlowLogic<SignedTransaction>() {
-
         companion object {
             object RECEIVED : ProgressTracker.Step("Received API call")
             object DEALING : ProgressTracker.Step("Starting the deal flow") {
@@ -55,11 +56,45 @@ object AutoOfferFlow {
                     AutoOffer(notary, dealToBeOffered),
                     progressTracker.getChildProgressTracker(DEALING)!!
             )
-            val stx = subFlow(instigator)
+            return subFlow(instigator)
-            return stx
         }
     }
 
+    // DOCSTART 1
     @InitiatedBy(Requester::class)
-    class AutoOfferAcceptor(otherSideSession: FlowSession) : Acceptor(otherSideSession)
+    class AutoOfferAcceptor(otherSideSession: FlowSession) : Acceptor(otherSideSession) {
+        @Suspendable
+        override fun call(): SignedTransaction {
+            val finalTx = super.call()
+            // Our transaction is now committed to the ledger, so report it to our regulator. We use a custom flow
+            // that wraps SendTransactionFlow to allow the receiver to customise how ReceiveTransactionFlow is run,
+            // and because in a real life app you'd probably have more complex logic here e.g. describing why the report
+            // was filed, checking that the reportee is a regulated entity and not some random node from the wrong
+            // country and so on.
+            val regulator = serviceHub.identityService.partiesFromName("Regulator", true).single()
+            subFlow(ReportToRegulatorFlow(regulator, finalTx))
+            return finalTx
+        }
+
+    }
+
+    @InitiatingFlow
+    class ReportToRegulatorFlow(private val regulator: Party, private val finalTx: SignedTransaction) : FlowLogic<Unit>() {
+        @Suspendable
+        override fun call() {
+            val session = initiateFlow(regulator)
+            subFlow(SendTransactionFlow(session, finalTx))
+        }
+    }
+
+    @InitiatedBy(ReportToRegulatorFlow::class)
+    class ReceiveRegulatoryReportFlow(private val otherSideSession: FlowSession) : FlowLogic<Unit>() {
+        @Suspendable
+        override fun call() {
+            // Start the matching side of SendTransactionFlow above, but tell it to record all visible states even
+            // though they (as far as the node can tell) are nothing to do with us.
+            subFlow(ReceiveTransactionFlow(otherSideSession, true, StatesToRecord.ALL_VISIBLE))
+        }
+    }
+    // DOCEND 1
 }

testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt

@@ -9,10 +9,7 @@ import net.corda.core.identity.PartyAndCertificate
 import net.corda.core.messaging.DataFeed
 import net.corda.core.messaging.FlowHandle
 import net.corda.core.messaging.FlowProgressHandle
-import net.corda.core.node.AppServiceHub
+import net.corda.core.node.*
-import net.corda.core.node.NodeInfo
-import net.corda.core.node.ServiceHub
-import net.corda.core.node.StateLoader
 import net.corda.core.node.services.*
 import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.serialization.SingletonSerializeAsToken
@@ -30,7 +27,6 @@ import net.corda.node.services.persistence.HibernateConfiguration
 import net.corda.node.services.persistence.InMemoryStateMachineRecordedTransactionMappingStorage
 import net.corda.node.services.schema.HibernateObserver
 import net.corda.node.services.schema.NodeSchemaService
-import net.corda.node.services.statemachine.FlowStateMachineImpl
 import net.corda.node.services.transactions.InMemoryTransactionVerifierService
 import net.corda.node.services.vault.NodeVaultService
 import net.corda.node.utilities.CordaPersistence
@@ -101,7 +97,7 @@ open class MockServices(
 
         /**
          * Makes database and mock services appropriate for unit tests.
-         * @param keys a list of [KeyPair] instances to be used by [MockServices]. Defualts to [MEGA_CORP_KEY]
+         * @param keys a list of [KeyPair] instances to be used by [MockServices]. Defaults to [MEGA_CORP_KEY]
          * @param createIdentityService a lambda function returning an instance of [IdentityService]. Defauts to [InMemoryIdentityService].
          *
          * @return a pair where the first element is the instance of [CordaPersistence] and the second is [MockServices].
@@ -118,14 +114,14 @@ open class MockServices(
             val mockService = database.transaction {
                 object : MockServices(cordappLoader, *(keys.toTypedArray())) {
                     override val identityService: IdentityService = database.transaction { identityServiceRef }
-                    override val vaultService = makeVaultService(database.hibernateConfig)
+                    override val vaultService: VaultServiceInternal = makeVaultService(database.hibernateConfig)
 
-                    override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+                    override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
                         for (stx in txs) {
                             validatedTransactions.addTransaction(stx)
                         }
                         // Refactored to use notifyAll() as we have no other unit test for that method with multiple transactions.
-                        vaultService.notifyAll(txs.map { it.tx })
+                        vaultService.notifyAll(statesToRecord, txs.map { it.tx })
                     }
 
                     override fun jdbcSession(): Connection = database.createSession()
@@ -142,7 +138,7 @@ open class MockServices(
 
     val key: KeyPair get() = keys.first()
 
-    override fun recordTransactions(notifyVault: Boolean, txs: Iterable<SignedTransaction>) {
+    override fun recordTransactions(statesToRecord: StatesToRecord, txs: Iterable<SignedTransaction>) {
         txs.forEach {
             stateMachineRecordedTransactionMapping.addMapping(StateMachineRunId.createRandom(), it.id)
         }

verifier/build.gradle

@@ -33,6 +33,10 @@ dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
     compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
     testCompile "org.jetbrains.kotlin:kotlin-test:$kotlin_version"
+
+    // TODO: remove the forced update of commons-collections and beanutils when artemis updates them
+    compile "org.apache.commons:commons-collections4:${commons_collections_version}"
+    compile "commons-beanutils:commons-beanutils:${beanutils_version}"
     compile "org.apache.activemq:artemis-core-client:${artemis_version}"
 
     // Log4J: logging framework (with SLF4J bindings)

webserver/build.gradle

@@ -49,6 +49,9 @@ dependencies {
     compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
 
     // Jersey for JAX-RS implementation for use in Jetty
+    // TODO: remove force upgrade when jersey catches up
+    compile "org.eclipse.jetty:jetty-continuation:${jetty_version}"
+
     compile "org.glassfish.jersey.core:jersey-server:$jersey_version"
     compile "org.glassfish.jersey.containers:jersey-container-servlet-core:$jersey_version"
     compile "org.glassfish.jersey.containers:jersey-container-jetty-http:$jersey_version"