diff --git a/docs/build/doctrees/building-the-docs.doctree b/docs/build/doctrees/building-the-docs.doctree
index 2a8c445d6e..bad1d66025 100644
Binary files a/docs/build/doctrees/building-the-docs.doctree and b/docs/build/doctrees/building-the-docs.doctree differ
diff --git a/docs/build/doctrees/creating-a-cordapp.doctree b/docs/build/doctrees/creating-a-cordapp.doctree
index cb0b8984ea..a0ff32e307 100644
Binary files a/docs/build/doctrees/creating-a-cordapp.doctree and b/docs/build/doctrees/creating-a-cordapp.doctree differ
diff --git a/docs/build/doctrees/environment.pickle b/docs/build/doctrees/environment.pickle
index a8da3a4318..8cdecd465f 100644
Binary files a/docs/build/doctrees/environment.pickle and b/docs/build/doctrees/environment.pickle differ
diff --git a/docs/build/doctrees/index.doctree b/docs/build/doctrees/index.doctree
index 415ec046ca..32026912ca 100644
Binary files a/docs/build/doctrees/index.doctree and b/docs/build/doctrees/index.doctree differ
diff --git a/docs/build/doctrees/merkle-trees.doctree b/docs/build/doctrees/merkle-trees.doctree
new file mode 100644
index 0000000000..8e199ca505
Binary files /dev/null and b/docs/build/doctrees/merkle-trees.doctree differ
diff --git a/docs/build/doctrees/running-the-demos.doctree b/docs/build/doctrees/running-the-demos.doctree
index 24bcbe8e63..33616c673d 100644
Binary files a/docs/build/doctrees/running-the-demos.doctree and b/docs/build/doctrees/running-the-demos.doctree differ
diff --git a/docs/build/html/.buildinfo b/docs/build/html/.buildinfo
index d958f98faa..6c71bb6975 100644
--- a/docs/build/html/.buildinfo
+++ b/docs/build/html/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 02da61908148262295ef57918c434b8d
+config: ad4aa434cfebd269fdbb85815bc6eb43
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/build/html/_images/merkleTree.png b/docs/build/html/_images/merkleTree.png
new file mode 100644
index 0000000000..696f510450
Binary files /dev/null and b/docs/build/html/_images/merkleTree.png differ
diff --git a/docs/build/html/_images/partialMerkle.png b/docs/build/html/_images/partialMerkle.png
new file mode 100644
index 0000000000..a2c5af7a23
Binary files /dev/null and b/docs/build/html/_images/partialMerkle.png differ
diff --git a/docs/build/html/_sources/creating-a-cordapp.txt b/docs/build/html/_sources/creating-a-cordapp.txt
index c208cd90f2..ef347bc690 100644
--- a/docs/build/html/_sources/creating-a-cordapp.txt
+++ b/docs/build/html/_sources/creating-a-cordapp.txt
@@ -197,7 +197,7 @@ Cordformation is the local node deployment system for Cordapps, the nodes genera
experimenting, debugging, and testing node configurations and setups but not intended for production or testnet
deployment.
-To use this gradle plugin you must add a new task that is of the type `com.r3corda.plugins.Cordform` to your
+To use this gradle plugin you must add a new task that is of the type ``com.r3corda.plugins.Cordform`` to your
build.gradle and then configure the nodes you wish to deploy with the Node and nodes configuration DSL.
This DSL is specified in the `JavaDoc `_. An example of this is in the template-cordapp and below
is a three node example;
@@ -211,8 +211,7 @@ is a three node example;
name "Controller"
dirName "controller"
nearestCity "London"
- notary true // Sets this node to be a notary
- advertisedServices []
+ advertisedServices = [ "corda.notary.validating" ]
artemisPort 12345
webPort 12346
cordapps []
@@ -221,7 +220,7 @@ is a three node example;
name "NodeA"
dirName "nodea"
nearestCity "London"
- advertisedServices []
+ advertisedServices = []
artemisPort 31337
webPort 31339
cordapps []
@@ -230,7 +229,7 @@ is a three node example;
name "NodeB"
dirName "nodeb"
nearestCity "New York"
- advertisedServices []
+ advertisedServices = []
artemisPort 31338
webPort 31340
cordapps []
@@ -244,4 +243,4 @@ run this task it will install the nodes to the directory specified and a script
at present) to run the nodes with one command.
Other cordapps can also be specified if they are already specified as classpath or compile dependencies in your
-build.gradle.
\ No newline at end of file
+``build.gradle``.
diff --git a/docs/build/html/_sources/index.txt b/docs/build/html/_sources/index.txt
index b10f28379c..ca54315def 100644
--- a/docs/build/html/_sources/index.txt
+++ b/docs/build/html/_sources/index.txt
@@ -28,6 +28,7 @@ Read on to learn:
getting-set-up
data-model
transaction-data-types
+ merkle-trees
consensus
messaging
persistence
diff --git a/docs/build/html/_sources/merkle-trees.txt b/docs/build/html/_sources/merkle-trees.txt
new file mode 100644
index 0000000000..1b30b68df3
--- /dev/null
+++ b/docs/build/html/_sources/merkle-trees.txt
@@ -0,0 +1,111 @@
+Transaction Tear-offs
+======================
+
+One of the basic data structures in our platform is a transaction. It can be passed around to be signed and verified,
+also by third parties. The construction of transactions assumes that they form a whole entity with input and output states,
+commands and attachments inside. However all sensitive data shouldn’t be revealed to other nodes that take part in
+the creation of transaction on validation level (a good example of this situation is the Oracle which validates only
+embedded commands). How to achive it in a way that convinces the other party the data they got for signing really did form
+a part of the transaction?
+
+We decided to use well known and described cryptographic scheme to provide proofs of inclusion and data integrity.
+Merkle trees are widely used in peer-to-peer networks, blockchain systems and git.
+You can read more on the concept `here `_.
+
+Merkle trees in Corda
+----------------------
+
+Transactions are split into leaves, each of them contains either input, output, command or attachment. Other fields like
+timestamp or signers are not used in the calculation.
+Next, the Merkle tree is built in the normal way by hashing the concatenation
+of nodes’ hashes below the current one together. It’s visible on the example image below, where ``H`` denotes sha256 function,
+"+" - concatenation.
+
+.. image:: resources/merkleTree.png
+
+The transaction has one input state, one output and three commands. If a tree is not a full binary tree, the rightmost nodes are
+duplicated in hash calculation (dotted lines).
+
+Finally, the hash of the root is the identifier of the transaction, it's also used for signing and verification of data integrity.
+Every change in transaction on a leaf level will change its identifier.
+
+Hiding data
+-----------
+
+Hiding data and providing the proof that it formed a part of a transaction is done by constructing Partial Merkle Trees
+(or Merkle branches). A Merkle branch is a set of hashes, that given the leaves’ data, is used to calculate the root’s hash.
+Then that hash is compared with the hash of a whole transaction and if they match it means that data we obtained belongs
+to that particular transaction.
+
+.. image:: resources/partialMerkle.png
+
+In the example above, the red node is the one holding data for signing Oracle service. Blue nodes' hashes form the Partial Merkle
+Tree, dotted ones are not included. Having the command that should be in a red node place and branch we are able to calculate
+root of this tree and compare it with original transaction identifier - we have a proof that this command belongs to this transaction.
+
+Example of usage
+-----------------
+
+Let’s focus on a code example. We want to construct a transaction with commands containing interest rate fix data as in:
+:doc:`oracles`.
+After construction of a partial transaction, with included ``Fix`` commands in it, we want to send it to the Oracle for checking
+and signing. To do so we need to specify which parts of the transaction are going to be revealed. That can be done by constructing
+filtering functions for inputs, outputs, attachments and commands separately. If a function is not provided by default none
+of the elements from this group will be included in a Partial Merkle Tree.
+
+.. container:: codeset
+
+ .. sourcecode:: kotlin
+
+ val partialTx = ...
+ val oracle: Party = ...
+ fun filterCommands(c: Command) = oracle.owningKey in c.signers && c.value is Fix
+ val filterFuns = FilterFuns(filterCommands = ::filterCommands)
+
+Assuming that we already assembled partialTx with some commands and know the identity of Oracle service,
+we pass filtering function over commands - ``filterCommands`` to ``FilterFuns``. It filters only
+commands of type ``Fix`` as in IRSDemo example. Then we can construct ``FilteredTransaction``:
+
+.. container:: codeset
+
+ .. sourcecode:: kotlin
+
+ val wtx: WireTransaction = partialTx.toWireTransaction()
+ val ftx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
+
+In the Oracle example this step takes place in ``RatesFixProtocol``:
+
+.. container:: codeset
+
+ .. sourcecode:: kotlin
+
+ val protocol = RatesFixProtocol(partialTx, filterFuns, oracle, fixOf, "0.675".bd, "0.1".bd)
+
+``FilteredTransaction`` holds ``filteredLeaves`` (data that we wanted to reveal) and Merkle branch for them.
+
+.. container:: codeset
+
+ .. sourcecode:: kotlin
+
+ // Getting included commands, inputs, outputs, attachments.
+ val cmds: List = ftx.filteredLeaves.commands
+ val ins: List = ftx.filteredLeaves.inputs
+ val outs: List> = ftx.filteredLeaves.outputs
+ val attchs: List = ftx.filteredLeaves.attachments
+
+
+If you want to verify obtained ``FilteredTransaction`` all you need is the root hash of the full transaction:
+
+.. container:: codeset
+
+ .. sourcecode:: kotlin
+
+ if (!ftx.verify(merkleRoot)){
+ throw MerkleTreeException("Rate Fix Oracle: Couldn't verify partial Merkle tree.")
+ }
+
+
+.. note:: The way the ``FilteredTransaction`` is constructed ensures that after signing of the root hash it's impossible to add or remove
+ leaves. However, it can happen that having transaction with multiple commands one party reveals only subset of them to the Oracle.
+ As signing is done now over the merkle root hash, the service signs all commands of given type, even though it didn't see
+ all of them. This issue will be handled after implementing partial signatures.
diff --git a/docs/build/html/_sources/running-the-demos.txt b/docs/build/html/_sources/running-the-demos.txt
index 2d3fc0e1b2..e6a75b6766 100644
--- a/docs/build/html/_sources/running-the-demos.txt
+++ b/docs/build/html/_sources/running-the-demos.txt
@@ -9,171 +9,57 @@ so far. We have:
2. The IRS demo, which shows two nodes establishing an interest rate swap between them and performing fixings with a
rates oracle, all driven via the HTTP API.
3. The IRS demo web interface - a web interface to the IRS demo.
+4. The attachment demo, which demonstrates uploading attachments to nodes.
+5. The SIMM valuation demo, a large demo which shows two nodes agreeing on a portfolio and valuing the initial margin
+ using the Standard Initial Margin Model.
-The demos create node data directories in the root of the project. If something goes wrong with them, blow away the
-directories and try again.
+.. note:: The demos currently must be run from IntelliJ, this will change before M6.
+
+.. note:: If any demos don't work please jump on our mailing list and let us know.
Trader demo
-----------
-Open two terminals, and in the first run:
-
-.. note:: If you are planning to use non-default configuration you will need to run with --role=SetupA and --role=SetupB
- beforehand with the same parameters you plan to supply to the respective nodes.
-
-**Windows**::
-
- gradlew.bat & .\build\install\r3prototyping\bin\trader-demo --role=BUYER
-
-**Other**::
-
- ./gradlew installDist && ./build/install/r3prototyping/bin/trader-demo --role=BUYER
-
-It will compile things, if necessary, then create a directory named trader-demo/buyer with a bunch of files inside and
-start the node. You should see it waiting for a trade to begin.
-
-In the second terminal, run:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\trader-demo --role=SELLER
-
-**Other**::
-
- ./build/install/r3prototyping/bin/trader-demo --role=SELLER
-
-You should see some log lines scroll past, and within a few seconds the messages "Purchase complete - we are a
-happy customer!" and "Sale completed - we have a happy customer!" should be printed.
-
-If it doesn't work, jump on the mailing list and let us know.
+1. Open the Corda project in IntelliJ and run the "Install" configuration
+2. Open the Corda samples project in IntelliJ and run the "Trader Demo: Run Nodes" configuration
+3. Run "Trader Demo: Run Buyer"
+4. Run "Trader Demo: Run Seller"
+In the "Trader Demo: Run Nodes" windows you should see some log lines scroll past, and within a few seconds the messages
+"Purchase complete - we are a happy customer!" and "Sale completed - we have a happy customer!" should be printed.
IRS demo
--------
-Open three terminals. In the first run:
-
-**Windows**::
-
- gradlew.bat installDist & .\build\install\r3prototyping\bin\irsdemo.bat --role=NodeA
-
-**Other**::
-
- ./gradlew installDist && ./build/install/r3prototyping/bin/irsdemo --role=NodeA
-
-And in the second run:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\irsdemo.bat --role=NodeB
-
-**Other**::
-
- ./build/install/r3prototyping/bin/irsdemo --role=NodeB
-
-NodeB also doubles up as the interest rates oracle and you should see some rates data get loaded.
-
-Now in the third terminal run:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\irsdemo.bat --role=Trade trade1
-
-**Other**::
-
- ./build/install/r3prototyping/bin/irsdemo --role=Trade trade1
-
-You should see some activity in the other two terminals as they set up the deal. You can now run this command in
-a separate window to roll the fake clock forward and trigger lots of fixing events. Things go fast so make sure you
-can see the other terminals whilst you run this command!:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\irsdemo.bat --role=Date 2017-01-30
-
-**Other**::
-
- ./build/install/r3prototyping/bin/irsdemo --role=Date 2017-01-30
+1. Open the Corda project in IntelliJ and run the "Install" configuration
+2. Open the Corda samples project in IntelliJ and run the "IRS Demo: Run Nodes" configuration
+3. Run "IRS Demo: Run Upload Rates" to upload rates to the oracle.
+4. Run "IRS Demo: Run Trade" to have nodes agree on a trade.
+5. Run "IRS Demo: Run Date Change" to run the fixings.
+In the "IRS Demo: Run Nodes" window you'll see a lot of activity when you run the trade and when you run the date change.
+The date change rolls the clock forwards and causes the nodes to agree on the fixings over a period.
IRS web demo
------------
-To install the web demo please follow these steps;
-
-1. Install Node: https://nodejs.org/en/download/ and ensure the npm executable is on your classpath
-2. Open a terminal
-3. Run `npm install -g bower` or `sudo npm install -g bower` if on a Unix system.
-4. In the terminal navigate to `/src/main/resources/com/r3corda/demos/irswebdemo`
-5. Run `bower install`
-
-To run the web demo, run the first two steps from the IRS Demo:
-
-Open two terminals and in the first:
-
-**Windows**::
-
- gradlew.bat installDist & .\build\install\r3prototyping\bin\irsdemo.bat --role=NodeA
-
-**Other**::
-
- ./gradlew installDist && ./build/install/r3prototyping/bin/irsdemo --role=NodeA
-
-And in the second run:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\irsdemo.bat --role=NodeB
-
-**Other**::
-
- ./build/install/r3prototyping/bin/irsdemo --role=NodeB
-
-Now open your web browser to this URL:
-
-.. note:: If using a custom node port address or port those must be used instead.
-
-**Node A**:
-
- http://localhost:31338/web/irsdemo
-
-**Node B**:
-
- http://localhost:31340/web/irsdemo
+There is also an IRS web demo installed. To use this follow steps 1-3 in the IRS demo and then navigate to
+http://localhost:10005/web/irsdemo and http://localhost:10005/web/irsdemo to see both node's view of the trades.
To use the demos click the "Create Deal" button, fill in the form, then click the "Submit" button. Now you will be
able to use the time controls at the top left of the home page to run the fixings. Click any individual trade in the
blotter to view it.
-
-
Attachment demo
-----------------
+---------------
-Open two terminals, and in the first run:
+1. Open the Corda project in IntelliJ and run the "Install" configuration
+2. Open the Corda samples project in IntelliJ and run the "Attachment Demo: Run Nodes" configuration
+3. Run "Attachment Demo: Run Recipient" - this waits for a trade to start
+4. Run "Attachment Demo: Run Sender" - sends the attachment
-**Windows**::
-
- gradlew.bat & .\build\install\r3prototyping\bin\attachment-demo --role=RECIPIENT
-
-**Other**::
-
- ./gradlew installDist && ./build/install/r3prototyping/bin/attachment-demo --role=RECIPIENT
-
-It will compile things, if necessary, then create a directory named attachment-demo/buyer with a bunch of files inside and
-start the node. You should see it waiting for a trade to begin.
-
-In the second terminal, run:
-
-**Windows**::
-
- .\build\install\r3prototyping\bin\attachment-demo --role=SENDER
-
-**Other**::
-
- ./build/install/r3prototyping/bin/attachment-demo --role=SENDER
-
-You should see some log lines scroll past, and within a few seconds the message "File received - we're happy!" should be printed.
+In the "Attachment Demo: Run Nodes" window you should see some log lines scroll past, and within a few seconds the
+message "File received - we're happy!" should be printed.
SIMM and Portfolio Demo
-----------------------
@@ -182,40 +68,9 @@ SIMM and Portfolio Demo
To run the demo run:
-**Windows**
+1. Open the Corda project in IntelliJ and run the "Install" configuration
+2. Open the Corda samples project in IntelliJ and run the "Simm Valuation Demo" configuration
-Open a terminal window and navigate to the root directory of Corda and run:
-
-.. sourcecode:: shell
-
- gradlew apps:vega:installTemplateNodes
- cd build\nodes\nameserver
- java -jar corda.jar
-
-Now open another terminal in the root directory of Corda and run:
-
-.. sourcecode:: shell
-
- cd build\nodes\nodea
- java -jar corda.jar
-
-Now open a third terminal in the root directory of Corda and run:
-
-.. sourcecode:: shell
-
- cd build\nodes\nodeb
- java -jar corda.jar
-
-**Other**
-
-Open one terminal window and run the following commands from the root directory of Corda:
-
-.. sourcecode:: shell
-
- ./gradlew apps:vega:installTemplateNodes
- cd build/nodes
- ./runnodes
-
-Now open http://localhost:31339/web/vega and http://localhost:31340/web/vega to view the two nodes that this
-will have started respectively.
+Now open http://localhost:10005/web/simmvaluationdemo and http://localhost:10005/web/simmvaluationdemo to view the two nodes that this
+will have started respectively. You can now use the demo by creating trades and agreeing the valuations.
diff --git a/docs/build/html/api/alltypes/index.html b/docs/build/html/api/alltypes/index.html
index bc87cd9131..a085e9d56c 100644
--- a/docs/build/html/api/alltypes/index.html
+++ b/docs/build/html/api/alltypes/index.html
@@ -2479,7 +2479,8 @@ a public key that is mentioned inside a transaction command. SignedTransaction i
and the type most frequently passed around the network and stored. The identity of a transaction is the hash
of a WireTransaction, therefore if you are storing data keyed by WT hash be aware that multiple different STs may
map to the same key (and they could be different in important ways, like validity). The signatures on a
-SignedTransaction might be invalid or missing: the type does not imply validity.
+SignedTransaction might be invalid or missing: the type does not imply validity.
+A transaction ID should be the hash of the WireTransaction Merkle tree root. Thus adding or removing a signature does not change it.
diff --git a/docs/build/html/api/com.r3corda.core.contracts/-named-by-hash/index.html b/docs/build/html/api/com.r3corda.core.contracts/-named-by-hash/index.html
index 60064a0f2c..d006131282 100644
--- a/docs/build/html/api/com.r3corda.core.contracts/-named-by-hash/index.html
+++ b/docs/build/html/api/com.r3corda.core.contracts/-named-by-hash/index.html
@@ -56,7 +56,8 @@ a public key that is mentioned inside a transaction command. SignedTransaction i
and the type most frequently passed around the network and stored. The identity of a transaction is the hash
of a WireTransaction, therefore if you are storing data keyed by WT hash be aware that multiple different STs may
map to the same key (and they could be different in important ways, like validity). The signatures on a
-SignedTransaction might be invalid or missing: the type does not imply validity.
+SignedTransaction might be invalid or missing: the type does not imply validity.
+A transaction ID should be the hash of the WireTransaction Merkle tree root. Thus adding or removing a signature does not change it.
diff --git a/docs/build/html/api/com.r3corda.core.transactions/-signed-transaction/-init-.html b/docs/build/html/api/com.r3corda.core.transactions/-signed-transaction/-init-.html
index e796a37fb4..ebd8529112 100644
--- a/docs/build/html/api/com.r3corda.core.transactions/-signed-transaction/-init-.html
+++ b/docs/build/html/api/com.r3corda.core.transactions/-signed-transaction/-init-.html
@@ -7,13 +7,14 @@
com.r3corda.core.transactions / SignedTransaction / <init>
<init>
-SignedTransaction(txBits: SerializedBytes<WireTransaction>, sigs: List<WithKey>)
+SignedTransaction(txBits: SerializedBytes<WireTransaction>, sigs: List<WithKey>, id: SecureHash)
SignedTransaction wraps a serialized WireTransaction. It contains one or more signatures, each one for
a public key that is mentioned inside a transaction command. SignedTransaction is the top level transaction type
and the type most frequently passed around the network and stored. The identity of a transaction is the hash
of a WireTransaction, therefore if you are storing data keyed by WT hash be aware that multiple different STs may
map to the same key (and they could be different in important ways, like validity). The signatures on a
-SignedTransaction might be invalid or missing: the type does not imply validity.
+SignedTransaction might be invalid or missing: the type does not imply validity.
+A transaction ID should be the hash of the WireTransaction Merkle tree root. Thus adding or removing a signature does not change it.