Tearoff fixes (#78)

* Move merkle building extension functions on wire tx to WireTransaction class.

* Add timestamp, notary, transaction type and signers to wire transaction id calculation.

* Change construction of MerkleTree from duplicating last node on a given level to padding leaves' list with zero hash to size of the nearest power of 2 - so we always have a full binary tree.
The problem was that it was possible to construct 2 different transactions with the same ids. Trick worked for txs having number of leaves that were not power of 2.

* Update tear-offs documentation and diagrams to reflect changes in construction of Merkle trees - padding with zero hashes and including all WireTransaction fields in id computation.

* Change in filtering API of WireTransaction for partial Merkle trees calculation.
Instead of many filtering functions over a transaction only one needs to be provided.
Additional change to check and verification of FilteredTransaction.

* IRS demo change. Make filtering function a protected method of RatesFixFlow class.
Comment on situation when capturing too much scope and connected problems with checkpointing.
Change oracle and tear-offs documentation.

core/src/main/kotlin/net/corda/core/crypto/PartialMerkleTree.kt

@@ -1,7 +1,7 @@
 package net.corda.core.crypto
 
 import net.corda.core.transactions.MerkleTree
-import net.corda.core.transactions.hashConcat
+import net.corda.core.crypto.SecureHash.Companion.zeroHash
 import java.util.*
 
 
@@ -19,14 +19,12 @@ class MerkleTreeException(val reason: String) : Exception() {
  *          /       \
  *         h14       h55
  *        /  \      /  \
- *      h12  h34   h5->d(h5)
+ *      h12  h34   h50 h00
- *     / \   / \   / \
+ *     / \   / \   / \  / \
- *    l1 l2 l3 l4 l5->d(l5)
+ *    l1 l2 l3 l4 l5 0 0  0
  *
- * l* denote hashes of leaves, h* - hashes of nodes below.
+ * l* denote hashes of leaves, h* - hashes of nodes below. 0 denotes zero hash, we use it to pad not full binary trees,
- * h5->d(h5) denotes duplication of the left hand side node. These nodes are kept in a full tree as DuplicatedLeaf.
+ * so the number of leaves is always a power of 2.
- * When filtering the tree for l5, we don't want to keep both l5 and its duplicate (it can also be solved using null
- * values in a tree, but this solution is clearer).
  *
  * Example of Partial tree based on the tree above.
  *
@@ -34,13 +32,13 @@ class MerkleTreeException(val reason: String) : Exception() {
  *          /       \
  *          _        _
  *        /  \      /  \
- *      h12   _     _   d(h5)
+ *      h12   _     _   h00
  *           / \   / \
- *          I3 l4 I5 d(l5)
+ *          I3 l4 I5 0
  *
  * We want to check l3 and l5 - now turned into IncudedLeaf (I3 and I5 above). To verify that these two leaves belong to
  * the tree with a hash root h15 we need to provide a Merkle branch (or partial tree). In our case we need hashes:
- * h12, l4, d(l5) and d(h5). Verification is done by hashing the partial tree to obtain the root and checking it against
+ * h12, l4, 0 and h00. Verification is done by hashing the partial tree to obtain the root and checking it against
  * the obtained h15 hash. Additionally we store included hashes used in calculation and compare them to leaves hashes we got
  * (there can be a difference in obtained leaves ordering - that's why it's a set comparison not hashing leaves into a tree).
  * If both equalities hold, we can assume that l3 and l5 belong to the transaction with root h15.
@@ -54,7 +52,7 @@ class PartialMerkleTree(val root: PartialTree) {
      * transaction and leaves that just keep hashes needed for calculation. Reason for this approach: during verification
      * it's easier to extract hashes used as a base for this tree.
      */
-    sealed class PartialTree() {
+    sealed class PartialTree {
         class IncludedLeaf(val hash: SecureHash) : PartialTree()
         class Leaf(val hash: SecureHash) : PartialTree()
         class Node(val left: PartialTree, val right: PartialTree) : PartialTree()
@@ -66,15 +64,31 @@ class PartialMerkleTree(val root: PartialTree) {
          * @param includeHashes Hashes that should be included in a partial tree.
          * @return Partial Merkle tree root.
          */
+        @Throws(IllegalArgumentException::class, MerkleTreeException::class)
         fun build(merkleRoot: MerkleTree, includeHashes: List<SecureHash>): PartialMerkleTree {
             val usedHashes = ArrayList<SecureHash>()
+            require(zeroHash !in includeHashes) { "Zero hashes shouldn't be included in partial tree." }
+            checkFull(merkleRoot) // Throws MerkleTreeException if it is not a full binary tree.
             val tree = buildPartialTree(merkleRoot, includeHashes, usedHashes)
-            //Too much included hashes or different ones.
+            // Too many included hashes or different ones.
             if (includeHashes.size != usedHashes.size)
                 throw MerkleTreeException("Some of the provided hashes are not in the tree.")
             return PartialMerkleTree(tree.second)
         }
 
+        // Check if a MerkleTree is full binary tree. Returns the height of the tree if full, otherwise throws exception.
+        private fun checkFull(tree: MerkleTree, level: Int = 0): Int {
+            return when (tree) {
+                is MerkleTree.Leaf -> level
+                is MerkleTree.Node -> {
+                    val l1 = checkFull(tree.left, level+1)
+                    val l2 = checkFull(tree.right, level+1)
+                    if (l1 != l2) throw MerkleTreeException("Got not full binary tree.")
+                    l1
+                }
+            }
+        }
+
         /**
          * @param root Root of full Merkle tree which is a base for a partial one.
          * @param includeHashes Hashes of leaves to be included in this partial tree.
@@ -93,18 +107,17 @@ class PartialMerkleTree(val root: PartialTree) {
                         usedHashes.add(root.value)
                         Pair(true, PartialTree.IncludedLeaf(root.value))
                     } else Pair(false, PartialTree.Leaf(root.value))
-                is MerkleTree.DuplicatedLeaf -> Pair(false, PartialTree.Leaf(root.value))
                 is MerkleTree.Node -> {
                     val leftNode = buildPartialTree(root.left, includeHashes, usedHashes)
                     val rightNode = buildPartialTree(root.right, includeHashes, usedHashes)
                     if (leftNode.first or rightNode.first) {
-                        //This node is on a path to some included leaves. Don't store hash.
+                        // This node is on a path to some included leaves. Don't store hash.
                         val newTree = PartialTree.Node(leftNode.second, rightNode.second)
-                        return Pair(true, newTree)
+                        Pair(true, newTree)
                     } else {
-                        //This node has no included leaves below. Cut the tree here and store a hash as a Leaf.
+                        // This node has no included leaves below. Cut the tree here and store a hash as a Leaf.
                         val newTree = PartialTree.Leaf(root.value)
-                        return Pair(false, newTree)
+                        Pair(false, newTree)
                     }
                 }
             }
@@ -118,7 +131,7 @@ class PartialMerkleTree(val root: PartialTree) {
     fun verify(merkleRootHash: SecureHash, hashesToCheck: List<SecureHash>): Boolean {
         val usedHashes = ArrayList<SecureHash>()
         val verifyRoot = verify(root, usedHashes)
-        //It means that we obtained more/less hashes than needed or different sets of hashes.
+        // It means that we obtained more/fewer hashes than needed or different sets of hashes.
         if (hashesToCheck.groupBy { it } != usedHashes.groupBy { it })
             return false
         return (verifyRoot == merkleRootHash)

core/src/main/kotlin/net/corda/core/crypto/SecureHash.kt

@@ -19,6 +19,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
     override fun toString() = BaseEncoding.base16().encode(bytes)
 
     fun prefixChars(prefixLen: Int = 6) = toString().substring(0, prefixLen)
+    fun hashConcat(other: SecureHash) = (this.bytes + other.bytes).sha256()
 
     // Like static methods in Java, except the 'companion' is a singleton that can have state.
     companion object {
@@ -35,6 +36,7 @@ sealed class SecureHash(bytes: ByteArray) : OpaqueBytes(bytes) {
         @JvmStatic fun sha256(str: String) = sha256(str.toByteArray())
 
         @JvmStatic fun randomSHA256() = sha256(newSecureRandom().generateSeed(32))
+        val zeroHash = SecureHash.SHA256(ByteArray(32, { 0.toByte() }))
     }
 
     // In future, maybe SHA3, truncated hashes etc.

core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt

@@ -1,39 +1,15 @@
 package net.corda.core.transactions
 
-import net.corda.core.contracts.Command
+import net.corda.core.contracts.*
-import net.corda.core.contracts.ContractState
+import net.corda.core.crypto.*
-import net.corda.core.contracts.StateRef
+import net.corda.core.crypto.SecureHash.Companion.zeroHash
-import net.corda.core.contracts.TransactionState
-import net.corda.core.crypto.MerkleTreeException
-import net.corda.core.crypto.PartialMerkleTree
-import net.corda.core.crypto.SecureHash
-import net.corda.core.crypto.sha256
 import net.corda.core.serialization.createKryo
 import net.corda.core.serialization.extendKryoHash
 import net.corda.core.serialization.serialize
 import java.util.*
 
-/**
- * Build filtered transaction using provided filtering functions.
- */
-fun WireTransaction.buildFilteredTransaction(filterFuns: FilterFuns): FilteredTransaction {
-    return FilteredTransaction.buildMerkleTransaction(this, filterFuns)
-}
-
-/**
- * Calculation of all leaves hashes that are needed for calculation of transaction id and partial Merkle branches.
- */
-fun WireTransaction.calculateLeavesHashes(): List<SecureHash> {
-    val resultHashes = ArrayList<SecureHash>()
-    val entries = listOf(inputs, outputs, attachments, commands)
-    entries.forEach { it.mapTo(resultHashes, { x -> serializedHash(x) }) }
-    return resultHashes
-}
-
-fun SecureHash.hashConcat(other: SecureHash) = (this.bytes + other.bytes).sha256()
-
 fun <T : Any> serializedHash(x: T): SecureHash {
-    val kryo = extendKryoHash(createKryo()) //Dealing with HashMaps inside states.
+    val kryo = extendKryoHash(createKryo()) // Dealing with HashMaps inside states.
     return x.serialize(kryo).hash
 }
 
@@ -42,55 +18,58 @@ fun <T : Any> serializedHash(x: T): SecureHash {
  *
  * See: https://en.wikipedia.org/wiki/Merkle_tree
  *
- * Transaction is split into following blocks: inputs, outputs, commands, attachments' refs. Merkle Tree is kept in
+ * Transaction is split into following blocks: inputs, attachments' refs, outputs, commands, notary,
- * a recursive data structure. Building is done bottom up, from all leaves' hashes.
+ * signers, tx type, timestamp. Merkle Tree is kept in a recursive data structure. Building is done bottom up,
- * If a row in a tree has an odd number of elements - the final hash is hashed with itself.
+ * from all leaves' hashes. If number of leaves is not a power of two, the tree is padded with zero hashes.
  */
 sealed class MerkleTree(val hash: SecureHash) {
     class Leaf(val value: SecureHash) : MerkleTree(value)
     class Node(val value: SecureHash, val left: MerkleTree, val right: MerkleTree) : MerkleTree(value)
-    //DuplicatedLeaf is storing a hash of the rightmost node that had to be duplicated to obtain the tree.
-    //That duplication can cause problems while building and verifying partial tree (especially for trees with duplicate
-    //attachments or commands).
-    class DuplicatedLeaf(val value: SecureHash) : MerkleTree(value)
-
-    fun hashNodes(right: MerkleTree): MerkleTree {
-        val newHash = this.hash.hashConcat(right.hash)
-        return Node(newHash, this, right)
-    }
 
     companion object {
+        private fun isPow2(num: Int): Boolean = num and (num-1) == 0
+
         /**
-         * Merkle tree building using hashes.
+         * Merkle tree building using hashes, with zero hash padding to full power of 2.
          */
+        @Throws(IllegalArgumentException::class)
         fun getMerkleTree(allLeavesHashes: List<SecureHash>): MerkleTree {
-            val leaves = allLeavesHashes.map { MerkleTree.Leaf(it) }
+            val leaves = padWithZeros(allLeavesHashes).map { MerkleTree.Leaf(it) }
             return buildMerkleTree(leaves)
         }
 
+        // If number of leaves in the tree is not a power of 2, we need to pad it with zero hashes.
+        private fun padWithZeros(allLeavesHashes: List<SecureHash>): List<SecureHash> {
+            var n = allLeavesHashes.size
+            if (isPow2(n)) return allLeavesHashes
+            val paddedHashes = ArrayList<SecureHash>(allLeavesHashes)
+            while (!isPow2(n)) {
+                paddedHashes.add(zeroHash)
+                n++
+            }
+            return paddedHashes
+        }
+
         /**
          * Tailrecursive function for building a tree bottom up.
          * @param lastNodesList MerkleTree nodes from previous level.
          * @return Tree root.
          */
         private tailrec fun buildMerkleTree(lastNodesList: List<MerkleTree>): MerkleTree {
-            if (lastNodesList.size < 1)
+            if (lastNodesList.isEmpty())
                 throw MerkleTreeException("Cannot calculate Merkle root on empty hash list.")
             if (lastNodesList.size == 1) {
                 return lastNodesList[0] //Root reached.
             } else {
                 val newLevelHashes: MutableList<MerkleTree> = ArrayList()
                 var i = 0
-                while (i < lastNodesList.size) {
+                val n = lastNodesList.size
+                while (i < n) {
                     val left = lastNodesList[i]
-                    val n = lastNodesList.size
+                    require(i+1 <= n-1) { "Sanity check: number of nodes should be even." }
-                    // If there is an odd number of elements at this level,
+                    val right = lastNodesList[i+1]
-                    // the last element is hashed with itself and stored as a Leaf.
+                    val newHash = left.hash.hashConcat(right.hash)
-                    val right = when {
+                    val combined = Node(newHash, left, right)
-                        i + 1 > n - 1 -> MerkleTree.DuplicatedLeaf(lastNodesList[n - 1].hash)
-                        else -> lastNodesList[i + 1]
-                    }
-                    val combined = left.hashNodes(right)
                     newLevelHashes.add(combined)
                     i += 2
                 }
@@ -101,40 +80,67 @@ sealed class MerkleTree(val hash: SecureHash) {
 }
 
 /**
- * Class that holds filtered leaves for a partial Merkle transaction. We assume mixed leaves types.
+ * Interface implemented by WireTransaction and FilteredLeaves.
+ * Property traversableList assures that we always calculate hashes in the same order, lets us define which
+ * fields of WireTransaction will be included in id calculation or partial merkle tree building.
  */
-class FilteredLeaves(
+interface TraversableTransaction {
-        val inputs: List<StateRef>,
+    val inputs: List<StateRef>
-        val outputs: List<TransactionState<ContractState>>,
+    val attachments: List<SecureHash>
-        val attachments: List<SecureHash>,
+    val outputs: List<TransactionState<ContractState>>
-        val commands: List<Command>
+    val commands: List<Command>
-) {
+    val notary: Party?
-    fun getFilteredHashes(): List<SecureHash> {
+    val mustSign: List<CompositeKey>
-        val resultHashes = ArrayList<SecureHash>()
+    val type: TransactionType?
-        val entries = listOf(inputs, outputs, attachments, commands)
+    val timestamp: Timestamp?
-        entries.forEach { it.mapTo(resultHashes, { x -> serializedHash(x) }) }
+
-        return resultHashes
+    /**
-    }
+     * Traversing transaction fields with a list function over transaction contents. Used for leaves hashes calculation
+     * and user provided filtering and checking of filtered transaction.
+     */
+    // We may want to specify our own behaviour on certain tx fields.
+    // Like if we include them at all, what to do with null values, if we treat list as one or not etc. for building
+    // torn-off transaction and id calculation.
+    val traversableList: List<Any>
+        get() {
+            val traverseList = mutableListOf(inputs, attachments, outputs, commands).flatten().toMutableList()
+            if (notary != null) traverseList.add(notary!!)
+            traverseList.addAll(mustSign)
+            if (type != null) traverseList.add(type!!)
+            if (timestamp != null) traverseList.add(timestamp!!)
+            return traverseList
+        }
+
+    // Calculation of all leaves hashes that are needed for calculation of transaction id and partial Merkle branches.
+    fun calculateLeavesHashes(): List<SecureHash> = traversableList.map { serializedHash(it) }
 }
 
 /**
- * Holds filter functions on transactions fields.
+ * Class that holds filtered leaves for a partial Merkle transaction. We assume mixed leaf types, notice that every
- * Functions are used to build a partial tree only out of some subset of original transaction fields.
+ * field from WireTransaction can be used in PartialMerkleTree calculation.
  */
-class FilterFuns(
+class FilteredLeaves(
-        val filterInputs: (StateRef) -> Boolean = { false },
+        override val inputs: List<StateRef>,
-        val filterOutputs: (TransactionState<ContractState>) -> Boolean = { false },
+        override val attachments: List<SecureHash>,
-        val filterAttachments: (SecureHash) -> Boolean = { false },
+        override val outputs: List<TransactionState<ContractState>>,
-        val filterCommands: (Command) -> Boolean = { false }
+        override val commands: List<Command>,
-) {
+        override val notary: Party?,
-    fun <T : Any> genericFilter(elem: T): Boolean {
+        override val mustSign: List<CompositeKey>,
-        return when (elem) {
+        override val type: TransactionType?,
-            is StateRef -> filterInputs(elem)
+        override val timestamp: Timestamp?
-            is TransactionState<*> -> filterOutputs(elem)
+) : TraversableTransaction {
-            is SecureHash -> filterAttachments(elem)
+    /**
-            is Command -> filterCommands(elem)
+     * Function that checks the whole filtered structure.
-            else -> throw IllegalArgumentException("Wrong argument type: ${elem.javaClass}")
+     * Force type checking on a structure that we obtained, so we don't sign more than expected.
-        }
+     * Example: Oracle is implemented to check only for commands, if it gets an attachment and doesn't expect it - it can sign
+     * over a transaction with the attachment that wasn't verified. Of course it depends on how you implement it, but else -> false
+     * should solve a problem with possible later extensions to WireTransaction.
+     * @param checkingFun function that performs type checking on the structure fields and provides verification logic accordingly.
+     * @returns false if no elements were matched on a structure or checkingFun returned false.
+     */
+    fun checkWithFun(checkingFun: (Any) -> Boolean): Boolean {
+        val checkList = traversableList.map { checkingFun(it) }
+        return (!checkList.isEmpty()) && checkList.all { true }
     }
 }
 
@@ -151,18 +157,14 @@ class FilteredTransaction(
         /**
          * Construction of filtered transaction with Partial Merkle Tree.
          * @param wtx WireTransaction to be filtered.
-         * @param filterFuns filtering functions for inputs, outputs, attachments, commands.
+         * @param filtering filtering over the whole WireTransaction
          */
         fun buildMerkleTransaction(wtx: WireTransaction,
-                                   filterFuns: FilterFuns
+                                   filtering: (Any) -> Boolean
         ): FilteredTransaction {
-            val filteredInputs = wtx.inputs.filter { filterFuns.genericFilter(it) }
+            val filteredLeaves = wtx.filterWithFun(filtering)
-            val filteredOutputs = wtx.outputs.filter { filterFuns.genericFilter(it) }
+            val merkleTree = wtx.getMerkleTree()
-            val filteredAttachments = wtx.attachments.filter { filterFuns.genericFilter(it) }
+            val pmt = PartialMerkleTree.build(merkleTree, filteredLeaves.calculateLeavesHashes())
-            val filteredCommands = wtx.commands.filter { filterFuns.genericFilter(it) }
-            val filteredLeaves = FilteredLeaves(filteredInputs, filteredOutputs, filteredAttachments, filteredCommands)
-
-            val pmt = PartialMerkleTree.build(wtx.merkleTree, filteredLeaves.getFilteredHashes())
             return FilteredTransaction(filteredLeaves, pmt)
         }
     }
@@ -170,10 +172,19 @@ class FilteredTransaction(
     /**
      * Runs verification of Partial Merkle Branch with merkleRootHash.
      */
+    @Throws(MerkleTreeException::class)
     fun verify(merkleRootHash: SecureHash): Boolean {
-        val hashes: List<SecureHash> = filteredLeaves.getFilteredHashes()
+        val hashes: List<SecureHash> = filteredLeaves.calculateLeavesHashes()
-        if (hashes.size == 0)
+        if (hashes.isEmpty())
             throw MerkleTreeException("Transaction without included leaves.")
         return partialMerkleTree.verify(merkleRootHash, hashes)
     }
+
+    /**
+     * Runs verification of Partial Merkle Branch with merkleRootHash. Checks filteredLeaves with provided checkingFun.
+     */
+    @Throws(MerkleTreeException::class)
+    fun verifyWithFunction(merkleRootHash: SecureHash, checkingFun: (Any) -> Boolean): Boolean {
+        return verify(merkleRootHash) && filteredLeaves.checkWithFun { checkingFun(it) }
+    }
 }

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -25,15 +25,15 @@ class WireTransaction(
         /** Pointers to the input states on the ledger, identified by (tx identity hash, output index). */
         override val inputs: List<StateRef>,
         /** Hashes of the ZIP/JAR files that are needed to interpret the contents of this wire transaction. */
-        val attachments: List<SecureHash>,
+        override val attachments: List<SecureHash>,
         outputs: List<TransactionState<ContractState>>,
         /** Ordered list of ([CommandData], [PublicKey]) pairs that instruct the contracts what to do. */
-        val commands: List<Command>,
+        override val commands: List<Command>,
         notary: Party?,
         signers: List<CompositeKey>,
         type: TransactionType,
         timestamp: Timestamp?
-) : BaseTransaction(inputs, outputs, notary, signers, type, timestamp) {
+) : BaseTransaction(inputs, outputs, notary, signers, type, timestamp), TraversableTransaction {
     init {
         checkInvariants()
     }
@@ -42,14 +42,7 @@ class WireTransaction(
     @Volatile @Transient private var cachedBytes: SerializedBytes<WireTransaction>? = null
     val serialized: SerializedBytes<WireTransaction> get() = cachedBytes ?: serialize().apply { cachedBytes = this }
 
-    //We need cashed leaves hashes and whole tree for an id and Partial Merkle Tree calculation.
+    override val id: SecureHash by lazy { getMerkleTree().hash }
-    @Volatile @Transient private var cachedLeavesHashes: List<SecureHash>? = null
-    val allLeavesHashes: List<SecureHash> get() = cachedLeavesHashes ?: calculateLeavesHashes().apply { cachedLeavesHashes = this }
-
-    @Volatile @Transient var cachedTree: MerkleTree? = null
-    val merkleTree: MerkleTree get() = cachedTree ?: MerkleTree.getMerkleTree(allLeavesHashes).apply { cachedTree = this }
-
-    override val id: SecureHash get() = merkleTree.hash
 
     companion object {
         fun deserialize(data: SerializedBytes<WireTransaction>, kryo: Kryo = THREAD_LOCAL_KRYO.get()): WireTransaction {
@@ -91,6 +84,39 @@ class WireTransaction(
         return LedgerTransaction(resolvedInputs, outputs, authenticatedArgs, attachments, id, notary, mustSign, timestamp, type)
     }
 
+    /**
+     * Build filtered transaction using provided filtering functions.
+     */
+    fun buildFilteredTransaction(filtering: (Any) -> Boolean): FilteredTransaction {
+        return FilteredTransaction.buildMerkleTransaction(this, filtering)
+    }
+
+    /**
+     * Builds whole Merkle tree for a transaction.
+     */
+    fun getMerkleTree(): MerkleTree {
+        return MerkleTree.getMerkleTree(calculateLeavesHashes())
+    }
+
+    /**
+     * Construction of partial transaction from WireTransaction based on filtering.
+     * @param filtering filtering over the whole WireTransaction
+     * @returns FilteredLeaves used in PartialMerkleTree calculation and verification.
+     */
+    fun filterWithFun(filtering: (Any) -> Boolean): FilteredLeaves {
+        fun notNullFalse(elem: Any?): Any? = if(elem == null || !filtering(elem)) null else elem
+        return FilteredLeaves(
+                inputs.filter { filtering(it) },
+                attachments.filter { filtering(it) },
+                outputs.filter { filtering(it) },
+                commands.filter { filtering(it) },
+                notNullFalse(notary) as Party?,
+                mustSign.filter { filtering(it) },
+                notNullFalse(type) as TransactionType?,
+                notNullFalse(timestamp) as Timestamp?
+        )
+    }
+
     override fun toString(): String {
         val buf = StringBuilder()
         buf.appendln("Transaction $id:")

core/src/test/kotlin/net/corda/core/crypto/PartialMerkleTreeTest.kt

@@ -3,26 +3,22 @@ package net.corda.core.crypto
 
 import com.esotericsoftware.kryo.serializers.MapSerializer
 import net.corda.contracts.asset.Cash
-import net.corda.core.contracts.DOLLARS
+import net.corda.core.contracts.*
-import net.corda.core.contracts.`issued by`
+import net.corda.core.crypto.SecureHash.Companion.zeroHash
 import net.corda.core.serialization.*
 import net.corda.core.transactions.*
-import net.corda.core.utilities.DUMMY_PUBKEY_1
+import net.corda.core.utilities.*
-import net.corda.testing.ALICE_PUBKEY
 import net.corda.testing.MEGA_CORP
 import net.corda.testing.MEGA_CORP_PUBKEY
 import net.corda.testing.ledger
 import org.junit.Test
 import java.util.*
-import kotlin.test.assertEquals
+import kotlin.test.*
-import kotlin.test.assertFailsWith
-import kotlin.test.assertFalse
-import kotlin.test.assertTrue
 
 class PartialMerkleTreeTest {
     val nodes = "abcdef"
     val hashed = nodes.map { it.serialize().sha256() }
-    val root = SecureHash.parse("F6D8FB3720114F8D040D64F633B0D9178EB09A55AA7D62FAE1A070D1BF561051")
+    val expectedRoot = MerkleTree.getMerkleTree(hashed.toMutableList() + listOf(zeroHash, zeroHash)).hash
     val merkleTree = MerkleTree.getMerkleTree(hashed)
 
     val testLedger = ledger {
@@ -33,22 +29,30 @@ class PartialMerkleTreeTest {
                         owner = MEGA_CORP_PUBKEY
                 )
             }
+            output("dummy cash 1") {
+                Cash.State(
+                        amount = 900.DOLLARS `issued by` MEGA_CORP.ref(1, 1),
+                        owner = DUMMY_PUBKEY_1
+                )
+            }
         }
 
         transaction {
             input("MEGA_CORP cash")
             output("MEGA_CORP cash".output<Cash.State>().copy(owner = DUMMY_PUBKEY_1))
             command(MEGA_CORP_PUBKEY) { Cash.Commands.Move() }
+            timestamp(TEST_TX_TIME)
             this.verifies()
         }
     }
 
-    val testTx = testLedger.interpreter.transactionsToVerify[0]
+    val txs = testLedger.interpreter.transactionsToVerify
+    val testTx = txs[0]
 
-    //Building full Merkle Tree tests.
+    // Building full Merkle Tree tests.
     @Test
     fun `building Merkle tree with 6 nodes - no rightmost nodes`() {
-        assertEquals(root, merkleTree.hash)
+        assertEquals(expectedRoot, merkleTree.hash)
     }
 
     @Test
@@ -67,25 +71,70 @@ class PartialMerkleTreeTest {
     fun `building Merkle tree odd number of nodes`() {
         val odd = hashed.subList(0, 3)
         val h1 = hashed[0].hashConcat(hashed[1])
-        val h2 = hashed[2].hashConcat(hashed[2])
+        val h2 = hashed[2].hashConcat(zeroHash)
         val expected = h1.hashConcat(h2)
         val mt = MerkleTree.getMerkleTree(odd)
         assertEquals(mt.hash, expected)
     }
 
+    @Test
+    fun `check full tree`() {
+        val h = SecureHash.randomSHA256()
+        val left = MerkleTree.Node(h, MerkleTree.Node(h, MerkleTree.Leaf(h), MerkleTree.Leaf(h)),
+                    MerkleTree.Node(h, MerkleTree.Leaf(h), MerkleTree.Leaf(h)))
+        val right = MerkleTree.Node(h, MerkleTree.Leaf(h), MerkleTree.Leaf(h))
+        val tree = MerkleTree.Node(h, left, right)
+        assertFailsWith<MerkleTreeException> { PartialMerkleTree.build(tree, listOf(h)) }
+        PartialMerkleTree.build(right, listOf(h, h)) // Node and two leaves.
+        PartialMerkleTree.build(MerkleTree.Leaf(h), listOf(h)) // Just a leaf.
+    }
+
     @Test
     fun `building Merkle tree for a transaction`() {
-        val filterFuns = FilterFuns(
+        fun filtering(elem: Any): Boolean {
-                filterCommands = { x -> ALICE_PUBKEY in x.signers },
+            return when (elem) {
-                filterOutputs = { true },
+                is StateRef -> true
-                filterInputs = { true })
+                is TransactionState<*> -> elem.data.participants[0].keys == DUMMY_PUBKEY_1.keys
-        val mt = testTx.buildFilteredTransaction(filterFuns)
+                is Command -> MEGA_CORP_PUBKEY in elem.signers
+                is Timestamp -> true
+                is CompositeKey -> elem == MEGA_CORP_PUBKEY
+                else -> false
+            }
+        }
+        val mt = testTx.buildFilteredTransaction(::filtering)
+        val leaves = mt.filteredLeaves
         val d = WireTransaction.deserialize(testTx.serialized)
         assertEquals(testTx.id, d.id)
+        assertEquals(1, leaves.commands.size)
+        assertEquals(1, leaves.outputs.size)
+        assertEquals(1, leaves.inputs.size)
+        assertEquals(1, leaves.mustSign.size)
+        assertEquals(0, leaves.attachments.size)
+        assertTrue(mt.filteredLeaves.timestamp != null)
+        assertEquals(null, mt.filteredLeaves.type)
+        assertEquals(null, mt.filteredLeaves.notary)
         assert(mt.verify(testTx.id))
     }
 
-    //Partial Merkle Tree building tests
+    @Test
+    fun `same transactions with different notaries have different ids`() {
+        val wtx1 = makeSimpleCashWtx(DUMMY_NOTARY)
+        val wtx2 = makeSimpleCashWtx(MEGA_CORP)
+        assertNotEquals(wtx1.id, wtx2.id)
+    }
+
+    @Test
+    fun `nothing filtered`() {
+        val mt = testTx.buildFilteredTransaction( {false} )
+        assertTrue(mt.filteredLeaves.attachments.isEmpty())
+        assertTrue(mt.filteredLeaves.commands.isEmpty())
+        assertTrue(mt.filteredLeaves.inputs.isEmpty())
+        assertTrue(mt.filteredLeaves.outputs.isEmpty())
+        assertTrue(mt.filteredLeaves.timestamp == null)
+        assertFailsWith<MerkleTreeException> { mt.verify(testTx.id) }
+    }
+
+    // Partial Merkle Tree building tests
     @Test
     fun `build Partial Merkle Tree, only left nodes branch`() {
         val inclHashes = listOf(hashed[3], hashed[5])
@@ -137,7 +186,7 @@ class PartialMerkleTreeTest {
 
     @Test
     fun `verify Partial Merkle Tree - duplicate leaves failure`() {
-        val mt = MerkleTree.getMerkleTree(hashed.subList(0, 5)) //Odd number of leaves. Last one is duplicated.
+        val mt = MerkleTree.getMerkleTree(hashed.subList(0, 5)) // Odd number of leaves. Last one is duplicated.
         val inclHashes = arrayListOf(hashed[3], hashed[4])
         val pmt = PartialMerkleTree.build(mt, inclHashes)
         inclHashes.add(hashed[4])
@@ -162,11 +211,24 @@ class PartialMerkleTreeTest {
     @Test
     fun `hash map serialization`() {
         val hm1 = hashMapOf("a" to 1, "b" to 2, "c" to 3, "e" to 4)
-        assert(serializedHash(hm1) == serializedHash(hm1.serialize().deserialize())) //It internally uses the ordered HashMap extension.
+        assert(serializedHash(hm1) == serializedHash(hm1.serialize().deserialize())) // It internally uses the ordered HashMap extension.
         val kryo = extendKryoHash(createKryo())
         assertTrue(kryo.getSerializer(HashMap::class.java) is OrderedSerializer)
         assertTrue(kryo.getSerializer(LinkedHashMap::class.java) is MapSerializer)
         val hm2 = hm1.serialize(kryo).deserialize(kryo)
         assert(hm1.hashCode() == hm2.hashCode())
     }
+
+    private fun makeSimpleCashWtx(notary: Party, timestamp: Timestamp? = null, attachments: List<SecureHash> = emptyList()): WireTransaction {
+        return WireTransaction(
+                inputs = testTx.inputs,
+                attachments = attachments,
+                outputs = testTx.outputs,
+                commands = testTx.commands,
+                notary = notary,
+                signers = listOf(MEGA_CORP_PUBKEY, DUMMY_PUBKEY_1),
+                type = TransactionType.General(),
+                timestamp = timestamp
+        )
+    }
 }

docs/source/merkle-trees.rst

@@ -15,17 +15,17 @@ You can read more on the concept `here <https://en.wikipedia.org/wiki/Merkle_tre
 Merkle trees in Corda
 ---------------------
 
-Transactions are split into leaves, each of them contains either input, output, command or attachment. Other fields like
+Transactions are split into leaves, each of them contains either input, output, command or attachment. Additionally, in
-timestamp or signers are not used in the calculation.
+transaction id calculation we use other fields of ``WireTransaction`` like timestamp, notary, type and signers.
-Next, the Merkle tree is built in the normal way by hashing the concatenation
+Next, the Merkle tree is built in the normal way by hashing the concatenation of nodes’ hashes below the current one together.
-of nodes’ hashes below the current one together. It’s visible on the example image below, where ``H`` denotes sha256 function,
+It’s visible on the example image below, where ``H`` denotes sha256 function, "+" - concatenation.
-"+" - concatenation.
 
 .. image:: resources/merkleTree.png
 
-The transaction has one input state, one output and three commands. If a tree is not a full binary tree, the rightmost nodes are
+The transaction has two input states, one of output, attachment and command each and timestamp. For brevity we didn't
-duplicated in hash calculation (dotted lines).
+include all leaves on the diagram (type, notary and signers are presented as one leaf labelled Rest - in reality
-
+they are separate leaves). Notice that if a tree is not a full binary tree, leaves are padded to the nearest power
+of 2 with zero hash (since finding a pre-image of sha256(x) == 0 is hard computational task) - marked light green above.
 Finally, the hash of the root is the identifier of the transaction, it's also used for signing and verification of data integrity.
 Every change in transaction on a leaf level will change its identifier.
 
@@ -39,9 +39,11 @@ to that particular transaction.
 
 .. image:: resources/partialMerkle.png
 
-In the example above, the red node is the one holding data for signing Oracle service. Blue nodes' hashes form the Partial Merkle
+In the example above, the node ``H(f)`` is the one holding command data for signing by Oracle service. Blue leaf ``H(g)`` is also
-Tree, dotted ones are not included. Having the command that should be in a red node place and branch we are able to calculate
+included since it's holding timestamp information. Nodes labelled ``Provided`` form the Partial Merkle Tree, black ones
-root of this tree and compare it with original transaction identifier - we have a proof that this command belongs to this transaction.
+are omitted. Having timestamp with the command that should be in a violet node place and branch we are able to calculate
+root of this tree and compare it with original transaction identifier - we have a proof that this command and timestamp
+belong to this transaction.
 
 Example of usage
 ----------------
@@ -50,36 +52,34 @@ Let’s focus on a code example. We want to construct a transaction with command
 :doc:`oracles`.
 After construction of a partial transaction, with included ``Fix`` commands in it, we want to send it to the Oracle for checking
 and signing. To do so we need to specify which parts of the transaction are going to be revealed. That can be done by constructing
-filtering functions for inputs, outputs, attachments and commands separately. If a function is not provided by default none
+filtering function over fields of ``WireTransaction`` of type ``(Any) -> Boolean``.
-of the elements from this group will be included in a Partial Merkle Tree.
 
 .. container:: codeset
 
    .. sourcecode:: kotlin
 
-        val partialTx  = ...
+        val partialTx = ...
         val oracle: Party = ...
-        fun filterCommands(c: Command) = oracle.owningKey in c.signers && c.value is Fix
+        fun filtering(elem: Any): Boolean {
-        val filterFuns = FilterFuns(filterCommands = ::filterCommands)
+                return when (elem) {
+                    is Command -> oracleParty.owningKey in elem.signers && elem.value is Fix
+                    else -> false
+                }
+        }
 
 Assuming that we already assembled partialTx with some commands and know the identity of Oracle service,
-we pass filtering function over commands - ``filterCommands`` to ``FilterFuns``. It filters only
+we construct filtering function over commands - ``filtering``. It performs type checking and filters only ``Fix`` commands
-commands of type ``Fix`` as in IRSDemo example. Then we can construct ``FilteredTransaction``:
+as in IRSDemo example. Then we can construct ``FilteredTransaction``:
 
 .. container:: codeset
 
    .. sourcecode:: kotlin
 
         val wtx: WireTransaction = partialTx.toWireTransaction()
-        val ftx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
+        val ftx: FilteredTransaction = wtx.buildFilteredTransaction(filtering)
 
-In the Oracle example this step takes place in ``RatesFixFlow``:
+In the Oracle example this step takes place in ``RatesFixFlow`` by overriding ``filtering`` function, see: :ref:`filtering_ref`
 
-.. container:: codeset
-
-   .. sourcecode:: kotlin
-
-        val flow = RatesFixFlow(partialTx, filterFuns, oracle, fixOf, "0.675".bd, "0.1".bd)
 
 ``FilteredTransaction`` holds ``filteredLeaves`` (data that we wanted to reveal) and Merkle branch for them.
 
@@ -87,14 +87,21 @@ In the Oracle example this step takes place in ``RatesFixFlow``:
 
    .. sourcecode:: kotlin
 
-        // Getting included commands, inputs, outputs, attachments.
+        // Direct accsess to included commands, inputs, outputs, attachments etc.
         val cmds: List<Command> = ftx.filteredLeaves.commands
         val ins: List<StateRef> = ftx.filteredLeaves.inputs
-        val outs: List<TransactionState<ContractState>> = ftx.filteredLeaves.outputs
+        val timestamp: Timestamp? = ftx.filteredLeaves.timestamp
-        val attchs: List<SecureHash> = ftx.filteredLeaves.attachments
+        ...
 
+.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
+    :language: kotlin
+    :start-after: DOCSTART 1
+    :end-before: DOCEND 1
 
-If you want to verify obtained ``FilteredTransaction`` all you need is the root hash of the full transaction:
+Above code snippet is taken from ``NodeInterestRates.kt`` file and implements a signing part of an Oracle.
+You can check only leaves using ``leaves.checkWithFun { check(it) }`` and then verify obtained ``FilteredTransaction``
+to see if data from ``PartialMerkleTree`` belongs to ``WireTransaction`` with provided ``id``. All you need is the root hash
+of the full transaction:
 
 .. container:: codeset
 
@@ -104,6 +111,13 @@ If you want to verify obtained ``FilteredTransaction`` all you need is the root
                 throw MerkleTreeException("Rate Fix Oracle: Couldn't verify partial Merkle tree.")
         }
 
+Or combine the two steps together:
+
+.. container:: codeset
+
+   .. sourcecode:: kotlin
+
+        ftx.verifyWithFunction(merkleRoot, ::check)
 
 .. note:: The way the ``FilteredTransaction`` is constructed ensures that after signing of the root hash it's impossible to add or remove
     leaves. However, it can happen that having transaction with multiple commands one party reveals only subset of them to the Oracle.

docs/source/oracles.rst

@@ -239,8 +239,11 @@ those for ``NodeInterestRates.Oracle``.
    :start-after: DOCSTART 1
    :end-before: DOCEND 1
 
-You'll note that the ``FixSignFlow`` requires a ``FilterFuns`` instance with the appropriate filter to include only
+You'll note that the ``FixSignFlow`` requires a ``FilterTransaction`` instance which includes only ``Fix`` commands.
-the ``Fix`` commands.  You can find a further explanation of this in :doc:`merkle-trees`.
+You can find a further explanation of this in :doc:`merkle-trees`. Below you will see how to build such transaction with
+hidden fields.
+
+.. _filtering_ref:
 
 Using an oracle
 ---------------
@@ -260,8 +263,9 @@ As you can see, this:
 2. Does some quick validation.
 3. Adds the command to the transaction containing the fact to be signed for by the oracle.
 4. Calls an extension point that allows clients to generate output states based on the fact from the oracle.
-5. Requests the signature from the oracle using the client sub-flow for signing from above.
+5. Builds filtered transaction based on filtering function extended from ``RatesFixFlow``.
-6. Adds the signature returned from the oracle.
+6. Requests the signature from the oracle using the client sub-flow for signing from above.
+7. Adds the signature returned from the oracle.
 
 Here's an example of it in action from ``FixingFlow.Fixer``.
 
@@ -269,3 +273,8 @@ Here's an example of it in action from ``FixingFlow.Fixer``.
    :language: kotlin
    :start-after: DOCSTART 1
    :end-before: DOCEND 1
+
+.. note::
+    When overriding be careful when making the sub-class an anonymous or inner class (object declarations in Kotlin),
+    because that kind of classes can access variables from the enclosing scope and cause serialization problems when
+    checkpointed.

samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt

@@ -192,31 +192,27 @@ object NodeInterestRates {
             if (!ftx.verify(merkleRoot)) {
                 throw MerkleTreeException("Rate Fix Oracle: Couldn't verify partial Merkle tree.")
             }
-
+            // Performing validation of obtained FilteredLeaves.
-            // Reject if we have something different than only commands.
+            fun commandValidator(elem: Command): Boolean {
-            val leaves = ftx.filteredLeaves
+                if (!(identity.owningKey in elem.signers && elem.value is Fix))
-            require(leaves.inputs.isEmpty() && leaves.outputs.isEmpty() && leaves.attachments.isEmpty())
+                    throw IllegalArgumentException("Oracle received unknown command (not in signers or not Fix).")
-
+                val fix = elem.value as Fix
-            val fixes: List<Fix> = ftx.filteredLeaves.commands.
-                    filter { identity.owningKey in it.signers && it.value is Fix }.
-                    map { it.value as Fix }
-
-            // Reject signing attempt if we received more commands than we should.
-            if (fixes.size != ftx.filteredLeaves.commands.size)
-                throw IllegalArgumentException()
-
-            // Reject this signing attempt if there are no commands of the right kind.
-            if (fixes.isEmpty())
-                throw IllegalArgumentException()
-
-            // For each fix, verify that the data is correct.
-            val knownFixes = knownFixes // Snapshot
-            for (fix in fixes) {
                 val known = knownFixes[fix.of]
                 if (known == null || known != fix)
                     throw UnknownFix(fix.of)
+                return true
             }
 
+            fun check(elem: Any): Boolean {
+                return when (elem) {
+                    is Command -> commandValidator(elem)
+                    else -> throw IllegalArgumentException("Oracle received data of different type than expected.")
+                }
+            }
+            val leaves = ftx.filteredLeaves
+            if (!leaves.checkWithFun(::check))
+                throw IllegalArgumentException()
+
             // It all checks out, so we can return a signature.
             //
             // Note that we will happily sign an invalid transaction, as we are only being presented with a filtered

samples/irs-demo/src/main/kotlin/net/corda/irs/flows/FixingFlow.kt

@@ -10,7 +10,6 @@ import net.corda.core.node.NodeInfo
 import net.corda.core.node.PluginServiceHub
 import net.corda.core.node.services.ServiceType
 import net.corda.core.seconds
-import net.corda.core.transactions.FilterFuns
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.ProgressTracker
 import net.corda.core.utilities.trace
@@ -68,12 +67,8 @@ object FixingFlow {
             val oracle = serviceHub.networkMapCache.getNodesWithService(handshake.payload.oracleType).first()
             val oracleParty = oracle.serviceIdentities(handshake.payload.oracleType).first()
 
-            // TODO Could it be solved in better way, move filtering here not in RatesFixFlow?
             // DOCSTART 1
-            fun filterCommands(c: Command) = oracleParty.owningKey in c.signers && c.value is Fix
+            val addFixing = object : RatesFixFlow(ptx, oracleParty, fixOf, BigDecimal.ZERO, BigDecimal.ONE) {
-
-            val filterFuns = FilterFuns(filterCommands = ::filterCommands)
-            val addFixing = object : RatesFixFlow(ptx, filterFuns, oracleParty, fixOf, BigDecimal.ZERO, BigDecimal.ONE) {
                 @Suspendable
                 override fun beforeSigning(fix: Fix) {
                     newDeal.generateFix(ptx, StateAndRef(txState, handshake.payload.ref), fix)
@@ -82,6 +77,14 @@ object FixingFlow {
                     // to have one.
                     ptx.setTime(serviceHub.clock.instant(), 30.seconds)
                 }
+
+                @Suspendable
+                override fun filtering(elem: Any): Boolean {
+                    return when (elem) {
+                        is Command -> oracleParty.owningKey in elem.signers && elem.value is Fix
+                        else -> false
+                    }
+                }
             }
             subFlow(addFixing)
             // DOCEND 1

samples/irs-demo/src/main/kotlin/net/corda/irs/flows/RatesFixFlow.kt

@@ -7,7 +7,6 @@ import net.corda.core.crypto.DigitalSignature
 import net.corda.core.crypto.Party
 import net.corda.core.crypto.SecureHash
 import net.corda.core.flows.FlowLogic
-import net.corda.core.transactions.FilterFuns
 import net.corda.core.transactions.FilteredTransaction
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.ProgressTracker
@@ -28,12 +27,10 @@ import java.util.*
  * @throws FixOutOfRange if the returned fix was further away from the expected rate by the given amount.
  */
 open class RatesFixFlow(protected val tx: TransactionBuilder,
-                        /** Filtering functions over transaction, used to build partial transaction presented to oracle. */
+                        protected val oracle: Party,
-                        private val filterFuns: FilterFuns,
+                        protected val fixOf: FixOf,
-                        private val oracle: Party,
+                        protected val expectedRate: BigDecimal,
-                        private val fixOf: FixOf,
+                        protected val rateTolerance: BigDecimal,
-                        private val expectedRate: BigDecimal,
-                        private val rateTolerance: BigDecimal,
                         override val progressTracker: ProgressTracker = RatesFixFlow.tracker(fixOf.name)) : FlowLogic<Unit>() {
 
     companion object {
@@ -59,7 +56,8 @@ open class RatesFixFlow(protected val tx: TransactionBuilder,
         tx.addCommand(fix, oracle.owningKey)
         beforeSigning(fix)
         progressTracker.currentStep = SIGNING
-        val signature = subFlow(FixSignFlow(tx, oracle, filterFuns))
+        val mtx = tx.toWireTransaction().buildFilteredTransaction({ filtering(it) })
+        val signature = subFlow(FixSignFlow(tx, oracle, mtx))
         tx.addSignatureUnchecked(signature)
     }
     // DOCEND 2
@@ -72,6 +70,15 @@ open class RatesFixFlow(protected val tx: TransactionBuilder,
     protected open fun beforeSigning(fix: Fix) {
     }
 
+    /**
+     * Filtering functions over transaction, used to build partial transaction with partial Merkle tree presented to oracle.
+     * When overriding be careful when making the sub-class an anonymous or inner class (object declarations in Kotlin),
+     * because that kind of classes can access variables from the enclosing scope and cause serialization problems when
+     * checkpointed.
+     */
+    @Suspendable
+    protected open fun filtering(elem: Any): Boolean = false
+
     private fun checkFixIsNearExpected(fix: Fix) {
         val delta = (fix.value - expectedRate).abs()
         if (delta > rateTolerance) {
@@ -97,13 +104,12 @@ open class RatesFixFlow(protected val tx: TransactionBuilder,
         }
     }
 
-    class FixSignFlow(val tx: TransactionBuilder, val oracle: Party, val filterFuns: FilterFuns) : FlowLogic<DigitalSignature.LegallyIdentifiable>() {
+    class FixSignFlow(val tx: TransactionBuilder, val oracle: Party,
+                      val partialMerkleTx: FilteredTransaction) : FlowLogic<DigitalSignature.LegallyIdentifiable>() {
         @Suspendable
         override fun call(): DigitalSignature.LegallyIdentifiable {
             val wtx = tx.toWireTransaction()
-            val partialMerkleTx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
             val rootHash = wtx.id
-
             val resp = sendAndReceive<DigitalSignature.LegallyIdentifiable>(oracle, SignRequest(rootHash, partialMerkleTx))
             return resp.unwrap { sig ->
                 check(sig.signer == oracle)

samples/irs-demo/src/test/kotlin/net/corda/irs/testing/NodeInterestRatesTest.kt

@@ -11,10 +11,10 @@ import net.corda.core.crypto.Party
 import net.corda.core.crypto.generateKeyPair
 import net.corda.core.getOrThrow
 import net.corda.core.node.services.ServiceInfo
-import net.corda.core.transactions.FilterFuns
+import net.corda.core.transactions.TransactionBuilder
-import net.corda.core.transactions.FilteredTransaction
 import net.corda.core.utilities.DUMMY_NOTARY
 import net.corda.core.utilities.LogHelper
+import net.corda.core.utilities.ProgressTracker
 import net.corda.irs.api.NodeInterestRates
 import net.corda.irs.flows.RatesFixFlow
 import net.corda.node.utilities.configureDatabase
@@ -30,6 +30,7 @@ import org.junit.Assert
 import org.junit.Before
 import org.junit.Test
 import java.io.Closeable
+import java.math.BigDecimal
 import java.time.Clock
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
@@ -53,6 +54,15 @@ class NodeInterestRatesTest {
     lateinit var dataSource: Closeable
     lateinit var database: Database
 
+    fun fixCmdFilter(elem: Any): Boolean {
+        return when (elem) {
+            is Command -> oracle.identity.owningKey in elem.signers && elem.value is Fix
+            else -> false
+        }
+    }
+
+    fun filterCmds(elem: Any): Boolean = elem is Command
+
     @Before
     fun setUp() {
         val dataSourceAndDatabase = configureDatabase(makeTestDataSourceProperties())
@@ -120,11 +130,17 @@ class NodeInterestRatesTest {
         databaseTransaction(database) {
             val tx = makeTX()
             val wtx1 = tx.toWireTransaction()
-            val ftx1 = FilteredTransaction.buildMerkleTransaction(wtx1, FilterFuns(filterOutputs = { true }))
+            fun filterAllOutputs(elem: Any): Boolean {
+                return when (elem) {
+                    is TransactionState<ContractState> -> true
+                    else -> false
+                }
+            }
+            val ftx1 = wtx1.buildFilteredTransaction(::filterAllOutputs)
             assertFailsWith<IllegalArgumentException> { oracle.sign(ftx1, wtx1.id) }
             tx.addCommand(Cash.Commands.Move(), ALICE_PUBKEY)
             val wtx2 = tx.toWireTransaction()
-            val ftx2 = FilteredTransaction.buildMerkleTransaction(wtx2, FilterFuns(filterCommands = { true }))
+            val ftx2 = wtx2.buildFilteredTransaction { x -> filterCmds(x) }
             assertFalse(wtx1.id == wtx2.id)
             assertFailsWith<IllegalArgumentException> { oracle.sign(ftx2, wtx2.id) }
         }
@@ -138,9 +154,7 @@ class NodeInterestRatesTest {
             tx.addCommand(fix, oracle.identity.owningKey)
             // Sign successfully.
             val wtx = tx.toWireTransaction()
-            fun filterCommands(c: Command) = oracle.identity.owningKey in c.signers && c.value is Fix
+            val ftx = wtx.buildFilteredTransaction { x -> fixCmdFilter(x) }
-            val filterFuns = FilterFuns(filterCommands = ::filterCommands)
-            val ftx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
             val signature = oracle.sign(ftx, wtx.id)
             tx.checkAndAddSignature(signature)
         }
@@ -154,9 +168,7 @@ class NodeInterestRatesTest {
             val badFix = Fix(fixOf, "0.6789".bd)
             tx.addCommand(badFix, oracle.identity.owningKey)
             val wtx = tx.toWireTransaction()
-            fun filterCommands(c: Command) = oracle.identity.owningKey in c.signers && c.value is Fix
+            val ftx = wtx.buildFilteredTransaction { x -> fixCmdFilter(x) }
-            val filterFuns = FilterFuns(filterCommands = ::filterCommands)
-            val ftx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
             val e1 = assertFailsWith<NodeInterestRates.UnknownFix> { oracle.sign(ftx, wtx.id) }
             assertEquals(fixOf, e1.fix)
         }
@@ -167,15 +179,28 @@ class NodeInterestRatesTest {
         databaseTransaction(database) {
             val tx = makeTX()
             val fix = oracle.query(listOf(NodeInterestRates.parseFixOf("LIBOR 2016-03-16 1M")), clock.instant()).first()
+            fun filtering(elem: Any): Boolean {
+                return when (elem) {
+                    is Command -> oracle.identity.owningKey in elem.signers && elem.value is Fix
+                    is TransactionState<ContractState> -> true
+                    else -> false
+                }
+            }
             tx.addCommand(fix, oracle.identity.owningKey)
             val wtx = tx.toWireTransaction()
-            fun filterCommands(c: Command) = oracle.identity.owningKey in c.signers && c.value is Fix
+            val ftx = wtx.buildFilteredTransaction(::filtering)
-            val filterFuns = FilterFuns(filterCommands = ::filterCommands, filterOutputs = { true })
-            val ftx = FilteredTransaction.buildMerkleTransaction(wtx, filterFuns)
             assertFailsWith<IllegalArgumentException> { oracle.sign(ftx, wtx.id) }
         }
     }
 
+    @Test
+    fun `empty partial transaction to sign`() {
+        val tx = makeTX()
+        val wtx = tx.toWireTransaction()
+        val ftx = wtx.buildFilteredTransaction({ false })
+        assertFailsWith<MerkleTreeException> { oracle.sign(ftx, wtx.id) }
+    }
+
     @Test
     fun `partial tree verification exception`() {
         databaseTransaction(database) {
@@ -183,7 +208,7 @@ class NodeInterestRatesTest {
             val wtx1 = tx.toWireTransaction()
             tx.addCommand(Cash.Commands.Move(), ALICE_PUBKEY)
             val wtx2 = tx.toWireTransaction()
-            val ftx2 = FilteredTransaction.buildMerkleTransaction(wtx2, FilterFuns(filterCommands = { true }))
+            val ftx2 = wtx2.buildFilteredTransaction { x -> filterCmds(x) }
             assertFalse(wtx1.id == wtx2.id)
             assertFailsWith<MerkleTreeException> { oracle.sign(ftx2, wtx1.id) }
         }
@@ -200,9 +225,7 @@ class NodeInterestRatesTest {
         val tx = TransactionType.General.Builder(null)
         val fixOf = NodeInterestRates.parseFixOf("LIBOR 2016-03-16 1M")
         val oracle = n2.info.serviceIdentities(NodeInterestRates.type).first()
-        fun filterCommands(c: Command) = oracle.owningKey in c.signers && c.value is Fix
+        val flow = FilteredRatesFlow(tx, oracle, fixOf, "0.675".bd, "0.1".bd)
-        val filterFuns = FilterFuns(filterCommands = ::filterCommands)
-        val flow = RatesFixFlow(tx, filterFuns, oracle, fixOf, "0.675".bd, "0.1".bd)
         LogHelper.setLevel("rates")
         net.runNetwork()
         val future = n1.services.startFlow(flow).resultFuture
@@ -214,5 +237,19 @@ class NodeInterestRatesTest {
         assertEquals("0.678".bd, fix.value)
     }
 
+    class FilteredRatesFlow(tx: TransactionBuilder,
+                            oracle: Party,
+                            fixOf: FixOf,
+                            expectedRate: BigDecimal,
+                            rateTolerance: BigDecimal,
+                            progressTracker: ProgressTracker = RatesFixFlow.tracker(fixOf.name)) : RatesFixFlow(tx, oracle, fixOf, expectedRate, rateTolerance, progressTracker) {
+        override fun filtering(elem: Any): Boolean {
+                return when (elem) {
+                    is Command -> oracle.owningKey in elem.signers && elem.value is Fix
+                    else -> false
+                }
+        }
+    }
+
     private fun makeTX() = TransactionType.General.Builder(DUMMY_NOTARY).withItems(1000.DOLLARS.CASH `issued by` DUMMY_CASH_ISSUER `owned by` ALICE_PUBKEY `with notary` DUMMY_NOTARY)
 }