Merged in mike-resolvetx-unit-tests (pull request #279)

Add unit tests for the tx resolution protocol

core/build.gradle

@@ -18,6 +18,9 @@ repositories {
     maven {
         url 'http://oss.sonatype.org/content/repositories/snapshots'
     }
+    maven {
+        url 'https://dl.bintray.com/kotlin/exposed'
+    }
 }
 
 sourceSets {
@@ -35,6 +38,9 @@ dependencies {
     // Guava: Google test library (collections test suite)
     testCompile "com.google.guava:guava-testlib:19.0"
 
+    // Bring in the MockNode infrastructure for writing protocol unit tests.
+    testCompile project(":node")
+
     compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
     compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
     compile "org.jetbrains.kotlin:kotlin-test:$kotlin_version"

core/src/main/kotlin/com/r3corda/core/Utils.kt

@@ -1,5 +1,6 @@
 package com.r3corda.core
 
+import com.google.common.base.Throwables
 import com.google.common.io.ByteStreams
 import com.google.common.util.concurrent.ListenableFuture
 import com.google.common.util.concurrent.MoreExecutors
@@ -215,3 +216,5 @@ fun extractZipFile(zipPath: Path, toPath: Path) {
 }
 
 // TODO: Generic csv printing utility for clases.
+
+val Throwable.rootCause: Throwable get() = Throwables.getRootCause(this)

core/src/main/kotlin/com/r3corda/core/contracts/DummyContract.kt

@@ -46,8 +46,21 @@ class DummyContract : Contract {
     // The "empty contract"
     override val legalContractReference: SecureHash = SecureHash.sha256("")
 
-    fun generateInitial(owner: PartyAndReference, magicNumber: Int, notary: Party): TransactionBuilder {
-        val state = SingleOwnerState(magicNumber, owner.party.owningKey)
-        return TransactionType.General.Builder(notary = notary).withItems(state, Command(Commands.Create(), owner.party.owningKey))
+    companion object {
+        @JvmStatic
+        fun generateInitial(owner: PartyAndReference, magicNumber: Int, notary: Party): TransactionBuilder {
+            val state = SingleOwnerState(magicNumber, owner.party.owningKey)
+            return TransactionType.General.Builder(notary = notary).withItems(state, Command(Commands.Create(), owner.party.owningKey))
+        }
+
+        fun move(prior: StateAndRef<DummyContract.SingleOwnerState>, newOwner: PublicKey): TransactionBuilder {
+            val priorState = prior.state.data
+            val (cmd, state) = priorState.withNewOwner(newOwner)
+            return TransactionType.General.Builder(notary = prior.state.notary).withItems(
+                    /* INPUT   */ prior,
+                    /* COMMAND */ Command(cmd, priorState.owner),
+                    /* OUTPUT  */ state
+            )
+        }
     }
 }

core/src/main/kotlin/com/r3corda/core/contracts/TransactionBuilder.kt

@@ -81,6 +81,7 @@ open class TransactionBuilder(
                 is TransactionState<*> -> addOutputState(t)
                 is ContractState -> addOutputState(t)
                 is Command -> addCommand(t)
+                is CommandData -> throw IllegalArgumentException("You passed an instance of CommandData, but that lacks the pubkey. You need to wrap it in a Command object first.")
                 else -> throw IllegalArgumentException("Wrong argument type: ${t.javaClass}")
             }
         }
@@ -120,8 +121,9 @@ open class TransactionBuilder(
     }
 
     /** Adds the signature directly to the transaction, without checking it for validity. */
-    fun addSignatureUnchecked(sig: DigitalSignature.WithKey) {
+    fun addSignatureUnchecked(sig: DigitalSignature.WithKey): TransactionBuilder {
         currentSigs.add(sig)
+        return this
     }
 
     fun toWireTransaction() = WireTransaction(ArrayList(inputs), ArrayList(attachments),

core/src/main/kotlin/com/r3corda/core/serialization/ByteArrays.kt

@@ -1,6 +1,7 @@
 package com.r3corda.core.serialization
 
 import com.google.common.io.BaseEncoding
+import java.io.ByteArrayInputStream
 import java.util.*
 
 /**
@@ -27,6 +28,9 @@ open class OpaqueBytes(val bits: ByteArray) {
     override fun toString() = "[" + BaseEncoding.base16().encode(bits) + "]"
 
     val size: Int get() = bits.size
+
+    /** Returns a [ByteArrayInputStream] of the bytes */
+    fun open() = ByteArrayInputStream(bits)
 }
 
 fun ByteArray.opaque(): OpaqueBytes = OpaqueBytes(this)

core/src/main/kotlin/com/r3corda/protocols/ResolveTransactionsProtocol.kt

@@ -41,6 +41,10 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
     private var stx: SignedTransaction? = null
     private var wtx: WireTransaction? = null
 
+    // TODO: Figure out a more appropriate DOS limit here, 5000 is simply a very bad guess.
+    /** The maximum number of transactions this protocol will try to download before bailing out. */
+    var transactionCountLimit = 5000
+
     /**
      * Resolve the full history of a transaction and verify it with its dependencies.
      */
@@ -65,11 +69,11 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
         // redundantly next time we attempt verification.
         val result = ArrayList<LedgerTransaction>()
 
-        for (tx in newTxns) {
+        for (stx in newTxns) {
             // Resolve to a LedgerTransaction and then run all contracts.
-            val ltx = tx.toLedgerTransaction(serviceHub)
+            val ltx = stx.toLedgerTransaction(serviceHub)
             ltx.verify()
-            serviceHub.recordTransactions(tx)
+            serviceHub.recordTransactions(stx)
             result += ltx
         }
 
@@ -114,6 +118,8 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
         nextRequests.addAll(depsToCheck)
         val resultQ = LinkedHashMap<SecureHash, SignedTransaction>()
 
+        val limit = transactionCountLimit
+        check(limit > 0) { "$limit is not a valid count limit" }
         var limitCounter = 0
         while (nextRequests.isNotEmpty()) {
             // Don't re-download the same tx when we haven't verified it yet but it's referenced multiple times in the
@@ -136,10 +142,8 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
             val inputHashes = downloads.flatMap { it.tx.inputs }.map { it.txhash }
             nextRequests.addAll(inputHashes)
 
-            // TODO: Figure out a more appropriate DOS limit here, 5000 is simply a very bad guess.
-            // TODO: Unit test the DoS limit.
             limitCounter = limitCounter checkedAdd nextRequests.size
-            if (limitCounter > 5000)
+            if (limitCounter > limit)
                 throw ExcessivelyLargeTransactionGraph()
         }
 
@@ -156,6 +160,7 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
         val missingAttachments = downloads.flatMap { wtx ->
             wtx.attachments.filter { serviceHub.storageService.attachments.openAttachment(it) == null }
         }
-        subProtocol(FetchAttachmentsProtocol(missingAttachments.toSet(), otherSide))
+        if (missingAttachments.isNotEmpty())
+            subProtocol(FetchAttachmentsProtocol(missingAttachments.toSet(), otherSide))
     }
 }

core/src/test/kotlin/com/r3corda/core/protocols/ResolveTransactionsProtocolTest.kt

@@ -0,0 +1,128 @@
+package com.r3corda.core.protocols
+
+import com.r3corda.core.contracts.DummyContract
+import com.r3corda.core.contracts.SignedTransaction
+import com.r3corda.core.crypto.NullSignature
+import com.r3corda.core.crypto.Party
+import com.r3corda.core.crypto.SecureHash
+import com.r3corda.core.serialization.opaque
+import com.r3corda.core.testing.*
+import com.r3corda.node.internal.testing.MockNetwork
+import com.r3corda.protocols.ResolveTransactionsProtocol
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+import java.security.SignatureException
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+import kotlin.test.assertNotNull
+import kotlin.test.assertNull
+
+class ResolveTransactionsProtocolTest {
+    lateinit var net: MockNetwork
+    lateinit var a: MockNetwork.MockNode
+    lateinit var b: MockNetwork.MockNode
+    lateinit var notary: Party
+
+    @Before
+    fun setup() {
+        net = MockNetwork()
+        val nodes = net.createSomeNodes()
+        a = nodes.partyNodes[0]
+        b = nodes.partyNodes[1]
+        notary = nodes.notaryNode.info.identity
+        net.runNetwork()
+    }
+
+    @After
+    fun tearDown() {
+        net.stopNodes()
+    }
+
+    @Test
+    fun `resolve from two hashes`() {
+        val (stx1, stx2) = makeTransactions()
+        val p = ResolveTransactionsProtocol(setOf(stx2.id), a.info.identity)
+        val future = b.services.startProtocol("resolve", p)
+        net.runNetwork()
+        val results = future.get()
+        assertEquals(listOf(stx1.id, stx2.id), results.map { it.id })
+        assertEquals(stx1, b.storage.validatedTransactions.getTransaction(stx1.id))
+        assertEquals(stx2, b.storage.validatedTransactions.getTransaction(stx2.id))
+    }
+
+    @Test
+    fun `dependency with an error`() {
+        val stx = makeTransactions(signFirstTX = false).second
+        val p = ResolveTransactionsProtocol(setOf(stx.id), a.info.identity)
+        val future = b.services.startProtocol("resolve", p)
+        net.runNetwork()
+        assertFailsWith(SignatureException::class) {
+            rootCauseExceptions { future.get() }
+        }
+    }
+
+    @Test
+    fun `resolve from a signed transaction`() {
+        val (stx1, stx2) = makeTransactions()
+        val p = ResolveTransactionsProtocol(stx2, a.info.identity)
+        val future = b.services.startProtocol("resolve", p)
+        net.runNetwork()
+        future.get()
+        assertEquals(stx1, b.storage.validatedTransactions.getTransaction(stx1.id))
+        // But stx2 wasn't inserted, just stx1.
+        assertNull(b.storage.validatedTransactions.getTransaction(stx2.id))
+    }
+
+    @Test
+    fun `denial of service check`() {
+        // Chain lots of txns together.
+        val stx2 = makeTransactions().second
+        val count = 50
+        var cursor = stx2
+        repeat(count) {
+            val stx = DummyContract.move(cursor.tx.outRef(0), MINI_CORP_PUBKEY)
+                    .addSignatureUnchecked(NullSignature)
+                    .toSignedTransaction(false)
+            a.services.recordTransactions(stx)
+            cursor = stx
+        }
+        val p = ResolveTransactionsProtocol(setOf(cursor.id), a.info.identity)
+        p.transactionCountLimit = 40
+        val future = b.services.startProtocol("resolve", p)
+        net.runNetwork()
+        assertFailsWith<ResolveTransactionsProtocol.ExcessivelyLargeTransactionGraph> {
+            rootCauseExceptions { future.get() }
+        }
+    }
+
+    @Test
+    fun attachment() {
+        val id = a.services.storageService.attachments.importAttachment("Some test file".toByteArray().opaque().open())
+        val stx2 = makeTransactions(withAttachment = id).second
+        val p = ResolveTransactionsProtocol(stx2, a.info.identity)
+        val future = b.services.startProtocol("resolve", p)
+        net.runNetwork()
+        future.get()
+        assertNotNull(b.services.storageService.attachments.openAttachment(id))
+    }
+
+    private fun makeTransactions(signFirstTX: Boolean = true, withAttachment: SecureHash? = null): Pair<SignedTransaction, SignedTransaction> {
+        // Make a chain of custody of dummy states and insert into node A.
+        val dummy1: SignedTransaction = DummyContract.generateInitial(MEGA_CORP.ref(1), 0, notary).let {
+            if (withAttachment != null)
+                it.addAttachment(withAttachment)
+            if (signFirstTX)
+                it.signWith(MEGA_CORP_KEY)
+            it.signWith(DUMMY_NOTARY_KEY)
+            it.toSignedTransaction(false)
+        }
+        val dummy2: SignedTransaction = DummyContract.move(dummy1.tx.outRef(0), MINI_CORP_PUBKEY).let {
+            it.signWith(MEGA_CORP_KEY)
+            it.signWith(DUMMY_NOTARY_KEY)
+            it.toSignedTransaction()
+        }
+        a.services.recordTransactions(dummy1, dummy2)
+        return Pair(dummy1, dummy2)
+    }
+}

core/src/test/kotlin/com/r3corda/protocols/ResolveTransactionsProtocolTest.kt

@@ -1,4 +0,0 @@
-package com.r3corda.protocols
-
-class ResolveTransactionsProtocolTest {
-}

docs/source/protocol-state-machines.rst

@@ -559,4 +559,106 @@ the sub-protocol to have the tracker it will use be passed in as a parameter. Th
 and linked ahead of time.
 
 In future, the progress tracking framework will become a vital part of how exceptions, errors, and other faults are
-surfaced to human operators for investigation and resolution.
+surfaced to human operators for investigation and resolution.
+
+Unit testing
+------------
+
+A protocol can be a fairly complex thing that interacts with many services and other parties over the network. That
+means unit testing one requires some infrastructure to provide lightweight mock implementations. The MockNetwork
+provides this testing infrastructure layer; you can find this class in the node module
+
+A good example to examine for learning how to unit test protocols is the ``ResolveTransactionsProtocol`` tests. This
+protocol takes care of downloading and verifying transaction graphs, with all the needed dependencies. We start
+with this basic skeleton:
+
+.. container:: codeset
+
+   .. sourcecode:: kotlin
+
+      class ResolveTransactionsProtocolTest {
+          lateinit var net: MockNetwork
+          lateinit var a: MockNetwork.MockNode
+          lateinit var b: MockNetwork.MockNode
+          lateinit var notary: Party
+
+          @Before
+          fun setup() {
+              net = MockNetwork()
+              val nodes = net.createSomeNodes()
+              a = nodes.partyNodes[0]
+              b = nodes.partyNodes[1]
+              notary = nodes.notaryNode.info.identity
+              net.runNetwork()
+          }
+
+          @After
+          fun tearDown() {
+              net.stopNodes()
+          }
+      }
+
+We create a mock network in our ``@Before`` setup method and create a couple of nodes. We also record the identity
+of the notary in our test network, which will come in handy later. We also tidy up when we're done.
+
+Next, we write a test case:
+
+.. container:: codeset
+
+   .. sourcecode:: kotlin
+
+      @Test
+      fun resolveFromTwoHashes() {
+          val (stx1, stx2) = makeTransactions()
+          val p = ResolveTransactionsProtocol(setOf(stx2.id), a.info.identity)
+          val future = b.services.startProtocol("resolve", p)
+          net.runNetwork()
+          val results = future.get()
+          assertEquals(listOf(stx1.id, stx2.id), results.map { it.id })
+          assertEquals(stx1, b.storage.validatedTransactions.getTransaction(stx1.id))
+          assertEquals(stx2, b.storage.validatedTransactions.getTransaction(stx2.id))
+      }
+
+We'll take a look at the ``makeTransactions`` function in a moment. For now, it's enough to know that it returns two
+``SignedTransaction`` objects, the second of which spends the first. Both transactions are known by node A
+but not node B.
+
+The test logic is simple enough: we create the protocol, giving it node A's identity as the target to talk to.
+Then we start it on node B and use the ``net.runNetwork()`` method to bounce messages around until things have
+settled (i.e. there are no more messages waiting to be delivered). All this is done using an in memory message
+routing implementation that is fast to initialise and use. Finally, we obtain the result of the protocol and do
+some tests on it. We also check the contents of node B's database to see that the protocol had the intended effect
+on the node's persistent state.
+
+Here's what ``makeTransactions`` looks like:
+
+.. container:: codeset
+
+   .. sourcecode:: kotlin
+
+      private fun makeTransactions(): Pair<SignedTransaction, SignedTransaction> {
+          // Make a chain of custody of dummy states and insert into node A.
+          val dummy1: SignedTransaction = DummyContract.generateInitial(MEGA_CORP.ref(1), 0, notary).let {
+              it.signWith(MEGA_CORP_KEY)
+              it.signWith(DUMMY_NOTARY_KEY)
+              it.toSignedTransaction(false)
+          }
+          val dummy2: SignedTransaction = DummyContract.move(dummy1.tx.outRef(0), MINI_CORP_PUBKEY).let {
+              it.signWith(MEGA_CORP_KEY)
+              it.signWith(DUMMY_NOTARY_KEY)
+              it.toSignedTransaction()
+          }
+          a.services.recordTransactions(dummy1, dummy2)
+          return Pair(dummy1, dummy2)
+      }
+
+We're using the ``DummyContract``, a simple test smart contract which stores a single number in its states, along
+with ownership and issuer information. You can issue such states, exit them and re-assign ownership (move them).
+It doesn't do anything else. This code simply creates a transaction that issues a dummy state (the issuer is
+``MEGA_CORP``, a pre-defined unit test identity), signs it with the test notary and MegaCorp keys and then
+converts the builder to the final ``SignedTransaction``. It then does so again, but this time instead of issuing
+it re-assigns ownership instead. The chain of two transactions is finally committed to node A by sending them
+directly to the ``a.services.recordTransaction`` method (note that this method doesn't check the transactions are
+valid).
+
+And that's it: you can explore the documentation for the `MockNode API <api/com.r3corda.node.internal.testing/-mock-network/index.html>`_ here.

node/src/main/kotlin/com/r3corda/node/internal/testing/MockNode.kt

@@ -11,6 +11,7 @@ import com.r3corda.core.node.services.ServiceType
 import com.r3corda.core.node.services.WalletService
 import com.r3corda.core.node.services.testing.MockIdentityService
 import com.r3corda.core.node.services.testing.makeTestDataSourceProperties
+import com.r3corda.core.testing.DUMMY_NOTARY_KEY
 import com.r3corda.core.testing.InMemoryWalletService
 import com.r3corda.core.utilities.loggerFor
 import com.r3corda.node.internal.AbstractNode
@@ -160,6 +161,7 @@ class MockNetwork(private val networkSendManuallyPumped: Boolean = false,
         }
     }
 
+    // TODO: Move this to using createSomeNodes which doesn't conflate network services with network users.
     /**
      * Sets up a two node network, in which the first node runs network map and notary services and the other
      * doesn't.
@@ -172,8 +174,35 @@ class MockNetwork(private val networkSendManuallyPumped: Boolean = false,
         )
     }
 
-    fun createNotaryNode(legalName: String? = null, keyPair: KeyPair? = null) = createNode(null, -1, defaultFactory, true, legalName, keyPair, false, NetworkMapService.Type, SimpleNotaryService.Type)
-    fun createPartyNode(networkMapAddr: NodeInfo, legalName: String? = null, keyPair: KeyPair? = null) = createNode(networkMapAddr, -1, defaultFactory, true, legalName, keyPair)
+    /**
+     * A bundle that separates the generic user nodes and service-providing nodes. A real network might not be so
+     * clearly separated, but this is convenient for testing.
+     */
+    data class BasketOfNodes(val partyNodes: List<MockNode>, val notaryNode: MockNode, val mapNode: MockNode)
+
+    /**
+     * Sets up a network with the requested number of nodes (defaulting to two), with one or more service nodes that
+     * run a notary, network map, any oracles etc. Can't be combined with [createTwoNodes].
+     */
+    fun createSomeNodes(numPartyNodes: Int = 2, nodeFactory: Factory = defaultFactory, notaryKeyPair: KeyPair? = DUMMY_NOTARY_KEY): BasketOfNodes {
+        require(nodes.isEmpty())
+        val mapNode = createNode(null, nodeFactory = nodeFactory, advertisedServices = NetworkMapService.Type)
+        val notaryNode = createNode(mapNode.info, nodeFactory = nodeFactory, keyPair = notaryKeyPair,
+                advertisedServices = SimpleNotaryService.Type)
+        val nodes = ArrayList<MockNode>()
+        repeat(numPartyNodes) {
+            nodes += createPartyNode(mapNode.info)
+        }
+        return BasketOfNodes(nodes, notaryNode, mapNode)
+    }
+
+    fun createNotaryNode(legalName: String? = null, keyPair: KeyPair? = null): MockNode {
+        return createNode(null, -1, defaultFactory, true, legalName, keyPair, false, NetworkMapService.Type, SimpleNotaryService.Type)
+    }
+
+    fun createPartyNode(networkMapAddr: NodeInfo, legalName: String? = null, keyPair: KeyPair? = null): MockNode {
+        return createNode(networkMapAddr, -1, defaultFactory, true, legalName, keyPair)
+    }
 
     @Suppress("unused")  // This is used from the network visualiser tool.
     fun addressToNode(address: SingleMessageRecipient): MockNode = nodes.single { it.net.myAddress == address }

node/src/test/kotlin/com/r3corda/node/services/NotaryChangeTests.kt

@@ -96,7 +96,7 @@ class NotaryChangeTests {
 }
 
 fun issueState(node: AbstractNode): StateAndRef<*> {
-    val tx = DummyContract().generateInitial(node.info.identity.ref(0), Random().nextInt(), DUMMY_NOTARY)
+    val tx = DummyContract.generateInitial(node.info.identity.ref(0), Random().nextInt(), DUMMY_NOTARY)
     tx.signWith(node.storage.myLegalIdentityKey)
     tx.signWith(DUMMY_NOTARY_KEY)
     val stx = tx.toSignedTransaction()
@@ -119,7 +119,7 @@ fun issueMultiPartyState(nodeA: AbstractNode, nodeB: AbstractNode): StateAndRef<
 }
 
 fun issueInvalidState(node: AbstractNode, notary: Party = DUMMY_NOTARY): StateAndRef<*> {
-    val tx = DummyContract().generateInitial(node.info.identity.ref(0), Random().nextInt(), notary)
+    val tx = DummyContract.generateInitial(node.info.identity.ref(0), Random().nextInt(), notary)
     tx.setTime(Instant.now(), 30.seconds)
     tx.signWith(node.storage.myLegalIdentityKey)
     val stx = tx.toSignedTransaction(false)