Resolve and insert attachments to the local store when resolving transactions. Attachments aren't yet exposed to contract code.

core/src/main/kotlin/core/Transactions.kt

@@ -30,7 +30,7 @@ import java.util.*
  * Views of a transaction as it progresses through the pipeline, from bytes loaded from disk/network to the object
  * tree passed into a contract.
  *
- * SignedWireTransaction wraps a serialized WireTransaction. It contains one or more ECDSA signatures, each one from
+ * SignedTransaction wraps a serialized WireTransaction. It contains one or more ECDSA signatures, each one from
  * a public key that is mentioned inside a transaction command.
  *
  * WireTransaction is a transaction in a form ready to be serialised/unserialised. A WireTransaction can be hashed
@@ -51,11 +51,13 @@ import java.util.*
  * All the above refer to inputs using a (txhash, output index) pair.
  *
  * TransactionForVerification is the same as LedgerTransaction but with the input states looked up from a local
- * database and replaced with the real objects. TFV is the form that is finally fed into the contracts.
+ * database and replaced with the real objects. Likewise, attachments are fully resolved at this point.
+ * TFV is the form that is finally fed into the contracts.
  */
 
 /** Transaction ready for serialisation, without any signatures attached. */
 data class WireTransaction(val inputs: List<StateRef>,
+                           val attachments: List<SecureHash>,
                            val outputs: List<ContractState>,
                            val commands: List<Command>) : NamedByHash {
 
@@ -77,7 +79,7 @@ data class WireTransaction(val inputs: List<StateRef>,
             val institutions = it.pubkeys.mapNotNull { pk -> identityService.partyFromKey(pk) }
             AuthenticatedObject(it.pubkeys, institutions, it.data)
         }
-        return LedgerTransaction(inputs, outputs, authenticatedArgs, id)
+        return LedgerTransaction(inputs, attachments, outputs, authenticatedArgs, id)
     }
 
     /** Serialises and returns this transaction as a [SignedTransaction] with no signatures attached. */
@@ -169,6 +171,7 @@ data class SignedTransaction(val txBits: SerializedBytes<WireTransaction>,
 
 /** A mutable transaction that's in the process of being built, before all signatures are present. */
 class TransactionBuilder(private val inputs: MutableList<StateRef> = arrayListOf(),
+                         private val attachments: MutableList<SecureHash> = arrayListOf(),
                          private val outputs: MutableList<ContractState> = arrayListOf(),
                          private val commands: MutableList<Command> = arrayListOf()) {
 
@@ -254,7 +257,8 @@ class TransactionBuilder(private val inputs: MutableList<StateRef> = arrayListOf
         currentSigs.add(sig)
     }
 
-    fun toWireTransaction() = WireTransaction(ArrayList(inputs), ArrayList(outputs), ArrayList(commands))
+    fun toWireTransaction() = WireTransaction(ArrayList(inputs), ArrayList(attachments),
+            ArrayList(outputs), ArrayList(commands))
 
     fun toSignedTransaction(checkSufficientSignatures: Boolean = true): SignedTransaction {
         if (checkSufficientSignatures) {
@@ -272,6 +276,11 @@ class TransactionBuilder(private val inputs: MutableList<StateRef> = arrayListOf
         inputs.add(ref)
     }
 
+    fun addAttachment(attachment: Attachment) {
+        check(currentSigs.isEmpty())
+        attachments.add(attachment.id)
+    }
+
     fun addOutputState(state: ContractState) {
         check(currentSigs.isEmpty())
         outputs.add(state)
@@ -291,6 +300,7 @@ class TransactionBuilder(private val inputs: MutableList<StateRef> = arrayListOf
     fun inputStates(): List<StateRef> = ArrayList(inputs)
     fun outputStates(): List<ContractState> = ArrayList(outputs)
     fun commands(): List<Command> = ArrayList(commands)
+    fun attachments(): List<SecureHash> = ArrayList(attachments)
 }
 
 /**
@@ -301,6 +311,8 @@ class TransactionBuilder(private val inputs: MutableList<StateRef> = arrayListOf
 data class LedgerTransaction(
         /** The input states which will be consumed/invalidated by the execution of this transaction. */
         val inputs: List<StateRef>,
+        /** A list of [Attachment] ids that need to be available for this transaction to verify. */
+        val attachments: List<SecureHash>,
         /** The states that will be generated by the execution of this transaction. */
         val outputs: List<ContractState>,
         /** Arbitrary data passed to the program of each input state. */
@@ -312,7 +324,7 @@ data class LedgerTransaction(
     fun <T : ContractState> outRef(index: Int) = StateAndRef(outputs[index] as T, StateRef(hash, index))
 
     fun toWireTransaction(): WireTransaction {
-        val wtx = WireTransaction(inputs, outputs, commands.map { Command(it.value, it.signers) })
+        val wtx = WireTransaction(inputs, attachments, outputs, commands.map { Command(it.value, it.signers) })
         check(wtx.serialize().hash == hash)
         return wtx
     }

src/main/kotlin/contracts/protocols/ResolveTransactionsProtocol.kt

@@ -14,10 +14,12 @@ import core.SignedTransaction
 import core.TransactionGroup
 import core.WireTransaction
 import core.crypto.SecureHash
-import core.protocols.ProtocolLogic
 import core.messaging.SingleMessageRecipient
+import core.protocols.ProtocolLogic
 import java.util.*
 
+// NB: This code is unit tested by TwoPartyTradeProtocolTests
+
 /**
  * This protocol fetches each transaction identified by the given hashes from either disk or network, along with all
  * their dependencies, and verifies them together using a single [TransactionGroup]. If no exception is thrown, then
@@ -100,7 +102,7 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
         // (2) If the identity service changes the assumed identity of one of the public keys, it's possible
         // that the "tx in db is valid" invariant is violated if one of the contracts checks the identity! Should
         // the db contain the identities that were resolved when the transaction was first checked, or should we
-        // accept this kind of change is possible?
+        // accept this kind of change is possible? Most likely solution is for identity data to be an attachment.
 
         val nextRequests = LinkedHashSet<SecureHash>()   // Keep things unique but ordered, for unit test stability.
         nextRequests.addAll(depsToCheck)
@@ -110,6 +112,9 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
             val (fromDisk, downloads) = subProtocol(FetchTransactionsProtocol(nextRequests, otherSide))
             nextRequests.clear()
 
+            // TODO: This could be done in parallel with other fetches for extra speed.
+            resolveMissingAttachments(downloads)
+
             // Resolve any legal identities from known public keys in the signatures.
             val downloadedTxns = downloads.map { it.verifyToLedgerTransaction(serviceHub.identityService) }
 
@@ -131,4 +136,12 @@ class ResolveTransactionsProtocol(private val txHashes: Set<SecureHash>,
                 throw ExcessivelyLargeTransactionGraph()
         }
     }
+
+    @Suspendable
+    private fun resolveMissingAttachments(downloads: List<SignedTransaction>) {
+        val missingAttachments = downloads.flatMap { stx ->
+            stx.tx.attachments.filter { serviceHub.storageService.attachments.openAttachment(it) == null }
+        }
+        subProtocol(FetchAttachmentsProtocol(missingAttachments.toSet(), otherSide))
+    }
 }

src/main/kotlin/core/TransactionVerification.kt

@@ -1,148 +0,0 @@
-/*
- * Copyright 2015 Distributed Ledger Group LLC.  Distributed as Licensed Company IP to DLG Group Members
- * pursuant to the August 7, 2015 Advisory Services Agreement and subject to the Company IP License terms
- * set forth therein.
- *
- * All other rights reserved.
- */
-
-package core
-
-import core.crypto.SecureHash
-import core.utilities.loggerFor
-import java.util.*
-
-class TransactionResolutionException(val hash: SecureHash) : Exception()
-class TransactionConflictException(val conflictRef: StateRef, val tx1: LedgerTransaction, val tx2: LedgerTransaction) : Exception()
-
-/**
- * A TransactionGroup defines a directed acyclic graph of transactions that can be resolved with each other and then
- * verified. Successful verification does not imply the non-existence of other conflicting transactions: simply that
- * this subgraph does not contain conflicts and is accepted by the involved contracts.
- *
- * The inputs of the provided transactions must be resolvable either within the [transactions] set, or from the
- * [nonVerifiedRoots] set. Transactions in the non-verified set are ignored other than for looking up input states.
- */
-class TransactionGroup(val transactions: Set<LedgerTransaction>, val nonVerifiedRoots: Set<LedgerTransaction>) {
-    companion object {
-        val logger = loggerFor<TransactionGroup>()
-    }
-
-    /**
-     * Verifies the group and returns the set of resolved transactions.
-     */
-    fun verify(programMap: ContractFactory): Set<TransactionForVerification> {
-        // Check that every input can be resolved to an output.
-        // Check that no output is referenced by more than one input.
-        // Cycles should be impossible due to the use of hashes as pointers.
-        check(transactions.intersect(nonVerifiedRoots).isEmpty())
-
-        val hashToTXMap: Map<SecureHash, List<LedgerTransaction>> = (transactions + nonVerifiedRoots).groupBy { it.hash }
-        val refToConsumingTXMap = hashMapOf<StateRef, LedgerTransaction>()
-
-        val resolved = HashSet<TransactionForVerification>(transactions.size)
-        for (tx in transactions) {
-            val inputs = ArrayList<ContractState>(tx.inputs.size)
-            for (ref in tx.inputs) {
-                val conflict = refToConsumingTXMap[ref]
-                if (conflict != null)
-                    throw TransactionConflictException(ref, tx, conflict)
-                refToConsumingTXMap[ref] = tx
-
-                // Look up the connecting transaction.
-                val ltx = hashToTXMap[ref.txhash]?.single() ?: throw TransactionResolutionException(ref.txhash)
-                // Look up the output in that transaction by index.
-                inputs.add(ltx.outputs[ref.index])
-            }
-            resolved.add(TransactionForVerification(inputs, tx.outputs, tx.commands, tx.hash))
-        }
-
-        for (tx in resolved)
-            tx.verify(programMap)
-
-        logger.trace("Successfully run the contracts for ${resolved.size} transaction(s)")
-
-        return resolved
-    }
-
-}
-
-/** A transaction in fully resolved and sig-checked form, ready for passing as input to a verification function. */
-data class TransactionForVerification(val inStates: List<ContractState>,
-                                      val outStates: List<ContractState>,
-                                      val commands: List<AuthenticatedObject<CommandData>>,
-                                      val origHash: SecureHash) {
-    override fun hashCode() = origHash.hashCode()
-    override fun equals(other: Any?) = other is TransactionForVerification && other.origHash == origHash
-
-    /**
-     * Runs the smart contracts governing this transaction.
-     *
-     * @throws TransactionVerificationException if a contract throws an exception, the original is in the cause field
-     * @throws IllegalStateException if a state refers to an unknown contract.
-     */
-    @Throws(TransactionVerificationException::class, IllegalStateException::class)
-    fun verify(programMap: ContractFactory) {
-        // For each input and output state, locate the program to run. Then execute the verification function. If any
-        // throws an exception, the entire transaction is invalid.
-        val programHashes = (inStates.map { it.programRef } + outStates.map { it.programRef }).toSet()
-        for (hash in programHashes) {
-            val program: Contract = programMap[hash]
-            try {
-                program.verify(this)
-            } catch(e: Throwable) {
-                throw TransactionVerificationException(this, program, e)
-            }
-        }
-    }
-
-    /**
-     * Utilities for contract writers to incorporate into their logic.
-     */
-
-    data class InOutGroup<T : ContractState>(val inputs: List<T>, val outputs: List<T>)
-
-    // A shortcut to make IDE auto-completion more intuitive for Java users.
-    fun getTimestampBy(timestampingAuthority: Party): TimestampCommand? = commands.getTimestampBy(timestampingAuthority)
-
-    // For Java users.
-    fun <T : ContractState> groupStates(ofType: Class<T>, selector: (T) -> Any): List<InOutGroup<T>> {
-        val inputs = inStates.filterIsInstance(ofType)
-        val outputs = outStates.filterIsInstance(ofType)
-
-        val inGroups = inputs.groupBy(selector)
-        val outGroups = outputs.groupBy(selector)
-
-        @Suppress("DEPRECATION")
-        return groupStatesInternal(inGroups, outGroups)
-    }
-
-    // For Kotlin users: this version has nicer syntax and avoids reflection/object creation for the lambda.
-    inline fun <reified T : ContractState> groupStates(selector: (T) -> Any): List<InOutGroup<T>> {
-        val inputs = inStates.filterIsInstance<T>()
-        val outputs = outStates.filterIsInstance<T>()
-
-        val inGroups = inputs.groupBy(selector)
-        val outGroups = outputs.groupBy(selector)
-
-        @Suppress("DEPRECATION")
-        return groupStatesInternal(inGroups, outGroups)
-    }
-
-    @Deprecated("Do not use this directly: exposed as public only due to function inlining")
-    fun <T : ContractState> groupStatesInternal(inGroups: Map<Any, List<T>>, outGroups: Map<Any, List<T>>): List<InOutGroup<T>> {
-        val result = ArrayList<InOutGroup<T>>()
-
-        for ((k, v) in inGroups.entries)
-            result.add(InOutGroup(v, outGroups[k] ?: emptyList()))
-        for ((k, v) in outGroups.entries) {
-            if (inGroups[k] == null)
-                result.add(InOutGroup(emptyList(), v))
-        }
-
-        return result
-    }
-}
-
-/** Thrown if a verification fails due to a contract rejection. */
-class TransactionVerificationException(val tx: TransactionForVerification, val contract: Contract, cause: Throwable?) : Exception(cause)

src/main/kotlin/core/node/TimestamperNodeService.kt

@@ -13,7 +13,10 @@ import co.paralleluniverse.fibers.Suspendable
 import core.*
 import core.crypto.DigitalSignature
 import core.crypto.signWithECDSA
-import core.messaging.*
+import core.messaging.LegallyIdentifiableNode
+import core.messaging.MessageRecipients
+import core.messaging.MessagingService
+import core.messaging.StateMachineManager
 import core.protocols.ProtocolLogic
 import core.serialization.SerializedBytes
 import core.serialization.deserialize

src/test/kotlin/contracts/CommercialPaperTests.kt

@@ -162,7 +162,7 @@ class CommercialPaperTestsGeneric {
     }
 
     fun cashOutputsToWallet(vararg states: Cash.State): Pair<LedgerTransaction, List<StateAndRef<Cash.State>>> {
-        val ltx = LedgerTransaction(emptyList(), listOf(*states), emptyList(), SecureHash.randomSHA256())
+        val ltx = LedgerTransaction(emptyList(), emptyList(), listOf(*states), emptyList(), SecureHash.randomSHA256())
         return Pair(ltx, states.mapIndexed { index, state -> StateAndRef(state, StateRef(ltx.hash, index)) })
     }
 

src/test/kotlin/contracts/CrowdFundTests.kt

@@ -99,7 +99,7 @@ class CrowdFundTests {
     }
 
     fun cashOutputsToWallet(vararg states: Cash.State): Pair<LedgerTransaction, List<StateAndRef<Cash.State>>> {
-        val ltx = LedgerTransaction(emptyList(), listOf(*states), emptyList(), SecureHash.randomSHA256())
+        val ltx = LedgerTransaction(emptyList(), emptyList(), listOf(*states), emptyList(), SecureHash.randomSHA256())
         return Pair(ltx, states.mapIndexed { index, state -> StateAndRef(state, StateRef(ltx.hash, index)) })
     }
 

src/test/kotlin/core/messaging/TwoPartyTradeProtocolTests.kt

@@ -22,11 +22,15 @@ import org.junit.After
 import org.junit.Before
 import org.junit.Test
 import org.slf4j.LoggerFactory
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
 import java.nio.file.Path
 import java.security.KeyPair
 import java.security.PublicKey
 import java.util.*
 import java.util.concurrent.ExecutionException
+import java.util.jar.JarOutputStream
+import java.util.zip.ZipEntry
 import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 import kotlin.test.assertTrue
@@ -62,7 +66,7 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
         transactionGroupFor<ContractState> {
             val (aliceNode, bobNode) = net.createTwoNodes()
             (bobNode.wallet as NodeWalletService).fillWithSomeTestCash(2000.DOLLARS)
-            val alicesFakePaper = fillUpForSeller(false, aliceNode.legallyIdentifableAddress.identity).second
+            val alicesFakePaper = fillUpForSeller(false, aliceNode.legallyIdentifableAddress.identity, null).second
 
             insertFakeTransactions(alicesFakePaper, aliceNode.services, aliceNode.storage.myLegalIdentityKey)
 
@@ -104,7 +108,7 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
             val timestamperAddr = aliceNode.legallyIdentifableAddress
 
             (bobNode.wallet as NodeWalletService).fillWithSomeTestCash(2000.DOLLARS)
-            val alicesFakePaper = fillUpForSeller(false, timestamperAddr.identity).second
+            val alicesFakePaper = fillUpForSeller(false, timestamperAddr.identity, null).second
 
             insertFakeTransactions(alicesFakePaper, aliceNode.services, aliceNode.storage.myLegalIdentityKey)
 
@@ -218,9 +222,18 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
             val timestamperAddr = aliceNode.legallyIdentifableAddress
             val bobNode = makeNodeWithTracking("bob")
 
+            // Insert a prospectus type attachment into the commercial paper transaction.
+            val stream = ByteArrayOutputStream()
+            JarOutputStream(stream).use {
+                it.putNextEntry(ZipEntry("Prospectus.txt"))
+                it.write("Our commercial paper is top notch stuff".toByteArray())
+                it.closeEntry()
+            }
+            val attachmentID = aliceNode.storage.attachments.importAttachment(ByteArrayInputStream(stream.toByteArray()))
+
             val bobsFakeCash = fillUpForBuyer(false, bobNode.keyManagement.freshKey().public).second
             val bobsSignedTxns = insertFakeTransactions(bobsFakeCash, bobNode.services)
-            val alicesFakePaper = fillUpForSeller(false, timestamperAddr.identity).second
+            val alicesFakePaper = fillUpForSeller(false, timestamperAddr.identity, attachmentID).second
             val alicesSignedTxns = insertFakeTransactions(alicesFakePaper, aliceNode.services, aliceNode.storage.myLegalIdentityKey)
 
             val buyerSessionID = random63BitValue()
@@ -260,6 +273,13 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
                         RecordingMap.Get(bobsFakeCash[0].id)
                 )
                 assertEquals(expected, records)
+
+                // Bob has downloaded the attachment.
+                bobNode.storage.attachments.openAttachment(attachmentID)!!.openAsJAR().use {
+                    it.nextJarEntry
+                    val contents = it.reader().readText()
+                    assertTrue(contents.contains("Our commercial paper is top notch stuff"))
+                }
             }
 
             // And from Alice's perspective ...
@@ -314,7 +334,7 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
 
         val bobKey = bobNode.keyManagement.freshKey()
         val bobsBadCash = fillUpForBuyer(bobError, bobKey.public).second
-        val alicesFakePaper = fillUpForSeller(aliceError, timestamperAddr.identity).second
+        val alicesFakePaper = fillUpForSeller(aliceError, timestamperAddr.identity, null).second
 
         insertFakeTransactions(bobsBadCash, bobNode.services, bobNode.storage.myLegalIdentityKey, bobKey)
         insertFakeTransactions(alicesFakePaper, aliceNode.services, aliceNode.storage.myLegalIdentityKey)
@@ -401,7 +421,7 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
         return Pair(wallet, listOf(eb1, bc1, bc2))
     }
 
-    private fun TransactionGroupDSL<ContractState>.fillUpForSeller(withError: Boolean, timestamper: Party): Pair<Wallet, List<WireTransaction>> {
+    private fun TransactionGroupDSL<ContractState>.fillUpForSeller(withError: Boolean, timestamper: Party, attachmentID: SecureHash?): Pair<Wallet, List<WireTransaction>> {
         val ap = transaction {
             output("alice's paper") {
                 CommercialPaper.State(MEGA_CORP.ref(1, 2, 3), ALICE, 1200.DOLLARS, TEST_TX_TIME + 7.days)
@@ -409,6 +429,8 @@ class TwoPartyTradeProtocolTests : TestWithInMemoryNetwork() {
             arg(MEGA_CORP_PUBKEY) { CommercialPaper.Commands.Issue() }
             if (!withError)
                 arg(timestamper.owningKey) { TimestampCommand(TEST_TX_TIME, 30.seconds) }
+            if (attachmentID != null)
+                attachment(attachmentID)
         }
 
         val wallet = Wallet(listOf<StateAndRef<Cash.State>>(lookup("alice's paper")))

src/test/kotlin/core/testutils/TestUtils.kt

@@ -110,6 +110,7 @@ class LabeledOutput(val label: String?, val state: ContractState) {
 infix fun ContractState.label(label: String) = LabeledOutput(label, this)
 
 abstract class AbstractTransactionForTest {
+    protected val attachments = ArrayList<SecureHash>()
     protected val outStates = ArrayList<LabeledOutput>()
     protected val commands  = ArrayList<Command>()
 
@@ -119,6 +120,10 @@ abstract class AbstractTransactionForTest {
         return commands.map { AuthenticatedObject(it.pubkeys, it.pubkeys.mapNotNull { TEST_KEYS_TO_CORP_MAP[it] }, it.data) }
     }
 
+    fun attachment(attachmentID: SecureHash) {
+        attachments.add(attachmentID)
+    }
+
     fun arg(vararg key: PublicKey, c: () -> CommandData) {
         val keys = listOf(*key)
         commands.add(Command(c(), keys))
@@ -223,7 +228,7 @@ class TransactionGroupDSL<T : ContractState>(private val stateType: Class<T>) {
             inStates.add(label.outputRef)
         }
 
-        fun toWireTransaction() = WireTransaction(inStates, outStates.map { it.state }, commands)
+        fun toWireTransaction() = WireTransaction(inStates, attachments, outStates.map { it.state }, commands)
     }
 
     val String.output: T get() = labelToOutputs[this] ?: throw IllegalArgumentException("State with label '$this' was not found")
@@ -257,7 +262,7 @@ class TransactionGroupDSL<T : ContractState>(private val stateType: Class<T>) {
     inner class Roots {
         fun transaction(vararg outputStates: LabeledOutput) {
             val outs = outputStates.map { it.state }
-            val wtx = WireTransaction(emptyList(), outs, emptyList())
+            val wtx = WireTransaction(emptyList(), emptyList(), outs, emptyList())
             for ((index, state) in outputStates.withIndex()) {
                 val label = state.label!!
                 labelToRefs[label] = StateRef(wtx.id, index)