CORDA-2083 verify transaction in AttachmentsClassloader (#4188)

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 fix tests

CORDA-2083 add support for explicit upgrade transactions

CORDA-2083 cleanup

CORDA-2083 cleanup

CORDA-2083 More cleanup

CORDA-2083 More cleanup

CORDA-2083 Clean up tests

CORDA-2083 Address code review comments

CORDA-2083 Fix merge

CORDA-2083 Fix merge

CORDA-2083 Address code review comments

revert file

CORDA-2083 Fix test

CORDA-2083 Add test

CORDA-2083 cleanup

CORDA-2083 Fix test

CORDA-2083 Address code review comments.

CORDA-2083 Remove unused functions.

CORDA-2083 Address code review comments.

CORDA-2083 Address code review comments.

CORDA-2083 Address code review comments.

CORDA-2083 Address code review comments.

CORDA-2083 Address code review comments.

core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt

@@ -27,6 +27,11 @@ fun isUploaderTrusted(uploader: String?): Boolean = uploader in TRUSTED_UPLOADER
 @KeepForDJVM
 abstract class AbstractAttachment(dataLoader: () -> ByteArray) : Attachment {
     companion object {
+        /**
+         * Returns a function that knows how to load an attachment.
+         *
+         * TODO - this code together with the rest of the Attachment handling (including [FetchedAttachment]) needs some refactoring as it is really hard to follow.
+         */
         @DeleteForDJVM
         fun SerializeAsTokenContext.attachmentDataLoader(id: SecureHash): () -> ByteArray {
             return {

core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt

@@ -1,6 +1,10 @@
 package net.corda.core.internal
 
 import net.corda.core.DeleteForDJVM
+import net.corda.core.KeepForDJVM
+import net.corda.core.contracts.ContractState
+import net.corda.core.contracts.StateRef
+import net.corda.core.contracts.TransactionState
 import net.corda.core.cordapp.Cordapp
 import net.corda.core.cordapp.CordappConfig
 import net.corda.core.cordapp.CordappContext
@@ -8,11 +12,14 @@ import net.corda.core.crypto.SecureHash
 import net.corda.core.flows.FlowLogic
 import net.corda.core.node.ServicesForResolution
 import net.corda.core.node.ZoneVersionTooLowException
+import net.corda.core.serialization.CordaSerializable
 import net.corda.core.serialization.SerializationContext
+import net.corda.core.serialization.SerializedBytes
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.transactions.WireTransaction
+import net.corda.core.utilities.OpaqueBytes
 import org.slf4j.MDC
 
 // *Internal* Corda-specific utilities
@@ -73,3 +80,11 @@ class LazyMappedList<T, U>(val originalList: List<T>, val transform: (T, Int) ->
     override fun get(index: Int) = partialResolvedList[index]
             ?: transform(originalList[index], index).also { computed -> partialResolvedList[index] = computed }
 }
+
+/**
+ * A SerializedStateAndRef is a pair (BinaryStateRepresentation, StateRef).
+ * The [serializedState] is the actual component from the original transaction.
+ */
+@KeepForDJVM
+@CordaSerializable
+data class SerializedStateAndRef(val serializedState: SerializedBytes<TransactionState<ContractState>>, val ref: StateRef)

core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt

@@ -221,15 +221,16 @@ data class InputStreamAndHash(val inputStream: InputStream, val sha256: SecureHa
          * Note that a slightly bigger than numOfExpectedBytes size is expected.
          */
         @DeleteForDJVM
-        fun createInMemoryTestZip(numOfExpectedBytes: Int, content: Byte): InputStreamAndHash {
+        fun createInMemoryTestZip(numOfExpectedBytes: Int, content: Byte, entryName: String = "z"): InputStreamAndHash {
             require(numOfExpectedBytes > 0){"Expected bytes must be greater than zero"}
+            require(numOfExpectedBytes > 0)
             val baos = ByteArrayOutputStream()
             ZipOutputStream(baos).use { zos ->
                 val arraySize = 1024
                 val bytes = ByteArray(arraySize) { content }
                 val n = (numOfExpectedBytes - 1) / arraySize + 1 // same as Math.ceil(numOfExpectedBytes/arraySize).
                 zos.setLevel(Deflater.NO_COMPRESSION)
-                zos.putNextEntry(ZipEntry("z"))
+                zos.putNextEntry(ZipEntry(entryName))
                 for (i in 0 until n) {
                     zos.write(bytes, 0, arraySize)
                 }
@@ -501,3 +502,18 @@ fun <T : Any> SerializedBytes<Any>.checkPayloadIs(type: Class<T>): Untrustworthy
     return type.castIfPossible(payloadData)?.let { UntrustworthyData(it) }
             ?: throw IllegalArgumentException("We were expecting a ${type.name} but we instead got a ${payloadData.javaClass.name} ($payloadData)")
 }
+
+/**
+ * Simple Map structure that can be used as a cache in the DJVM.
+ */
+fun <K, V> createSimpleCache(maxSize: Int, onEject: (MutableMap.MutableEntry<K, V>) -> Unit = {}): MutableMap<K, V> {
+    return object : LinkedHashMap<K, V>() {
+        override fun removeEldestEntry(eldest: MutableMap.MutableEntry<K, V>?): Boolean {
+            val eject = size > maxSize
+            if (eject) onEject(eldest!!)
+            return eject
+        }
+    }
+}
+
+fun <K,V> MutableMap<K,V>.toSynchronised(): MutableMap<K,V> = Collections.synchronizedMap(this)

core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt

@@ -1,15 +1,23 @@
 package net.corda.core.internal
 
-import net.corda.core.contracts.ContractClassName
+import net.corda.core.contracts.*
-import net.corda.core.contracts.PrivacySalt
-import net.corda.core.contracts.StateRef
 import net.corda.core.crypto.SecureHash
+import net.corda.core.crypto.componentHash
 import net.corda.core.crypto.sha256
 import net.corda.core.identity.Party
+import net.corda.core.serialization.MissingAttachmentsException
+import net.corda.core.serialization.SerializationContext
+import net.corda.core.serialization.SerializationFactory
 import net.corda.core.serialization.serialize
+import net.corda.core.transactions.ComponentGroup
 import net.corda.core.transactions.ContractUpgradeWireTransaction
+import net.corda.core.transactions.FilteredComponentGroup
 import net.corda.core.transactions.NotaryChangeWireTransaction
+import net.corda.core.utilities.OpaqueBytes
+import net.corda.core.utilities.lazyMapped
 import java.io.ByteArrayOutputStream
+import java.security.PublicKey
+import kotlin.reflect.KClass
 
 /** Constructs a [NotaryChangeWireTransaction]. */
 class NotaryChangeTransactionBuilder(val inputs: List<StateRef>,
@@ -43,3 +51,74 @@ fun combinedHash(components: Iterable<SecureHash>): SecureHash {
     }
     return stream.toByteArray().sha256()
 }
+
+/**
+ * This function knows how to deserialize a transaction component group.
+ *
+ * In case the [componentGroups] is an instance of [LazyMappedList], this function will just use the original deserialized version, and avoid an unnecessary deserialization.
+ * The [forceDeserialize] will force deserialization. In can be used in case the SerializationContext changes.
+ */
+fun <T : Any> deserialiseComponentGroup(componentGroups: List<ComponentGroup>,
+                                        clazz: KClass<T>,
+                                        groupEnum: ComponentGroupEnum,
+                                        forceDeserialize: Boolean = false,
+                                        factory: SerializationFactory = SerializationFactory.defaultFactory,
+                                        context: SerializationContext = factory.defaultContext): List<T> {
+    val group = componentGroups.firstOrNull { it.groupIndex == groupEnum.ordinal }
+
+    if (group == null || group.components.isEmpty()) {
+        return emptyList()
+    }
+
+    // If the componentGroup is a [LazyMappedList] it means that the original deserialized version is already available.
+    val components = group.components
+    if (!forceDeserialize && components is LazyMappedList<*, OpaqueBytes>) {
+        return components.originalList as List<T>
+    }
+
+    return components.lazyMapped { component, internalIndex ->
+        try {
+            factory.deserialize(component, clazz.java, context)
+        } catch (e: MissingAttachmentsException) {
+            throw e
+        } catch (e: Exception) {
+            throw Exception("Malformed transaction, $groupEnum at index $internalIndex cannot be deserialised", e)
+        }
+    }
+}
+
+/**
+ * Method to deserialise Commands from its two groups:
+ *  * COMMANDS_GROUP which contains the CommandData part
+ *  * and SIGNERS_GROUP which contains the Signers part.
+ *
+ *  This method used the [deserialiseComponentGroup] method.
+ */
+fun deserialiseCommands(componentGroups: List<ComponentGroup>,
+                        forceDeserialize: Boolean = false,
+                        factory: SerializationFactory = SerializationFactory.defaultFactory,
+                        context: SerializationContext = factory.defaultContext): List<Command<*>> {
+    // TODO: we could avoid deserialising unrelated signers.
+    //      However, current approach ensures the transaction is not malformed
+    //      and it will throw if any of the signers objects is not List of public keys).
+    val signersList: List<List<PublicKey>> = uncheckedCast(deserialiseComponentGroup(componentGroups, List::class, ComponentGroupEnum.SIGNERS_GROUP, forceDeserialize))
+    val commandDataList: List<CommandData> = deserialiseComponentGroup(componentGroups, CommandData::class, ComponentGroupEnum.COMMANDS_GROUP, forceDeserialize)
+    val group = componentGroups.firstOrNull { it.groupIndex == ComponentGroupEnum.COMMANDS_GROUP.ordinal }
+    return if (group is FilteredComponentGroup) {
+        check(commandDataList.size <= signersList.size) {
+            "Invalid Transaction. Less Signers (${signersList.size}) than CommandData (${commandDataList.size}) objects"
+        }
+        val componentHashes = group.components.mapIndexed { index, component -> componentHash(group.nonces[index], component) }
+        val leafIndices = componentHashes.map { group.partialMerkleTree.leafIndex(it) }
+        if (leafIndices.isNotEmpty())
+            check(leafIndices.max()!! < signersList.size) { "Invalid Transaction. A command with no corresponding signer detected" }
+        commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[leafIndices[index]]) }
+    } else {
+        // It is a WireTransaction
+        // or a FilteredTransaction with no Commands (in which case group is null).
+        check(commandDataList.size == signersList.size) {
+            "Invalid Transaction. Sizes of CommandData (${commandDataList.size}) and Signers (${signersList.size}) do not match"
+        }
+        commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[index]) }
+    }
+}

core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt

@@ -177,10 +177,10 @@ interface SerializationContext {
     fun withClassLoader(classLoader: ClassLoader): SerializationContext
 
     /**
-     * Helper method to return a new context based on this context with the appropriate class loader constructed from the passed attachment identifiers.
+     * Does not do anything.
-     * (Requires the attachment storage to have been enabled).
      */
     @Throws(MissingAttachmentsException::class)
+    @Deprecated("There is no reason to call this. This method does not actually do anything.")
     fun withAttachmentsClassLoader(attachmentHashes: List<SecureHash>): SerializationContext
 
     /**

core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt

@@ -0,0 +1,161 @@
+package net.corda.core.serialization.internal
+
+import net.corda.core.contracts.Attachment
+import net.corda.core.contracts.ContractAttachment
+import net.corda.core.crypto.SecureHash
+import net.corda.core.internal.VisibleForTesting
+import net.corda.core.internal.isUploaderTrusted
+import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.SerializationFactory
+import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory.toUrl
+import net.corda.core.internal.createSimpleCache
+import net.corda.core.internal.toSynchronised
+import java.io.IOException
+import java.io.InputStream
+import java.net.*
+
+/**
+ * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only
+ * need to provide JAR streams, and so could be fetched from a database, local disk, etc. Constructing an
+ * AttachmentsClassLoader is somewhat expensive, as every attachment is scanned to ensure that there are no overlapping
+ * file paths.
+ */
+class AttachmentsClassLoader(attachments: List<Attachment>, parent: ClassLoader = ClassLoader.getSystemClassLoader()) :
+        URLClassLoader(attachments.map(::toUrl).toTypedArray(), parent) {
+
+    companion object {
+
+        init {
+            // This is required to register the AttachmentURLStreamHandlerFactory.
+            URL.setURLStreamHandlerFactory(AttachmentURLStreamHandlerFactory)
+        }
+
+        private const val `META-INF` = "meta-inf"
+        private val excludeFromNoOverlapCheck = setOf(
+                "manifest.mf",
+                "license",
+                "license.txt",
+                "notice",
+                "notice.txt",
+                "index.list"
+        )
+
+        private fun shouldCheckForNoOverlap(path: String): Boolean {
+            if (!path.startsWith(`META-INF`)) return true
+            val p = path.substring(`META-INF`.length + 1)
+            if (p in excludeFromNoOverlapCheck) return false
+            if (p.endsWith(".sf") || p.endsWith(".dsa")) return false
+            return true
+        }
+
+        @CordaSerializable
+        class OverlappingAttachments(val path: String) : Exception() {
+            override fun toString() = "Multiple attachments define a file at path $path"
+        }
+
+        private fun requireNoDuplicates(attachments: List<Attachment>) {
+            val classLoaderEntries = mutableSetOf<String>()
+            for (attachment in attachments) {
+                attachment.openAsJAR().use { jar ->
+                    while (true) {
+                        val entry = jar.nextJarEntry ?: break
+
+                        // We already verified that paths are not strange/game playing when we inserted the attachment
+                        // into the storage service. So we don't need to repeat it here.
+                        //
+                        // We forbid files that differ only in case, or path separator to avoid issues for Windows/Mac developers where the
+                        // filesystem tries to be case insensitive. This may break developers who attempt to use ProGuard.
+                        //
+                        // Also convert to Unix path separators as all resource/class lookups will expect this.
+                        // If 2 entries have the same CRC, it means the same file is present in both attachments, so that is ok. TODO - Mike, wdyt?
+                        val path = entry.name.toLowerCase().replace('\\', '/')
+                        if (shouldCheckForNoOverlap(path)) {
+                            if (path in classLoaderEntries) throw OverlappingAttachments(path)
+                            classLoaderEntries.add(path)
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    init {
+        require(attachments.mapNotNull { it as? ContractAttachment }.all { isUploaderTrusted(it.uploader) }) {
+            "Attempting to load Contract Attachments downloaded from the network"
+        }
+
+        requireNoDuplicates(attachments)
+    }
+}
+
+/**
+ * This is just a factory that provides a cache to avoid constructing expensive [AttachmentsClassLoader]s.
+ */
+@VisibleForTesting
+internal object AttachmentsClassLoaderBuilder {
+
+    private const val ATTACHMENT_CLASSLOADER_CACHE_SIZE = 1000
+
+    // This runs in the DJVM so it can't use caffeine.
+    private val cache: MutableMap<List<SecureHash>, AttachmentsClassLoader> = createSimpleCache<List<SecureHash>, AttachmentsClassLoader>(ATTACHMENT_CLASSLOADER_CACHE_SIZE)
+            .toSynchronised()
+
+    fun build(attachments: List<Attachment>): AttachmentsClassLoader {
+        return cache.computeIfAbsent(attachments.map { it.id }.sorted()) {
+            AttachmentsClassLoader(attachments)
+        }
+    }
+
+    fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>, block: (ClassLoader) -> T): T {
+
+        // Create classloader from the attachments.
+        val transactionClassLoader = AttachmentsClassLoaderBuilder.build(attachments)
+
+        // Create a new serializationContext for the current Transaction.
+        val transactionSerializationContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(transactionClassLoader)
+
+        // Deserialize all relevant classes in the transaction classloader.
+        return SerializationFactory.defaultFactory.withCurrentContext(transactionSerializationContext) {
+            block(transactionClassLoader)
+        }
+    }
+}
+
+/**
+ * Registers a new internal "attachment" protocol.
+ * This will not be exposed as an API.
+ */
+object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory {
+    private const val attachmentScheme = "attachment"
+
+    // TODO - what happens if this grows too large?
+    private val loadedAttachments = mutableMapOf<String, Attachment>().toSynchronised()
+
+    override fun createURLStreamHandler(protocol: String): URLStreamHandler? {
+        return if (attachmentScheme == protocol) {
+            AttachmentURLStreamHandler
+        } else null
+    }
+
+    fun toUrl(attachment: Attachment): URL {
+        val id = attachment.id.toString()
+        loadedAttachments[id] = attachment
+        return URL(attachmentScheme, "", -1, id, AttachmentURLStreamHandler)
+    }
+
+    private object AttachmentURLStreamHandler : URLStreamHandler() {
+        override fun openConnection(url: URL): URLConnection {
+            if (url.protocol != attachmentScheme) throw IOException("Cannot handle protocol: ${url.protocol}")
+            val attachment = loadedAttachments[url.path] ?: throw IOException("Could not load url: $url .")
+            return AttachmentURLConnection(url, attachment)
+        }
+    }
+
+    private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) {
+        override fun getContentLengthLong(): Long = attachment.size.toLong()
+        override fun getInputStream(): InputStream = attachment.open()
+        override fun connect() {
+            connected = true
+        }
+    }
+}

core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt

@@ -73,13 +73,6 @@ interface CheckpointSerializationContext {
      */
     fun withClassLoader(classLoader: ClassLoader): CheckpointSerializationContext
 
-    /**
-     * Helper method to return a new context based on this context with the appropriate class loader constructed from the passed attachment identifiers.
-     * (Requires the attachment storage to have been enabled).
-     */
-    @Throws(MissingAttachmentsException::class)
-    fun withAttachmentsClassLoader(attachmentHashes: List<SecureHash>): CheckpointSerializationContext
-
     /**
      * Helper method to return a new context based on this context with the given class specifically whitelisted.
      */

core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt

@@ -1,5 +1,6 @@
 package net.corda.core.transactions
 
+import net.corda.core.CordaInternal
 import net.corda.core.KeepForDJVM
 import net.corda.core.contracts.*
 import net.corda.core.crypto.SecureHash
@@ -11,10 +12,12 @@ import net.corda.core.internal.AttachmentWithContext
 import net.corda.core.internal.combinedHash
 import net.corda.core.node.NetworkParameters
 import net.corda.core.node.ServicesForResolution
-import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.*
-import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
 import net.corda.core.transactions.ContractUpgradeFilteredTransaction.FilteredComponent
+import net.corda.core.transactions.ContractUpgradeWireTransaction.Companion.calculateUpgradedState
 import net.corda.core.transactions.ContractUpgradeWireTransaction.Component.*
+import net.corda.core.transactions.WireTransaction.Companion.resolveStateRefBinaryComponent
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.toBase58String
 import java.security.PublicKey
@@ -35,6 +38,32 @@ data class ContractUpgradeWireTransaction(
         /** Required for hiding components in [ContractUpgradeFilteredTransaction]. */
         val privacySalt: PrivacySalt = PrivacySalt()
 ) : CoreTransaction() {
+
+    companion object {
+        /**
+         * Runs the explicit upgrade logic.
+         */
+        @CordaInternal
+        internal fun <T : ContractState, S : ContractState> calculateUpgradedState(state: TransactionState<T>, upgradedContract: UpgradedContract<T, S>, upgradedContractAttachment: Attachment): TransactionState<S> {
+            // TODO: if there are encumbrance states in the inputs, just copy them across without modifying
+            val upgradedState: S = upgradedContract.upgrade(state.data)
+            val inputConstraint = state.constraint
+            val outputConstraint = when (inputConstraint) {
+                is HashAttachmentConstraint -> HashAttachmentConstraint(upgradedContractAttachment.id)
+                WhitelistedByZoneAttachmentConstraint -> WhitelistedByZoneAttachmentConstraint
+                else -> throw IllegalArgumentException("Unsupported input contract constraint $inputConstraint")
+            }
+            // TODO: re-map encumbrance pointers
+            return TransactionState(
+                    data = upgradedState,
+                    contract = upgradedContract::class.java.name,
+                    constraint = outputConstraint,
+                    notary = state.notary,
+                    encumbrance = state.encumbrance
+            )
+        }
+    }
+
     override val inputs: List<StateRef> = serializedComponents[INPUTS.ordinal].deserialize()
     override val notary: Party by lazy { serializedComponents[NOTARY.ordinal].deserialize<Party>() }
     val legacyContractAttachmentId: SecureHash by lazy { serializedComponents[LEGACY_ATTACHMENT.ordinal].deserialize<SecureHash>() }
@@ -90,6 +119,32 @@ data class ContractUpgradeWireTransaction(
         )
     }
 
+    private fun upgradedContract(className: ContractClassName, classLoader: ClassLoader): UpgradedContract<ContractState, ContractState> = try {
+        classLoader.loadClass(className).asSubclass(UpgradedContract::class.java as Class<UpgradedContract<ContractState, ContractState>>)
+                .newInstance()
+    } catch (e: Exception) {
+        throw TransactionVerificationException.ContractCreationError(id, className, e)
+    }
+
+    /**
+     * Creates a binary serialized component for a virtual output state serialised and executed with the attachments from the transaction.
+     */
+    @CordaInternal
+    internal fun resolveOutputComponent(services: ServicesForResolution, stateRef: StateRef): SerializedBytes<TransactionState<ContractState>> {
+        val binaryInput = resolveStateRefBinaryComponent(inputs[stateRef.index], services)!!
+        val legacyAttachment = services.attachments.openAttachment(legacyContractAttachmentId)
+                ?: throw MissingContractAttachments(emptyList())
+        val upgradedAttachment = services.attachments.openAttachment(upgradedContractAttachmentId)
+                ?: throw MissingContractAttachments(emptyList())
+
+        return AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(listOf(legacyAttachment, upgradedAttachment)) { transactionClassLoader ->
+            val resolvedInput = binaryInput.deserialize<TransactionState<ContractState>>()
+            val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader)
+            val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment)
+            outputState.serialize()
+        }
+    }
+
     /** Constructs a filtered transaction: the inputs and the notary party are always visible, while the rest are hidden. */
     fun buildFilteredTransaction(): ContractUpgradeFilteredTransaction {
         val totalComponents = (0 until serializedComponents.size).toSet()
@@ -222,22 +277,7 @@ data class ContractUpgradeLedgerTransaction(
      * Outputs are computed by running the contract upgrade logic on input states. This is done eagerly so that the
      * transaction is verified during construction.
      */
-    override val outputs: List<TransactionState<ContractState>> = inputs.map { (state) ->
+    override val outputs: List<TransactionState<ContractState>> = inputs.map { calculateUpgradedState(it.state, upgradedContract, upgradedContractAttachment) }
-        // TODO: if there are encumbrance states in the inputs, just copy them across without modifying
-        val upgradedState = upgradedContract.upgrade(state.data)
-        val inputConstraint = state.constraint
-        val outputConstraint = when (inputConstraint) {
-            is HashAttachmentConstraint -> HashAttachmentConstraint(upgradedContractAttachment.id)
-            WhitelistedByZoneAttachmentConstraint -> WhitelistedByZoneAttachmentConstraint
-            else -> throw IllegalArgumentException("Unsupported input contract constraint $inputConstraint")
-        }
-        // TODO: re-map encumbrance pointers
-        state.copy(
-                data = upgradedState,
-                contract = upgradedContractClassName,
-                constraint = outputConstraint
-        )
-    }
 
     /** The required signers are the set of all input states' participants. */
     override val requiredSigningKeys: Set<PublicKey>

core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt

@@ -1,22 +1,21 @@
 package net.corda.core.transactions
 
+import net.corda.core.CordaInternal
 import net.corda.core.KeepForDJVM
 import net.corda.core.contracts.*
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.isFulfilledBy
 import net.corda.core.identity.Party
-import net.corda.core.internal.AttachmentWithContext
+import net.corda.core.internal.*
-import net.corda.core.internal.castIfPossible
-import net.corda.core.internal.checkMinimumPlatformVersion
-import net.corda.core.internal.uncheckedCast
 import net.corda.core.node.NetworkParameters
 import net.corda.core.serialization.CordaSerializable
-import net.corda.core.utilities.Try
+import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
 import net.corda.core.utilities.loggerFor
+import net.corda.core.utilities.warnOnce
 import java.util.*
 import java.util.function.Predicate
 import kotlin.collections.HashSet
-import net.corda.core.utilities.warnOnce
 
 /**
  * A LedgerTransaction is derived from a [WireTransaction]. It is the result of doing the following operations:
@@ -34,7 +33,7 @@ import net.corda.core.utilities.warnOnce
 // DOCSTART 1
 @KeepForDJVM
 @CordaSerializable
-data class LedgerTransaction @JvmOverloads constructor(
+data class LedgerTransaction private constructor(
         /** The resolved input states which will be consumed/invalidated by the execution of this transaction. */
         override val inputs: List<StateAndRef<ContractState>>,
         override val outputs: List<TransactionState<ContractState>>,
@@ -47,9 +46,38 @@ data class LedgerTransaction @JvmOverloads constructor(
         override val notary: Party?,
         val timeWindow: TimeWindow?,
         val privacySalt: PrivacySalt,
-        private val networkParameters: NetworkParameters? = null,
+        private val networkParameters: NetworkParameters?,
-        override val references: List<StateAndRef<ContractState>> = emptyList()
+        override val references: List<StateAndRef<ContractState>>,
+        val componentGroups: List<ComponentGroup>?,
+        val resolvedInputBytes: List<SerializedStateAndRef>?,
+        val resolvedReferenceBytes: List<SerializedStateAndRef>?
 ) : FullTransaction() {
+
+    @Deprecated("Client code should not instantiate LedgerTransaction.")
+    constructor(
+            inputs: List<StateAndRef<ContractState>>,
+            outputs: List<TransactionState<ContractState>>,
+            commands: List<CommandWithParties<CommandData>>,
+            attachments: List<Attachment>,
+            id: SecureHash,
+            notary: Party?,
+            timeWindow: TimeWindow?,
+            privacySalt: PrivacySalt
+    ) : this(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, null, emptyList(), null, null, null)
+
+    @Deprecated("Client code should not instantiate LedgerTransaction.")
+    constructor(
+            inputs: List<StateAndRef<ContractState>>,
+            outputs: List<TransactionState<ContractState>>,
+            commands: List<CommandWithParties<CommandData>>,
+            attachments: List<Attachment>,
+            id: SecureHash,
+            notary: Party?,
+            timeWindow: TimeWindow?,
+            privacySalt: PrivacySalt,
+            networkParameters: NetworkParameters?
+    ) : this(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, emptyList(), null, null, null)
+
     //DOCEND 1
     init {
         checkBaseInvariants()
@@ -58,19 +86,25 @@ data class LedgerTransaction @JvmOverloads constructor(
         checkEncumbrancesValid()
     }
 
-    private companion object {
+    companion object {
-        val logger = loggerFor<LedgerTransaction>()
+        private val logger = loggerFor<LedgerTransaction>()
-        private fun contractClassFor(className: ContractClassName, classLoader: ClassLoader?): Try<Class<out Contract>> {
-            return Try.on {
-                (classLoader ?: this::class.java.classLoader)
-                        .loadClass(className)
-                        .asSubclass(Contract::class.java)
-            }
-        }
 
-        private fun stateToContractClass(state: TransactionState<ContractState>): Try<Class<out Contract>> {
+        @CordaInternal
-            return contractClassFor(state.contract, state.data::class.java.classLoader)
+        internal fun makeLedgerTransaction(
-        }
+                inputs: List<StateAndRef<ContractState>>,
+                outputs: List<TransactionState<ContractState>>,
+                commands: List<CommandWithParties<CommandData>>,
+                attachments: List<Attachment>,
+                id: SecureHash,
+                notary: Party?,
+                timeWindow: TimeWindow?,
+                privacySalt: PrivacySalt,
+                networkParameters: NetworkParameters?,
+                references: List<StateAndRef<ContractState>>,
+                componentGroups: List<ComponentGroup>,
+                resolvedInputBytes: List<SerializedStateAndRef>,
+                resolvedReferenceBytes: List<SerializedStateAndRef>
+        ) = LedgerTransaction(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, resolvedInputBytes, resolvedReferenceBytes)
     }
 
     val inputStates: List<ContractState> get() = inputs.map { it.state.data }
@@ -88,6 +122,12 @@ data class LedgerTransaction @JvmOverloads constructor(
 
     /**
      * Verifies this transaction and runs contract code. At this stage it is assumed that signatures have already been verified.
+
+     * The contract verification logic is run in a custom [AttachmentsClassLoader] created for the current transaction.
+     * This classloader is only used during verification and does not leak to the client code.
+     *
+     * The reason for this is that classes (contract states) deserialized in this classloader would actually be a different type from what
+     * the calling code would expect.
      *
      * @throws TransactionVerificationException if anything goes wrong.
      */
@@ -95,12 +135,17 @@ data class LedgerTransaction @JvmOverloads constructor(
     fun verify() {
         val contractAttachmentsByContract: Map<ContractClassName, ContractAttachment> = getUniqueContractAttachmentsByContract()
 
-        // TODO - verify for version downgrade
+        AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(this.attachments) { transactionClassLoader ->
-        validatePackageOwnership(contractAttachmentsByContract)
+
-        validateStatesAgainstContract()
+            val internalTx = createInternalLedgerTransaction()
-        verifyConstraintsValidity(contractAttachmentsByContract)
+
-        verifyConstraints(contractAttachmentsByContract)
+            // TODO - verify for version downgrade
-        verifyContracts()
+            validatePackageOwnership(contractAttachmentsByContract)
+            validateStatesAgainstContract(internalTx)
+            verifyConstraintsValidity(internalTx, contractAttachmentsByContract, transactionClassLoader)
+            verifyConstraints(internalTx, contractAttachmentsByContract)
+            verifyContracts(internalTx)
+        }
     }
 
     /**
@@ -133,7 +178,7 @@ data class LedgerTransaction @JvmOverloads constructor(
      *
      * A warning will be written to the log if any mismatch is detected.
      */
-    private fun validateStatesAgainstContract() = allStates.forEach(::validateStateAgainstContract)
+    private fun validateStatesAgainstContract(internalTx: LedgerTransaction) = internalTx.allStates.forEach { validateStateAgainstContract(it) }
 
     private fun validateStateAgainstContract(state: TransactionState<ContractState>) {
         state.data.requiredContractClassName?.let { requiredContractClassName ->
@@ -150,25 +195,25 @@ data class LedgerTransaction @JvmOverloads constructor(
      * * Constraints should be one of the valid supported ones.
      * * Constraints should propagate correctly if not marked otherwise.
      */
-    private fun verifyConstraintsValidity(contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) {
+    private fun verifyConstraintsValidity(internalTx: LedgerTransaction, contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>, transactionClassLoader: ClassLoader) {
         // First check that the constraints are valid.
-        for (state in allStates) {
+        for (state in internalTx.allStates) {
             checkConstraintValidity(state)
         }
 
         // Group the inputs and outputs by contract, and for each contract verify the constraints propagation logic.
         // This is not required for reference states as there is nothing to propagate.
-        val inputContractGroups = inputs.groupBy { it.state.contract }
+        val inputContractGroups = internalTx.inputs.groupBy { it.state.contract }
-        val outputContractGroups = outputs.groupBy { it.contract }
+        val outputContractGroups = internalTx.outputs.groupBy { it.contract }
 
         for (contractClassName in (inputContractGroups.keys + outputContractGroups.keys)) {
-            if (contractClassName.contractHasAutomaticConstraintPropagation()) {
+            if (contractClassName.contractHasAutomaticConstraintPropagation(transactionClassLoader)) {
                 // Verify that the constraints of output states have at least the same level of restriction as the constraints of the corresponding input states.
                 val inputConstraints = inputContractGroups[contractClassName]?.map { it.state.constraint }?.toSet()
                 val outputConstraints = outputContractGroups[contractClassName]?.map { it.constraint }?.toSet()
                 outputConstraints?.forEach { outputConstraint ->
                     inputConstraints?.forEach { inputConstraint ->
-                        if (!(outputConstraint.canBeTransitionedFrom(inputConstraint, contractAttachmentsByContract[contractClassName]!! ))) {
+                        if (!(outputConstraint.canBeTransitionedFrom(inputConstraint, contractAttachmentsByContract[contractClassName]!!))) {
                             throw TransactionVerificationException.ConstraintPropagationRejection(id, contractClassName, inputConstraint, outputConstraint)
                         }
                     }
@@ -186,8 +231,8 @@ data class LedgerTransaction @JvmOverloads constructor(
      *
      * @throws TransactionVerificationException if the constraints fail to verify
      */
-    private fun verifyConstraints(contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) {
+    private fun verifyConstraints(internalTx: LedgerTransaction, contractAttachmentsByContract: Map<ContractClassName, ContractAttachment>) {
-        for (state in allStates) {
+        for (state in internalTx.allStates) {
             val contractAttachment = contractAttachmentsByContract[state.contract]
                     ?: throw TransactionVerificationException.MissingAttachmentRejection(id, state.contract)
 
@@ -226,38 +271,64 @@ data class LedgerTransaction @JvmOverloads constructor(
         return result
     }
 
+    private fun contractClassFor(className: ContractClassName, classLoader: ClassLoader): Class<out Contract> = try {
+        classLoader.loadClass(className).asSubclass(Contract::class.java)
+    } catch (e: Exception) {
+        throw TransactionVerificationException.ContractCreationError(id, className, e)
+    }
+
+    private fun createInternalLedgerTransaction(): LedgerTransaction {
+        return if (resolvedInputBytes != null && resolvedReferenceBytes != null && componentGroups != null) {
+
+            // Deserialize all relevant classes in the transaction classloader.
+            val resolvedDeserializedInputs = resolvedInputBytes.map { StateAndRef(it.serializedState.deserialize(), it.ref) }
+            val resolvedDeserializedReferences = resolvedReferenceBytes.map { StateAndRef(it.serializedState.deserialize(), it.ref) }
+            val deserializedOutputs = deserialiseComponentGroup(componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true)
+            val deserializedCommands = deserialiseCommands(this.componentGroups, forceDeserialize = true)
+            val authenticatedArgs = deserializedCommands.map { cmd ->
+                val parties = commands.find { it.value.javaClass.name == cmd.value.javaClass.name }!!.signingParties
+                CommandWithParties(cmd.signers, parties, cmd.value)
+            }
+
+            val ledgerTransactionToVerify = this.copy(
+                    inputs = resolvedDeserializedInputs,
+                    outputs = deserializedOutputs,
+                    commands = authenticatedArgs,
+                    references = resolvedDeserializedReferences)
+
+            ledgerTransactionToVerify
+        } else {
+            // This branch is only present for backwards compatibility.
+            // TODO - it should be removed once the constructor of LedgerTransaction is no longer public api.
+            logger.warn("The LedgerTransaction should not be instantiated directly from client code. Please use WireTransaction.toLedgerTransaction. The result of the verify method might not be accurate.")
+            this
+        }
+    }
+
     /**
      * Check the transaction is contract-valid by running the verify() for each input and output state contract.
      * If any contract fails to verify, the whole transaction is considered to be invalid.
      */
-    private fun verifyContracts() = inputAndOutputStates.forEach { ts ->
+    private fun verifyContracts(internalTx: LedgerTransaction) {
-        val contractClass = getContractClass(ts)
+        val contractClasses = (internalTx.inputs.map { it.state } + internalTx.outputs).toSet()
-        val contract = createContractInstance(contractClass)
+                .map { it.contract to contractClassFor(it.contract, it.data.javaClass.classLoader) }
 
-        try {
+        val contractInstances = contractClasses.map { (contractClassName, contractClass) ->
-            contract.verify(this)
-        } catch (e: Exception) {
-            throw TransactionVerificationException.ContractRejection(id, contract, e)
-        }
-    }
-
-    // Obtain the contract class from the class name, wrapping any exception as a [ContractCreationError]
-    private fun getContractClass(ts: TransactionState<ContractState>): Class<out Contract> =
-            try {
-                (ts.data::class.java.classLoader ?: this::class.java.classLoader)
-                        .loadClass(ts.contract)
-                        .asSubclass(Contract::class.java)
-            } catch (e: Exception) {
-                throw TransactionVerificationException.ContractCreationError(id, ts.contract, e)
-            }
-
-    // Obtain an instance of the contract class, wrapping any exception as a [ContractCreationError]
-    private fun createContractInstance(contractClass: Class<out Contract>): Contract =
             try {
                 contractClass.newInstance()
             } catch (e: Exception) {
-                throw TransactionVerificationException.ContractCreationError(id, contractClass.name, e)
+                throw TransactionVerificationException.ContractCreationError(id, contractClassName, e)
             }
+        }
+
+        contractInstances.forEach { contract ->
+            try {
+                contract.verify(internalTx)
+            } catch (e: Exception) {
+                throw TransactionVerificationException.ContractRejection(id, contract, e)
+            }
+        }
+    }
 
     /**
      * Make sure the notary has stayed the same. As we can't tell how inputs and outputs connect, if there
@@ -286,7 +357,8 @@ data class LedgerTransaction @JvmOverloads constructor(
         // b) the number of outputs can contain the encumbrance
         // c) the bi-directionality (full cycle) property is satisfied
         // d) encumbered output states are assigned to the same notary.
-        val statesAndEncumbrance = outputs.withIndex().filter { it.value.encumbrance != null }.map { Pair(it.index, it.value.encumbrance!!) }
+        val statesAndEncumbrance = outputs.withIndex().filter { it.value.encumbrance != null }
+                .map { Pair(it.index, it.value.encumbrance!!) }
         if (!statesAndEncumbrance.isEmpty()) {
             checkBidirectionalOutputEncumbrances(statesAndEncumbrance)
             checkNotariesOutputEncumbrance(statesAndEncumbrance)

core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt

@@ -6,14 +6,14 @@ import net.corda.core.contracts.*
 import net.corda.core.contracts.ComponentGroupEnum.*
 import net.corda.core.crypto.*
 import net.corda.core.identity.Party
-import net.corda.core.internal.LazyMappedList
+import net.corda.core.internal.deserialiseCommands
-import net.corda.core.internal.uncheckedCast
+import net.corda.core.internal.deserialiseComponentGroup
-import net.corda.core.serialization.*
+import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.SerializedBytes
+import net.corda.core.serialization.deserialize
 import net.corda.core.utilities.OpaqueBytes
-import net.corda.core.utilities.lazyMapped
 import java.security.PublicKey
 import java.util.function.Predicate
-import kotlin.reflect.KClass
 
 /**
  * Implemented by [WireTransaction] and [FilteredTransaction]. A TraversableTransaction allows you to iterate
@@ -23,27 +23,27 @@ import kotlin.reflect.KClass
  */
 abstract class TraversableTransaction(open val componentGroups: List<ComponentGroup>) : CoreTransaction() {
     /** Hashes of the ZIP/JAR files that are needed to interpret the contents of this wire transaction. */
-    val attachments: List<SecureHash> = deserialiseComponentGroup(SecureHash::class, ATTACHMENTS_GROUP)
+    val attachments: List<SecureHash> = deserialiseComponentGroup(componentGroups, SecureHash::class, ATTACHMENTS_GROUP)
 
     /** Pointers to the input states on the ledger, identified by (tx identity hash, output index). */
-    override val inputs: List<StateRef> = deserialiseComponentGroup(StateRef::class, INPUTS_GROUP)
+    override val inputs: List<StateRef> = deserialiseComponentGroup(componentGroups, StateRef::class, INPUTS_GROUP)
 
     /** Pointers to reference states, identified by (tx identity hash, output index). */
-    override val references: List<StateRef> = deserialiseComponentGroup(StateRef::class, REFERENCES_GROUP)
+    override val references: List<StateRef> = deserialiseComponentGroup(componentGroups, StateRef::class, REFERENCES_GROUP)
 
-    override val outputs: List<TransactionState<ContractState>> = deserialiseComponentGroup(TransactionState::class, OUTPUTS_GROUP, attachmentsContext = true)
+    override val outputs: List<TransactionState<ContractState>> = deserialiseComponentGroup(componentGroups, TransactionState::class, OUTPUTS_GROUP)
 
     /** Ordered list of ([CommandData], [PublicKey]) pairs that instruct the contracts what to do. */
-    val commands: List<Command<*>> = deserialiseCommands()
+    val commands: List<Command<*>> = deserialiseCommands(componentGroups)
 
     override val notary: Party? = let {
-        val notaries: List<Party> = deserialiseComponentGroup(Party::class, NOTARY_GROUP)
+        val notaries: List<Party> = deserialiseComponentGroup(componentGroups, Party::class, NOTARY_GROUP)
         check(notaries.size <= 1) { "Invalid Transaction. More than 1 notary party detected." }
         notaries.firstOrNull()
     }
 
     val timeWindow: TimeWindow? = let {
-        val timeWindows: List<TimeWindow> = deserialiseComponentGroup(TimeWindow::class, TIMEWINDOW_GROUP)
+        val timeWindows: List<TimeWindow> = deserialiseComponentGroup(componentGroups, TimeWindow::class, TIMEWINDOW_GROUP)
         check(timeWindows.size <= 1) { "Invalid Transaction. More than 1 time-window detected." }
         timeWindows.firstOrNull()
     }
@@ -66,65 +66,6 @@ abstract class TraversableTransaction(open val componentGroups: List<ComponentGr
             timeWindow?.let { result += listOf(it) }
             return result
         }
-
-    // Helper function to return a meaningful exception if deserialisation of a component fails.
-    private fun <T : Any> deserialiseComponentGroup(clazz: KClass<T>,
-                                                    groupEnum: ComponentGroupEnum,
-                                                    attachmentsContext: Boolean = false): List<T> {
-        val group = componentGroups.firstOrNull { it.groupIndex == groupEnum.ordinal }
-
-        if (group == null || group.components.isEmpty()) {
-            return emptyList()
-        }
-
-        // If the componentGroup is a [LazyMappedList] it means that the original deserialized version is already available.
-        val components = group.components
-        if (components is LazyMappedList<*, OpaqueBytes>) {
-            return components.originalList as List<T>
-        }
-
-        val factory = SerializationFactory.defaultFactory
-        val context = factory.defaultContext.let { if (attachmentsContext) it.withAttachmentsClassLoader(attachments) else it }
-
-        return components.lazyMapped { component, internalIndex ->
-            try {
-                factory.deserialize(component, clazz.java , context)
-            } catch (e: MissingAttachmentsException) {
-                throw e
-            } catch (e: Exception) {
-                throw Exception("Malformed transaction, $groupEnum at index $internalIndex cannot be deserialised", e)
-            }
-        }
-    }
-
-    // Method to deserialise Commands from its two groups:
-    // COMMANDS_GROUP which contains the CommandData part
-    // and SIGNERS_GROUP which contains the Signers part.
-    private fun deserialiseCommands(): List<Command<*>> {
-        // TODO: we could avoid deserialising unrelated signers.
-        //      However, current approach ensures the transaction is not malformed
-        //      and it will throw if any of the signers objects is not List of public keys).
-        val signersList: List<List<PublicKey>> = uncheckedCast(deserialiseComponentGroup(List::class, SIGNERS_GROUP))
-        val commandDataList: List<CommandData> = deserialiseComponentGroup(CommandData::class, COMMANDS_GROUP, attachmentsContext = true)
-        val group = componentGroups.firstOrNull { it.groupIndex == COMMANDS_GROUP.ordinal }
-        return if (group is FilteredComponentGroup) {
-            check(commandDataList.size <= signersList.size) {
-                "Invalid Transaction. Less Signers (${signersList.size}) than CommandData (${commandDataList.size}) objects"
-            }
-            val componentHashes = group.components.mapIndexed { index, component -> componentHash(group.nonces[index], component) }
-            val leafIndices = componentHashes.map { group.partialMerkleTree.leafIndex(it) }
-            if (leafIndices.isNotEmpty())
-                check(leafIndices.max()!! < signersList.size) { "Invalid Transaction. A command with no corresponding signer detected" }
-            commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[leafIndices[index]]) }
-        } else {
-            // It is a WireTransaction
-            // or a FilteredTransaction with no Commands (in which case group is null).
-            check(commandDataList.size == signersList.size) {
-                "Invalid Transaction. Sizes of CommandData (${commandDataList.size}) and Signers (${signersList.size}) do not match"
-            }
-            commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[index]) }
-        }
-    }
 }
 
 /**

core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt

@@ -1,5 +1,6 @@
 package net.corda.core.transactions
 
+import net.corda.core.CordaInternal
 import net.corda.core.DeleteForDJVM
 import net.corda.core.KeepForDJVM
 import net.corda.core.contracts.*
@@ -10,6 +11,7 @@ import net.corda.core.identity.Party
 import net.corda.core.node.ServiceHub
 import net.corda.core.node.ServicesForResolution
 import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.SerializedBytes
 import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
 import net.corda.core.transactions.NotaryChangeWireTransaction.Component.*
@@ -75,6 +77,20 @@ data class NotaryChangeWireTransaction(
     @DeleteForDJVM
     fun resolve(services: ServiceHub, sigs: List<TransactionSignature>) = resolve(services as ServicesForResolution, sigs)
 
+    /**
+     * This should return a serialized virtual output state, that will be used to verify spending transactions.
+     * The binary output should not depend on the classpath of the node that is verifying the transaction.
+     *
+     * Ideally the serialization engine would support partial deserialization so that only the Notary ( and the encumbrance can be replaced from the binary input state)
+     *
+     *
+     * TODO - currently this uses the main classloader.
+     */
+    @CordaInternal
+    internal fun resolveOutputComponent(services: ServicesForResolution, stateRef: StateRef): SerializedBytes<TransactionState<ContractState>> {
+        return services.loadState(stateRef).serialize()
+    }
+
     enum class Component {
         INPUTS, NOTARY, NEW_NOTARY
     }

core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt

@@ -270,7 +270,7 @@ open class TransactionBuilder @JvmOverloads constructor(
         }
 
         // The final step is to resolve AutomaticPlaceholderConstraint.
-        val automaticConstraintPropagation = contractClassName.contractHasAutomaticConstraintPropagation(serializationContext?.deserializationClassLoader)
+        val automaticConstraintPropagation = contractClassName.contractHasAutomaticConstraintPropagation(inputsAndOutputs.first().data::class.java.classLoader)
 
         // When automaticConstraintPropagation is disabled for a contract, output states must an explicit Constraint.
         require(automaticConstraintPropagation) { "Contract $contractClassName was marked with @NoConstraintPropagation, which means the constraint of the output states has to be set explicitly." }

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -7,11 +7,15 @@ import net.corda.core.contracts.*
 import net.corda.core.contracts.ComponentGroupEnum.*
 import net.corda.core.crypto.*
 import net.corda.core.identity.Party
+import net.corda.core.internal.SerializedStateAndRef
 import net.corda.core.internal.Emoji
 import net.corda.core.node.NetworkParameters
+import net.corda.core.node.ServiceHub
 import net.corda.core.node.ServicesForResolution
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.SerializedBytes
+import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.lazyMapped
@@ -99,7 +103,7 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
         return toLedgerTransactionInternal(
                 resolveIdentity = { services.identityService.partyFromKey(it) },
                 resolveAttachment = { services.attachments.openAttachment(it) },
-                resolveStateRef = { services.loadState(it) },
+                resolveStateRefComponent = { resolveStateRefBinaryComponent(it, services) },
                 networkParameters = services.networkParameters
         )
     }
@@ -119,13 +123,14 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
             resolveStateRef: (StateRef) -> TransactionState<*>?,
             @Suppress("UNUSED_PARAMETER") resolveContractAttachment: (TransactionState<ContractState>) -> AttachmentId?
     ): LedgerTransaction {
-        return toLedgerTransactionInternal(resolveIdentity, resolveAttachment, resolveStateRef, null)
+        // This reverts to serializing the resolved transaction state.
+        return toLedgerTransactionInternal(resolveIdentity, resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, null)
     }
 
     private fun toLedgerTransactionInternal(
             resolveIdentity: (PublicKey) -> Party?,
             resolveAttachment: (SecureHash) -> Attachment?,
-            resolveStateRef: (StateRef) -> TransactionState<*>?,
+            resolveStateRefComponent: (StateRef) -> SerializedBytes<TransactionState<ContractState>>?,
             networkParameters: NetworkParameters?
     ): LedgerTransaction {
         // Look up public keys to authenticated identities.
@@ -133,20 +138,38 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
             val parties = cmd.signers.mapNotNull { pk -> resolveIdentity(pk) }
             CommandWithParties(cmd.signers, parties, cmd.value)
         }
-        val resolvedInputs = inputs.lazyMapped { ref, _ ->
+
-            resolveStateRef(ref)?.let { StateAndRef(it, ref) } ?: throw TransactionResolutionException(ref.txhash)
+        val resolvedInputBytes = inputs.map { ref ->
+            SerializedStateAndRef(resolveStateRefComponent(ref)
+                    ?: throw TransactionResolutionException(ref.txhash), ref)
         }
-        val resolvedReferences = references.lazyMapped { ref, _ ->
+        val resolvedInputs = resolvedInputBytes.lazyMapped { (serialized, ref), _ ->
-            resolveStateRef(ref)?.let { StateAndRef(it, ref) } ?: throw TransactionResolutionException(ref.txhash)
+            StateAndRef(serialized.deserialize(), ref)
         }
+
+        val resolvedReferenceBytes = references.map { ref ->
+            SerializedStateAndRef(resolveStateRefComponent(ref)
+                    ?: throw TransactionResolutionException(ref.txhash), ref)
+        }
+        val resolvedReferences = resolvedReferenceBytes.lazyMapped { (serialized, ref), _ ->
+            StateAndRef(serialized.deserialize(), ref)
+        }
+
         val attachments = attachments.lazyMapped { att, _ ->
             resolveAttachment(att) ?: throw AttachmentResolutionException(att)
         }
-        val ltx = LedgerTransaction(resolvedInputs, outputs, authenticatedArgs, attachments, id, notary, timeWindow, privacySalt, networkParameters, resolvedReferences)
+
-        checkTransactionSize(ltx, networkParameters?.maxTransactionSize ?: 10485760)
+        val ltx = LedgerTransaction.makeLedgerTransaction(resolvedInputs, outputs, authenticatedArgs, attachments, id, notary, timeWindow, privacySalt, networkParameters, resolvedReferences, componentGroups, resolvedInputBytes, resolvedReferenceBytes)
+
+        checkTransactionSize(ltx, networkParameters?.maxTransactionSize ?: DEFAULT_MAX_TX_SIZE)
+
         return ltx
     }
 
+    /**
+     * Deterministic function that checks if the transaction is below the maximum allowed size.
+     * It uses the binary representation of transactions.
+     */
     private fun checkTransactionSize(ltx: LedgerTransaction, maxTransactionSize: Int) {
         var remainingTransactionSize = maxTransactionSize
 
@@ -164,9 +187,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
         // it's likely that the same underlying Attachment CorDapp will occur more than once so we dedup on the attachment id.
         ltx.attachments.distinctBy { it.id }.forEach { minus(it.size) }
 
-        // TODO - these can be optimized by creating a LazyStateAndRef class, that just stores (a pointer) the serialized output componentGroup from the previous transaction.
+        minus(ltx.resolvedInputBytes!!.sumBy { it.serializedState.size })
-        minus(ltx.references.serialize().size)
+        minus(ltx.resolvedReferenceBytes!!.sumBy { it.serializedState.size })
-        minus(ltx.inputs.serialize().size)
 
         // For Commands and outputs we can use the component groups as they are already serialized.
         minus(componentGroupSize(COMMANDS_GROUP))
@@ -253,6 +275,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
     }
 
     companion object {
+        private const val DEFAULT_MAX_TX_SIZE = 10485760
+
         /**
          * Creating list of [ComponentGroup] used in one of the constructors of [WireTransaction] required
          * for backwards compatibility purposes.
@@ -281,6 +305,28 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
             if (commands.isNotEmpty()) componentGroupMap.add(ComponentGroup(SIGNERS_GROUP.ordinal, commands.map { it.signers }.lazyMapped(serialize)))
             return componentGroupMap
         }
+
+        /**
+         * This is the main logic that knows how to retrieve the binary representation of [StateRef]s.
+         *
+         * For [ContractUpgradeWireTransaction] or [NotaryChangeWireTransaction] it knows how to recreate the output state in the correct classloader independent of the node's classpath.
+         */
+        @CordaInternal
+        fun resolveStateRefBinaryComponent(stateRef: StateRef, services: ServicesForResolution): SerializedBytes<TransactionState<ContractState>>? {
+            return if (services is ServiceHub) {
+                val coreTransaction = services.validatedTransactions.getTransaction(stateRef.txhash)?.coreTransaction
+                        ?: throw TransactionResolutionException(stateRef.txhash)
+                when (coreTransaction) {
+                    is WireTransaction -> coreTransaction.componentGroups.firstOrNull { it.groupIndex == ComponentGroupEnum.OUTPUTS_GROUP.ordinal }?.components?.get(stateRef.index) as SerializedBytes<TransactionState<ContractState>>?
+                    is ContractUpgradeWireTransaction -> coreTransaction.resolveOutputComponent(services, stateRef)
+                    is NotaryChangeWireTransaction -> coreTransaction.resolveOutputComponent(services, stateRef)
+                    else -> throw UnsupportedOperationException("Attempting to resolve input ${stateRef.index} of a ${coreTransaction.javaClass} transaction. This is not supported.")
+                }
+            } else {
+                // For backwards compatibility revert to using the node classloader.
+                services.loadState(stateRef).serialize()
+            }
+        }
     }
 
     @DeleteForDJVM

core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt

@@ -6,6 +6,7 @@ import net.corda.core.DeleteForDJVM
 import net.corda.core.KeepForDJVM
 import net.corda.core.internal.LazyMappedList
 import net.corda.core.internal.concurrent.get
+import net.corda.core.internal.createSimpleCache
 import net.corda.core.internal.uncheckedCast
 import net.corda.core.serialization.CordaSerializable
 import org.slf4j.Logger
@@ -149,9 +150,7 @@ fun <V> Future<V>.getOrThrow(timeout: Duration? = null): V = try {
 fun <T, U> List<T>.lazyMapped(transform: (T, Int) -> U): List<U> = LazyMappedList(this, transform)
 
 private const val MAX_SIZE = 100
-private val warnings = Collections.newSetFromMap(object : LinkedHashMap<String, Boolean>() {
+private val warnings = Collections.newSetFromMap(createSimpleCache<String, Boolean>(MAX_SIZE))
-    override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Boolean>?) = size > MAX_SIZE
-})
 
 /**
  * Utility to help log a warning message only once.

core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt

@@ -14,18 +14,13 @@ import net.corda.core.internal.FetchAttachmentsFlow
 import net.corda.core.internal.FetchDataFlow
 import net.corda.core.internal.hash
 import net.corda.node.services.persistence.NodeAttachmentService
-import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.*
-import net.corda.testing.core.BOB_NAME
+import net.corda.testing.internal.fakeAttachment
-import net.corda.testing.core.makeUnique
-import net.corda.testing.core.singleIdentity
 import net.corda.testing.node.internal.InternalMockNetwork
 import net.corda.testing.node.internal.InternalMockNodeParameters
 import net.corda.testing.node.internal.TestStartedNode
 import org.junit.AfterClass
 import org.junit.Test
-import java.io.ByteArrayOutputStream
-import java.util.jar.JarOutputStream
-import java.util.zip.ZipEntry
 
 class AttachmentTests : WithMockNet {
     companion object {
@@ -46,7 +41,7 @@ class AttachmentTests : WithMockNet {
     @Test
     fun `download and store`() {
         // Insert an attachment into node zero's store directly.
-        val id = aliceNode.importAttachment(fakeAttachment())
+        val id = aliceNode.importAttachment(fakeAttachment("file1.txt", "Some useful content"))
 
         // Get node one to run a flow to fetch it and insert it.
         assert.that(
@@ -87,7 +82,7 @@ class AttachmentTests : WithMockNet {
         val badAlice = badAliceNode.info.singleIdentity()
 
         // Insert an attachment into node zero's store directly.
-        val attachment = fakeAttachment()
+        val attachment = fakeAttachment("file1.txt", "Some useful content")
         val id = badAliceNode.importAttachment(attachment)
 
         // Corrupt its store.
@@ -134,18 +129,6 @@ class AttachmentTests : WithMockNet {
                 }
             }).apply { registerInitiatedFlow(FetchAttachmentsResponse::class.java) }
 
-    private fun fakeAttachment(): ByteArray =
-        ByteArrayOutputStream().use { baos ->
-            JarOutputStream(baos).use { jos ->
-                jos.putNextEntry(ZipEntry("file1.txt"))
-                jos.writer().apply {
-                    append("Some useful content")
-                    flush()
-                }
-                jos.closeEntry()
-            }
-            baos.toByteArray()
-        }
     //endregion
 
     //region Operations

core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderSerializationTests.kt

@@ -0,0 +1,98 @@
+package net.corda.core.transactions
+
+import net.corda.core.contracts.Contract
+import net.corda.core.crypto.SecureHash
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.internal.declaredField
+import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
+import net.corda.core.serialization.serialize
+import net.corda.core.utilities.ByteSequence
+import net.corda.core.utilities.OpaqueBytes
+import net.corda.nodeapi.DummyContractBackdoor
+import net.corda.testing.core.DUMMY_NOTARY_NAME
+import net.corda.testing.core.SerializationEnvironmentRule
+import net.corda.testing.core.TestIdentity
+import net.corda.testing.internal.fakeAttachment
+import net.corda.testing.services.MockAttachmentStorage
+import org.apache.commons.io.IOUtils
+import org.junit.Assert.assertEquals
+import org.junit.Rule
+import org.junit.Test
+import java.io.NotSerializableException
+import java.net.URL
+import kotlin.test.assertFailsWith
+
+class AttachmentsClassLoaderSerializationTests {
+
+    companion object {
+        val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderSerializationTests::class.java.getResource("isolated.jar")
+        private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract"
+    }
+
+    @Rule
+    @JvmField
+    val testSerialization = SerializationEnvironmentRule()
+
+    val storage = MockAttachmentStorage()
+
+    @Test
+    fun `Can serialize and deserialize with an attachment classloader`() {
+
+        val DUMMY_NOTARY = TestIdentity(DUMMY_NOTARY_NAME, 20).party
+        val MEGA_CORP = TestIdentity(CordaX500Name("MegaCorp", "London", "GB")).party
+
+        val isolatedId = storage.importAttachment(ISOLATED_CONTRACTS_JAR_PATH.openStream(), "app", "isolated.jar")
+        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar")
+        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream(), "app", "file2.jar")
+
+        val serialisedState = AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! }) { classLoader ->
+            val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader)
+            val contract = contractClass.newInstance() as Contract
+            assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)
+
+            val txt = IOUtils.toString(classLoader.getResourceAsStream("file1.txt"), Charsets.UTF_8.name())
+            assertEquals("some data", txt)
+
+            val state = (contract as DummyContractBackdoor).generateInitial(MEGA_CORP.ref(1), 1, DUMMY_NOTARY).outputStates().first()
+            val serialisedState = state.serialize()
+
+            val state1 = serialisedState.deserialize()
+            assertEquals(state, state1)
+            serialisedState
+        }
+
+        assertFailsWith<NotSerializableException> {
+            serialisedState.deserialize()
+        }
+    }
+
+    // These tests are not Attachment specific. Should they be removed?
+    @Test
+    fun `test serialization of SecureHash`() {
+        val secureHash = SecureHash.randomSHA256()
+        val bytes = secureHash.serialize()
+        val copiedSecuredHash = bytes.deserialize()
+
+        assertEquals(secureHash, copiedSecuredHash)
+    }
+
+    @Test
+    fun `test serialization of OpaqueBytes`() {
+        val opaqueBytes = OpaqueBytes("0123456789".toByteArray())
+        val bytes = opaqueBytes.serialize()
+        val copiedOpaqueBytes = bytes.deserialize()
+
+        assertEquals(opaqueBytes, copiedOpaqueBytes)
+    }
+
+    @Test
+    fun `test serialization of sub-sequence OpaqueBytes`() {
+        val bytesSequence = ByteSequence.of("0123456789".toByteArray(), 3, 2)
+        val bytes = bytesSequence.serialize()
+        val copiedBytesSequence = bytes.deserialize()
+
+        assertEquals(bytesSequence, copiedBytesSequence)
+    }
+}
+

core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderTests.kt

@@ -0,0 +1,102 @@
+package net.corda.core.transactions
+
+import net.corda.core.contracts.Attachment
+import net.corda.core.contracts.Contract
+import net.corda.core.internal.declaredField
+import net.corda.core.serialization.internal.AttachmentsClassLoader
+import net.corda.testing.internal.fakeAttachment
+import net.corda.testing.services.MockAttachmentStorage
+import org.apache.commons.io.IOUtils
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import java.io.ByteArrayOutputStream
+import java.net.URL
+import kotlin.test.assertFailsWith
+
+class AttachmentsClassLoaderTests {
+
+    companion object {
+        val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderTests::class.java.getResource("isolated.jar")
+        private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract"
+
+        private fun readAttachment(attachment: Attachment, filepath: String): ByteArray {
+            ByteArrayOutputStream().use {
+                attachment.extractFile(filepath, it)
+                return it.toByteArray()
+            }
+        }
+    }
+
+    val storage = MockAttachmentStorage()
+
+    @Test
+    fun `Loading AnotherDummyContract without using the AttachmentsClassLoader fails`() {
+        assertFailsWith<ClassNotFoundException> {
+            Class.forName(ISOLATED_CONTRACT_CLASS_NAME)
+        }
+    }
+
+    @Test
+    fun `Dynamically load AnotherDummyContract from isolated contracts jar using the AttachmentsClassLoader`() {
+        val isolatedId = storage.importAttachment(ISOLATED_CONTRACTS_JAR_PATH.openStream(), "app", "isolated.jar")
+
+        val classloader = AttachmentsClassLoader(listOf(storage.openAttachment(isolatedId)!!))
+        val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classloader)
+        val contract = contractClass.newInstance() as Contract
+        assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)
+    }
+
+    @Test
+    fun `Load text resources from AttachmentsClassLoader`() {
+        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar")
+        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream(), "app", "file2.jar")
+
+        val cl = AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! })
+        val txt = IOUtils.toString(cl.getResourceAsStream("file1.txt"), Charsets.UTF_8.name())
+        assertEquals("some data", txt)
+
+        val txt1 = IOUtils.toString(cl.getResourceAsStream("file2.txt"), Charsets.UTF_8.name())
+        assertEquals("some other data", txt1)
+    }
+
+    @Test
+    fun `Test overlapping file exception`() {
+        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar")
+        val att2 = storage.importAttachment(fakeAttachment("file1.txt", "some other data").inputStream(), "app", "file2.jar")
+
+        assertFailsWith(AttachmentsClassLoader.Companion.OverlappingAttachments::class) {
+            AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! })
+        }
+    }
+
+    @Test
+    fun `No overlapping exception thrown on certain META-INF files`() {
+        listOf("meta-inf/manifest.mf", "meta-inf/license", "meta-inf/test.dsa", "meta-inf/test.sf").forEach { path ->
+            val att1 = storage.importAttachment(fakeAttachment(path, "some data").inputStream(), "app", "file1.jar")
+            val att2 = storage.importAttachment(fakeAttachment(path, "some other data").inputStream(), "app", "file2.jar")
+
+            AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! })
+        }
+    }
+
+    @Test
+    fun `Check platform independent path handling in attachment jars`() {
+        val storage = MockAttachmentStorage()
+
+        val att1 = storage.importAttachment(fakeAttachment("/folder1/foldera/file1.txt", "some data").inputStream(), "app", "file1.jar")
+        val att2 = storage.importAttachment(fakeAttachment("\\folder1\\folderb\\file2.txt", "some other data").inputStream(), "app", "file2.jar")
+
+        val data1a = readAttachment(storage.openAttachment(att1)!!, "/folder1/foldera/file1.txt")
+        assertArrayEquals("some data".toByteArray(), data1a)
+
+        val data1b = readAttachment(storage.openAttachment(att1)!!, "\\folder1\\foldera\\file1.txt")
+        assertArrayEquals("some data".toByteArray(), data1b)
+
+        val data2a = readAttachment(storage.openAttachment(att2)!!, "\\folder1\\folderb\\file2.txt")
+        assertArrayEquals("some other data".toByteArray(), data2a)
+
+        val data2b = readAttachment(storage.openAttachment(att2)!!, "/folder1/folderb/file2.txt")
+        assertArrayEquals("some other data".toByteArray(), data2b)
+    }
+}

core/src/test/kotlin/net/corda/core/transactions/TransactionTests.kt

@@ -10,6 +10,7 @@ import net.corda.testing.common.internal.testNetworkParameters
 import net.corda.testing.contracts.DummyContract
 import net.corda.testing.core.*
 import net.corda.testing.internal.createWireTransaction
+import net.corda.testing.internal.fakeAttachment
 import net.corda.testing.internal.rigorousMock
 import org.junit.Rule
 import org.junit.Test
@@ -118,7 +119,8 @@ class TransactionTests {
         val commands = emptyList<CommandWithParties<CommandData>>()
         val attachments = listOf<Attachment>(ContractAttachment(rigorousMock<Attachment>().also {
             doReturn(SecureHash.zeroHash).whenever(it).id
-        }, DummyContract.PROGRAM_ID))
+            doReturn(fakeAttachment("nothing", "nada").inputStream()).whenever(it).open()
+        }, DummyContract.PROGRAM_ID, uploader = "app"))
         val id = SecureHash.randomSHA256()
         val timeWindow: TimeWindow? = null
         val privacySalt = PrivacySalt()

docs/source/changelog.rst

@@ -7,6 +7,11 @@ release, see :doc:`upgrade-notes`.
 Unreleased
 ----------
 
+* Deprecated `SerializationContext.withAttachmentsClassLoader`. This functionality has always been disabled by flags
+and there is no reason for a CorDapp developer to use it. It is just an internal implementation detail of Corda.
+
+* Deprecated the `LedgerTransaction` constructor. No client code should call it directly. LedgerTransactions can be created from WireTransactions if required.
+
 * Introduced new optional network bootstrapper command line options (--register-package-owner, --unregister-package-owner)
   to register/unregister a java package namespace with an associated owner in the network parameter packageOwnership whitelist.
 

finance/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt

@@ -25,6 +25,7 @@ import net.corda.testing.contracts.DummyState
 import net.corda.testing.core.*
 import net.corda.testing.dsl.*
 import net.corda.testing.internal.TEST_TX_TIME
+import net.corda.testing.internal.fakeAttachment
 import net.corda.testing.internal.rigorousMock
 import net.corda.testing.internal.vault.CommodityState
 import net.corda.testing.node.MockServices
@@ -565,7 +566,7 @@ class ObligationTests {
 
     @Test
     fun `commodity settlement`() {
-        val commodityContractBytes = "https://www.big-book-of-banking-law.gov/commodity-claims.html".toByteArray()
+        val commodityContractBytes = fakeAttachment("file1.txt", "https://www.big-book-of-banking-law.gov/commodity-claims.html")
         val defaultFcoj = Issued(defaultIssuer, Commodity.getInstance("FCOJ")!!)
         val oneUnitFcoj = Amount(1, defaultFcoj)
         val obligationDef = Obligation.Terms(NonEmptySet.of(commodityContractBytes.sha256() as SecureHash), NonEmptySet.of(defaultFcoj), TEST_TX_TIME)
@@ -957,7 +958,7 @@ class ObligationTests {
         assertEquals(expected, actual)
     }
 
-    private val cashContractBytes = "https://www.big-book-of-banking-law.gov/cash-claims.html".toByteArray()
+    private val cashContractBytes = fakeAttachment("file1.txt", "https://www.big-book-of-banking-law.gov/cash-claims.html")
     private val Issued<Currency>.OBLIGATION_DEF: Obligation.Terms<Currency>
         get() = Obligation.Terms(NonEmptySet.of(cashContractBytes.sha256() as SecureHash), NonEmptySet.of(this), TEST_TX_TIME)
     private val Amount<Issued<Currency>>.OBLIGATION: Obligation.State<Currency>

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt

@@ -61,7 +61,8 @@ class CordaPersistence(
         schemas: Set<MappedSchema>,
         val jdbcUrl: String,
         cacheFactory: NamedCacheFactory,
-        attributeConverters: Collection<AttributeConverter<*, *>> = emptySet()
+        attributeConverters: Collection<AttributeConverter<*, *>> = emptySet(),
+        customClassLoader: ClassLoader? = null
 ) : Closeable {
     companion object {
         private val log = contextLogger()
@@ -70,7 +71,7 @@ class CordaPersistence(
     private val defaultIsolationLevel = databaseConfig.transactionIsolationLevel
     val hibernateConfig: HibernateConfiguration by lazy {
         transaction {
-            HibernateConfiguration(schemas, databaseConfig, attributeConverters, jdbcUrl, cacheFactory)
+            HibernateConfiguration(schemas, databaseConfig, attributeConverters, jdbcUrl, cacheFactory, customClassLoader)
         }
     }
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt

@@ -32,7 +32,7 @@ class HibernateConfiguration(
         private val attributeConverters: Collection<AttributeConverter<*, *>>,
         private val jdbcUrl: String,
         cacheFactory: NamedCacheFactory,
-        val cordappClassLoader: ClassLoader? = null
+        val customClassLoader: ClassLoader? = null
 ) {
     companion object {
         private val logger = contextLogger()
@@ -86,7 +86,7 @@ class HibernateConfiguration(
             schema.mappedTypes.forEach { config.addAnnotatedClass(it) }
         }
 
-        val sessionFactory = buildSessionFactory(config, metadataSources, cordappClassLoader)
+        val sessionFactory = buildSessionFactory(config, metadataSources, customClassLoader)
         logger.info("Created session factory for schemas: $schemas")
 
         // export Hibernate JMX statistics
@@ -112,13 +112,13 @@ class HibernateConfiguration(
         }
     }
 
-    private fun buildSessionFactory(config: Configuration, metadataSources: MetadataSources, cordappClassLoader: ClassLoader?): SessionFactory {
+    private fun buildSessionFactory(config: Configuration, metadataSources: MetadataSources, customClassLoader: ClassLoader?): SessionFactory {
         config.standardServiceRegistryBuilder.applySettings(config.properties)
 
-        if (cordappClassLoader != null) {
+        if (customClassLoader != null) {
             config.standardServiceRegistryBuilder.addService(
                     ClassLoaderService::class.java,
-                    ClassLoaderServiceImpl(cordappClassLoader))
+                    ClassLoaderServiceImpl(customClassLoader))
         }
 
         val metadataBuilder = metadataSources.getMetadataBuilder(config.standardServiceRegistryBuilder.build())

node/src/integration-test/kotlin/net/corda/node/services/statemachine/LargeTransactionsTest.kt

@@ -69,10 +69,10 @@ class LargeTransactionsTest {
     fun checkCanSendLargeTransactions() {
         // These 4 attachments yield a transaction that's got >10mb attached, so it'd push us over the Artemis
         // max message size.
-        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 0)
+        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 0, "a")
-        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 1)
+        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 1, "b")
-        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 2)
+        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 2, "c")
-        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 3)
+        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 3, "d")
         driver(DriverParameters(
                 startNodesInProcess = true,
                 extraCordappPackagesToScan = listOf("net.corda.testing.contracts"),

node/src/main/java/CordaCaplet.java

@@ -112,8 +112,6 @@ public class CordaCaplet extends Capsule {
                 // If it fails, just return the existing class path. The main Corda jar will detect the error and fail gracefully.
                 return cp;
             }
-            // Add additional directories of JARs to the classpath (at the end), e.g., for JDBC drivers.
-            augmentClasspath((List<Path>) cp, cordappsDir);
             try {
                 List<String> jarDirs = nodeConfig.getStringList("jarDirs");
                 log(LOG_VERBOSE, "Configured JAR directories = " + jarDirs);

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -155,7 +155,8 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             identityService::wellKnownPartyFromAnonymous,
             schemaService,
             configuration.dataSourceProperties,
-            cacheFactory)
+            cacheFactory,
+            this.cordappLoader.appClassLoader)
 
     init {
         // TODO Break cyclic dependency
@@ -748,7 +749,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
     protected open fun startDatabase() {
         val props = configuration.dataSourceProperties
         if (props.isEmpty) throw DatabaseConfigurationException("There must be a database configured.")
-        database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry)
+        database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry, this.cordappLoader.appClassLoader)
         // Now log the vendor string as this will also cause a connection to be tested eagerly.
         logVendorString(database, log)
     }
@@ -1061,7 +1062,8 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig,
                            wellKnownPartyFromAnonymous: (AbstractParty) -> Party?,
                            schemaService: SchemaService,
                            hikariProperties: Properties,
-                           cacheFactory: NamedCacheFactory): CordaPersistence {
+                           cacheFactory: NamedCacheFactory,
+                           customClassLoader: ClassLoader?): CordaPersistence {
     // Register the AbstractPartyDescriptor so Hibernate doesn't warn when encountering AbstractParty. Unfortunately
     // Hibernate warns about not being able to find a descriptor if we don't provide one, but won't use it by default
     // so we end up providing both descriptor and converter. We should re-examine this in later versions to see if
@@ -1069,13 +1071,13 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig,
     JavaTypeDescriptorRegistry.INSTANCE.addDescriptor(AbstractPartyDescriptor(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous))
     val attributeConverters = listOf(PublicKeyToTextConverter(), AbstractPartyToX500NameAsStringConverter(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous))
     val jdbcUrl = hikariProperties.getProperty("dataSource.url", "")
-    return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters)
+    return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters, customClassLoader)
 }
 
-fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set<MappedSchema>, metricRegistry: MetricRegistry? = null) {
+fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set<MappedSchema>, metricRegistry: MetricRegistry? = null, classloader: ClassLoader = Thread.currentThread().contextClassLoader) {
     try {
         val dataSource = DataSourceFactory.createDataSource(hikariProperties, metricRegistry = metricRegistry)
-        val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig)
+        val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig, classloader)
         schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L })
         start(dataSource)
     } catch (ex: Exception) {

node/src/main/kotlin/net/corda/node/serialization/kryo/CordaClassResolver.kt

@@ -11,7 +11,7 @@ import net.corda.core.internal.writer
 import net.corda.core.serialization.internal.CheckpointSerializationContext
 import net.corda.core.serialization.ClassWhitelist
 import net.corda.core.utilities.contextLogger
-import net.corda.serialization.internal.AttachmentsClassLoader
+import net.corda.core.serialization.internal.AttachmentsClassLoader
 import net.corda.serialization.internal.MutableClassWhitelist
 import net.corda.serialization.internal.TransientClassWhiteList
 import net.corda.serialization.internal.amqp.hasCordaSerializable

node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt

@@ -229,7 +229,7 @@ class NodeAttachmentService(
             val attachmentImpl = AttachmentImpl(id, { attachment.content }, checkAttachmentsOnLoad).let {
                 val contracts = attachment.contractClassNames
                 if (contracts != null && contracts.isNotEmpty()) {
-                    ContractAttachment(it, contracts.first(), contracts.drop(1).toSet(), attachment.uploader, attachment.signers
+                    ContractAttachment(it, contracts.first(), contracts.drop(1).toSet(), attachment.uploader, attachment.signers?.toList()
                             ?: emptyList())
                 } else {
                     it

node/src/test/kotlin/net/corda/node/services/transactions/MaxTransactionSizeTests.kt

@@ -54,10 +54,10 @@ class MaxTransactionSizeTests {
     @Test
     fun `check transaction will fail when exceed max transaction size limit`() {
         // These 4 attachments yield a transaction that's got ~ 4mb, which will exceed the 3mb max transaction size limit
-        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0)
+        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0, "a")
-        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1)
+        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1, "b")
-        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2)
+        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2, "c")
-        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3)
+        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3, "d")
         val flow = aliceNode.transaction {
             val hash1 = aliceNode.importAttachment(bigFile1.inputStream)
             val hash2 = aliceNode.importAttachment(bigFile2.inputStream)
@@ -77,10 +77,10 @@ class MaxTransactionSizeTests {
     @Test
     fun `check transaction will be rejected by counterparty when exceed max transaction size limit`() {
         // These 4 attachments yield a transaction that's got ~ 4mb, which will exceed the 3mb max transaction size limit
-        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0)
+        val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0, "a")
-        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1)
+        val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1, "b")
-        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2)
+        val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2, "c")
-        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3)
+        val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3, "c")
         val flow = aliceNode.transaction {
             val hash1 = aliceNode.importAttachment(bigFile1.inputStream)
             val hash2 = aliceNode.importAttachment(bigFile2.inputStream)

serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderBuilder.kt

@@ -1,12 +0,0 @@
-package net.corda.serialization.internal
-
-import net.corda.core.crypto.SecureHash
-
-/**
- * Drop-in replacement for [AttachmentsClassLoaderBuilder] in the serialization module.
- * This version is not strongly-coupled to [net.corda.core.node.ServiceHub].
- */
-@Suppress("UNUSED", "UNUSED_PARAMETER")
-internal class AttachmentsClassLoaderBuilder() {
-    fun build(attachmentHashes: List<SecureHash>, properties: Map<Any, Any>, deserializationClassLoader: ClassLoader): AttachmentsClassLoader? = null
-}

serialization/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoader.kt

@@ -1,115 +0,0 @@
-package net.corda.serialization.internal
-
-import net.corda.core.KeepForDJVM
-import net.corda.core.contracts.Attachment
-import net.corda.core.contracts.ContractAttachment
-import net.corda.core.crypto.SecureHash
-import net.corda.core.internal.isUploaderTrusted
-import net.corda.core.serialization.CordaSerializable
-import java.io.ByteArrayOutputStream
-import java.io.FileNotFoundException
-import java.io.InputStream
-import java.net.URL
-import java.net.URLConnection
-import java.net.URLStreamHandler
-import java.security.CodeSigner
-import java.security.CodeSource
-import java.security.SecureClassLoader
-import java.util.*
-
-/**
- * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only
- * need to provide JAR streams, and so could be fetched from a database, local disk, etc. Constructing an
- * AttachmentsClassLoader is somewhat expensive, as every attachment is scanned to ensure that there are no overlapping
- * file paths.
- */
-@KeepForDJVM
-class AttachmentsClassLoader(attachments: List<Attachment>, parent: ClassLoader = ClassLoader.getSystemClassLoader()) : SecureClassLoader(parent) {
-    private val pathsToAttachments = HashMap<String, Attachment>()
-    private val idsToAttachments = HashMap<SecureHash, Attachment>()
-
-    @CordaSerializable
-    class OverlappingAttachments(val path: String) : Exception() {
-        override fun toString() = "Multiple attachments define a file at path $path"
-    }
-
-    init {
-        require(attachments.mapNotNull { it as? ContractAttachment }.all { isUploaderTrusted(it.uploader) }) {
-            "Attempting to load Contract Attachments downloaded from the network"
-        }
-
-        for (attachment in attachments) {
-            attachment.openAsJAR().use { jar ->
-                while (true) {
-                    val entry = jar.nextJarEntry ?: break
-
-                    // We already verified that paths are not strange/game playing when we inserted the attachment
-                    // into the storage service. So we don't need to repeat it here.
-                    //
-                    // We forbid files that differ only in case, or path separator to avoid issues for Windows/Mac developers where the
-                    // filesystem tries to be case insensitive. This may break developers who attempt to use ProGuard.
-                    //
-                    // Also convert to Unix path separators as all resource/class lookups will expect this.
-                    val path = entry.name.toLowerCase().replace('\\', '/')
-                    if (path in pathsToAttachments)
-                        throw OverlappingAttachments(path)
-                    pathsToAttachments[path] = attachment
-                }
-            }
-            idsToAttachments[attachment.id] = attachment
-        }
-    }
-
-    // Example: attachment://0b4fc1327f3bbebf1bfe98330ea402ae035936c3cb6da9bd3e26eeaa9584e74d/some/file.txt
-    //
-    // We have to provide a fake stream handler to satisfy the URL class that the scheme is known. But it's not
-    // a real scheme and we don't register it. It's just here to ensure that there aren't codepaths that could
-    // lead to data loading that we don't control right here in this class (URLs can have evil security properties!)
-    private val fakeStreamHandler = object : URLStreamHandler() {
-        override fun openConnection(u: URL?): URLConnection? {
-            throw UnsupportedOperationException()
-        }
-    }
-
-    private fun Attachment.toURL(path: String?) = URL(null, "attachment://$id/" + (path ?: ""), fakeStreamHandler)
-
-    override fun findClass(name: String): Class<*> {
-        val path = name.replace('.', '/').toLowerCase() + ".class"
-        val attachment = pathsToAttachments[path] ?: throw ClassNotFoundException(name)
-        val stream = ByteArrayOutputStream()
-        try {
-            attachment.extractFile(path, stream)
-        } catch (e: FileNotFoundException) {
-            throw ClassNotFoundException(name)
-        }
-        val bytes = stream.toByteArray()
-        // We don't attempt to propagate signatures from the JAR into the codesource, because our sandbox does not
-        // depend on external policy files to specify what it can do, so the data wouldn't be useful.
-        val codesource = CodeSource(attachment.toURL(null), emptyArray<CodeSigner>())
-        // TODO: Define an empty ProtectionDomain to start enforcing the standard Java sandbox.
-        // The standard Java sandbox is insufficient for our needs and a much more sophisticated sandboxing
-        // ClassLoader will appear here in future, but it can't hurt to use the default one too: defence in depth!
-        return defineClass(name, bytes, 0, bytes.size, codesource)
-    }
-
-    override fun findResource(name: String): URL? {
-        val attachment = pathsToAttachments[name.toLowerCase()] ?: return null
-        return attachment.toURL(name)
-    }
-
-    override fun getResourceAsStream(name: String): InputStream? {
-        val url = getResource(name) ?: return null   // May check parent classloaders, for example.
-        if (url.protocol != "attachment") return null
-        val attachment = idsToAttachments[SecureHash.parse(url.host)] ?: return null
-        val path = url.path?.substring(1) ?: return null   // Chop off the leading slash.
-        return try {
-            val stream = ByteArrayOutputStream()
-            attachment.extractFile(path, stream)
-            stream.toByteArray().inputStream()
-        } catch (e: FileNotFoundException) {
-            null
-        }
-    }
-}
-
-

serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt

@@ -16,15 +16,6 @@ data class CheckpointSerializationContextImpl @JvmOverloads constructor(
                                                               override val objectReferencesEnabled: Boolean,
                                                               override val encoding: SerializationEncoding?,
                                                               override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist) : CheckpointSerializationContext {
-    /**
-     * {@inheritDoc}
-     *
-     * Unsupported for checkpoints.
-     */
-    override fun withAttachmentsClassLoader(attachmentHashes: List<SecureHash>): CheckpointSerializationContext {
-        throw UnsupportedOperationException()
-    }
-
     override fun withProperty(property: Any, value: Any): CheckpointSerializationContext {
         return copy(properties = properties + (property to value))
     }

serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt

@@ -8,6 +8,7 @@ import net.corda.core.contracts.Attachment
 import net.corda.core.crypto.SecureHash
 import net.corda.core.internal.copyBytes
 import net.corda.core.serialization.*
+import net.corda.core.serialization.internal.AttachmentsClassLoader
 import net.corda.core.utilities.ByteSequence
 import net.corda.serialization.internal.amqp.amqpMagic
 import org.slf4j.LoggerFactory
@@ -31,20 +32,12 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe
                                                               override val useCase: SerializationContext.UseCase,
                                                               override val encoding: SerializationEncoding?,
                                                               override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist,
-                                                              override val lenientCarpenterEnabled: Boolean = false,
+                                                              override val lenientCarpenterEnabled: Boolean = false) : SerializationContext {
-                                                              private val builder: AttachmentsClassLoaderBuilder = AttachmentsClassLoaderBuilder()
-) : SerializationContext {
-
-
     /**
      * {@inheritDoc}
-     *
-     * We need to cache the AttachmentClassLoaders to avoid too many contexts, since the class loader is part of cache key for the context.
      */
     override fun withAttachmentsClassLoader(attachmentHashes: List<SecureHash>): SerializationContext {
-        properties[attachmentsClassLoaderEnabledPropertyName] as? Boolean == true || return this
+        return this
-        val classLoader = builder.build(attachmentHashes, properties, deserializationClassLoader) ?: return this
-        return withClassLoader(classLoader)
     }
 
     override fun withProperty(property: Any, value: Any): SerializationContext {
@@ -72,34 +65,6 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe
     override fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist) = copy(encodingWhitelist = encodingWhitelist)
 }
 
-/*
- * This class is internal rather than private so that serialization-deterministic
- * can replace it with an alternative version.
- */
-@DeleteForDJVM
-class AttachmentsClassLoaderBuilder() {
-    private val cache: Cache<Pair<List<SecureHash>, ClassLoader>, AttachmentsClassLoader> = Caffeine.newBuilder().weakValues().maximumSize(1024).build()
-
-    fun build(attachmentHashes: List<SecureHash>, properties: Map<Any, Any>, deserializationClassLoader: ClassLoader): AttachmentsClassLoader? {
-        val serializationContext = properties[serializationContextKey] as? SerializeAsTokenContext ?: return null // Some tests don't set one.
-        try {
-            return cache.get(Pair(attachmentHashes, deserializationClassLoader)) {
-                val missing = ArrayList<SecureHash>()
-                val attachments = ArrayList<Attachment>()
-                attachmentHashes.forEach { id ->
-                    serializationContext.serviceHub.attachments.openAttachment(id)?.let { attachments += it }
-                        ?: run { missing += id }
-                }
-                missing.isNotEmpty() && throw MissingAttachmentsException(missing)
-                AttachmentsClassLoader(attachments, parent = deserializationClassLoader)
-            }!!
-        } catch (e: ExecutionException) {
-            // Caught from within the cache get, so unwrap.
-            throw e.cause!!
-        }
-    }
-}
-
 @KeepForDJVM
 open class SerializationFactoryImpl(
     // TODO: This is read-mostly. Probably a faster implementation to be found.

serialization/src/test/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderTests.kt

@@ -1,382 +0,0 @@
-package net.corda.serialization.internal
-
-import com.nhaarman.mockito_kotlin.doReturn
-import com.nhaarman.mockito_kotlin.whenever
-import net.corda.core.contracts.Attachment
-import net.corda.core.contracts.Contract
-import net.corda.core.crypto.SecureHash
-import net.corda.core.identity.CordaX500Name
-import net.corda.core.internal.declaredField
-import net.corda.core.internal.toWireTransaction
-import net.corda.core.node.ServiceHub
-import net.corda.core.node.services.AttachmentStorage
-import net.corda.core.serialization.*
-import net.corda.core.utilities.ByteSequence
-import net.corda.core.utilities.OpaqueBytes
-import net.corda.node.internal.cordapp.JarScanningCordappLoader
-import net.corda.node.internal.cordapp.CordappProviderImpl
-import net.corda.nodeapi.DummyContractBackdoor
-import net.corda.testing.common.internal.testNetworkParameters
-import net.corda.testing.core.DUMMY_NOTARY_NAME
-import net.corda.testing.core.SerializationEnvironmentRule
-import net.corda.testing.core.TestIdentity
-import net.corda.testing.internal.MockCordappConfigProvider
-import net.corda.testing.internal.kryoSpecific
-import net.corda.testing.internal.rigorousMock
-import net.corda.testing.services.MockAttachmentStorage
-import org.apache.commons.io.IOUtils
-import org.junit.Assert.*
-import org.junit.Rule
-import org.junit.Test
-import java.io.ByteArrayOutputStream
-import java.net.URL
-import java.net.URLClassLoader
-import java.util.jar.JarOutputStream
-import java.util.zip.ZipEntry
-import kotlin.test.assertFailsWith
-
-class AttachmentsClassLoaderTests {
-    companion object {
-        val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderTests::class.java.getResource("isolated.jar")
-        private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract"
-        private val DUMMY_NOTARY = TestIdentity(DUMMY_NOTARY_NAME, 20).party
-        private val MEGA_CORP = TestIdentity(CordaX500Name("MegaCorp", "London", "GB")).party
-        private fun SerializationContext.withAttachmentStorage(attachmentStorage: AttachmentStorage): SerializationContext {
-            val serviceHub = rigorousMock<ServiceHub>()
-            doReturn(attachmentStorage).whenever(serviceHub).attachments
-            return this.withServiceHub(serviceHub)
-        }
-
-        private fun SerializationContext.withServiceHub(serviceHub: ServiceHub): SerializationContext {
-            return this.withTokenContext(SerializeAsTokenContextImpl(serviceHub) {}).withProperty(attachmentsClassLoaderEnabledPropertyName, true)
-        }
-    }
-
-    @Rule
-    @JvmField
-    val testSerialization = SerializationEnvironmentRule()
-    private val attachments = MockAttachmentStorage()
-    private val networkParameters = testNetworkParameters()
-    private val cordappProvider = CordappProviderImpl(JarScanningCordappLoader.fromJarUrls(listOf(ISOLATED_CONTRACTS_JAR_PATH)), MockCordappConfigProvider(), attachments).apply {
-        start(networkParameters.whitelistedContractImplementations)
-    }
-    private val cordapp get() = cordappProvider.cordapps.first()
-    private val attachmentId get() = cordappProvider.getCordappAttachmentId(cordapp)!!
-    private val appContext get() = cordappProvider.getAppContext(cordapp)
-    private val serviceHub = rigorousMock<ServiceHub>().also {
-        doReturn(attachments).whenever(it).attachments
-        doReturn(cordappProvider).whenever(it).cordappProvider
-        doReturn(networkParameters).whenever(it).networkParameters
-    }
-
-    // These ClassLoaders work together to load 'AnotherDummyContract' in a disposable way, such that even though
-    // the class may be on the unit test class path (due to default IDE settings, etc), it won't be loaded into the
-    // regular app classloader but rather than ClassLoaderForTests. This helps keep our environment clean and
-    // ensures we have precise control over where it's loaded.
-    object FilteringClassLoader : ClassLoader() {
-        @Throws(ClassNotFoundException::class)
-        override fun loadClass(name: String, resolve: Boolean): Class<*> {
-            if ("AnotherDummyContract" in name) {
-                throw ClassNotFoundException(name)
-            }
-            return super.loadClass(name, resolve)
-        }
-    }
-
-    class ClassLoaderForTests : URLClassLoader(arrayOf(ISOLATED_CONTRACTS_JAR_PATH), FilteringClassLoader)
-    @Test
-    fun `dynamically load AnotherDummyContract from isolated contracts jar`() {
-        ClassLoaderForTests().use { child ->
-            val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child)
-            val contract = contractClass.newInstance() as Contract
-
-            assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)
-        }
-    }
-
-    private fun fakeAttachment(filepath: String, content: String): ByteArray {
-        val bs = ByteArrayOutputStream()
-        JarOutputStream(bs).use { js ->
-            js.putNextEntry(ZipEntry(filepath))
-            js.writer().apply { append(content); flush() }
-            js.closeEntry()
-        }
-        return bs.toByteArray()
-    }
-
-    private fun readAttachment(attachment: Attachment, filepath: String): ByteArray {
-        ByteArrayOutputStream().use {
-            attachment.extractFile(filepath, it)
-            return it.toByteArray()
-        }
-    }
-
-    @Test
-    fun `test MockAttachmentStorage open as jar`() {
-        val storage = attachments
-        val key = attachmentId
-        val attachment = storage.openAttachment(key)!!
-
-        val jar = attachment.openAsJAR()
-
-        assertNotNull(jar.nextEntry)
-    }
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun `test overlapping file exception`() {
-        val storage = attachments
-        val att0 = attachmentId
-        val att1 = storage.importAttachment(fakeAttachment("file.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("file.txt", "some other data").inputStream())
-
-        assertFailsWith(AttachmentsClassLoader.OverlappingAttachments::class) {
-            AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! })
-        }
-    }
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun basic() {
-        val storage = attachments
-        val att0 = attachmentId
-        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream())
-
-        val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! })
-        val txt = IOUtils.toString(cl.getResourceAsStream("file1.txt"), Charsets.UTF_8.name())
-        assertEquals("some data", txt)
-    }
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun `Check platform independent path handling in attachment jars`() {
-        val storage = MockAttachmentStorage()
-
-        val att1 = storage.importAttachment(fakeAttachment("/folder1/foldera/file1.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("\\folder1\\folderb\\file2.txt", "some other data").inputStream())
-
-        val data1a = readAttachment(storage.openAttachment(att1)!!, "/folder1/foldera/file1.txt")
-        assertArrayEquals("some data".toByteArray(), data1a)
-
-        val data1b = readAttachment(storage.openAttachment(att1)!!, "\\folder1\\foldera\\file1.txt")
-        assertArrayEquals("some data".toByteArray(), data1b)
-
-        val data2a = readAttachment(storage.openAttachment(att2)!!, "\\folder1\\folderb\\file2.txt")
-        assertArrayEquals("some other data".toByteArray(), data2a)
-
-        val data2b = readAttachment(storage.openAttachment(att2)!!, "/folder1/folderb/file2.txt")
-        assertArrayEquals("some other data".toByteArray(), data2b)
-    }
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun `loading class AnotherDummyContract`() {
-        val storage = attachments
-        val att0 = attachmentId
-        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream())
-
-        val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader)
-        val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl)
-        val contract = contractClass.newInstance() as Contract
-        assertEquals(cl, contract.javaClass.classLoader)
-        assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)
-    }
-
-    private fun createContract2Cash(): Contract {
-        ClassLoaderForTests().use { cl ->
-            val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl)
-            return contractClass.newInstance() as Contract
-        }
-    }
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun `testing Kryo with ClassLoader (with top level class name)`() {
-        val contract = createContract2Cash()
-
-        val bytes = contract.serialize()
-        val storage = attachments
-        val att0 = attachmentId
-        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream())
-
-        val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader)
-
-        val context = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl).withWhitelisted(contract.javaClass)
-        val state2 = bytes.deserialize(context = context)
-        assertTrue(state2.javaClass.classLoader is AttachmentsClassLoader)
-        assertNotNull(state2)
-    }
-
-    // top level wrapper
-    @CordaSerializable
-    class Data(val contract: Contract)
-
-    @Test
-    @Suppress("DEPRECATION")
-    fun `testing Kryo with ClassLoader (without top level class name)`() {
-        val data = Data(createContract2Cash())
-
-        assertNotNull(data.contract)
-
-        val context2 = SerializationFactory.defaultFactory.defaultContext.withWhitelisted(data.contract.javaClass)
-
-        val bytes = data.serialize(context = context2)
-        val storage = attachments
-        val att0 = attachmentId
-        val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream())
-        val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream())
-
-        val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader)
-
-        val context = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl).withWhitelisted(Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl))
-
-        val state2 = bytes.deserialize(context = context)
-        assertEquals(cl, state2.contract.javaClass.classLoader)
-        assertNotNull(state2)
-
-        // We should be able to load same class from a different class loader and have them be distinct.
-        val cl2 = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader)
-
-        val context3 = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl2).withWhitelisted(Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl2))
-
-        val state3 = bytes.deserialize(context = context3)
-        assertEquals(cl2, state3.contract.javaClass.classLoader)
-        assertNotNull(state3)
-    }
-
-    @Test
-    fun `test serialization of SecureHash`() {
-        val secureHash = SecureHash.randomSHA256()
-        val bytes = secureHash.serialize()
-        val copiedSecuredHash = bytes.deserialize()
-
-        assertEquals(secureHash, copiedSecuredHash)
-    }
-
-    @Test
-    fun `test serialization of OpaqueBytes`() {
-        val opaqueBytes = OpaqueBytes("0123456789".toByteArray())
-        val bytes = opaqueBytes.serialize()
-        val copiedOpaqueBytes = bytes.deserialize()
-
-        assertEquals(opaqueBytes, copiedOpaqueBytes)
-    }
-
-    @Test
-    fun `test serialization of sub-sequence OpaqueBytes`() {
-        val bytesSequence = ByteSequence.of("0123456789".toByteArray(), 3, 2)
-        val bytes = bytesSequence.serialize()
-        val copiedBytesSequence = bytes.deserialize()
-
-        assertEquals(bytesSequence, copiedBytesSequence)
-    }
-
-    @Test
-    fun `test serialization of WireTransaction with dynamically loaded contract`() {
-        val child = appContext.classLoader
-        val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child)
-        val contract = contractClass.newInstance() as DummyContractBackdoor
-        val tx = contract.generateInitial(MEGA_CORP.ref(0), 42, DUMMY_NOTARY)
-        val context = SerializationFactory.defaultFactory.defaultContext
-                .withWhitelisted(contract.javaClass)
-                .withWhitelisted(Class.forName("$ISOLATED_CONTRACT_CLASS_NAME\$State", true, child))
-                .withWhitelisted(Class.forName("$ISOLATED_CONTRACT_CLASS_NAME\$Commands\$Create", true, child))
-                .withServiceHub(serviceHub)
-                .withClassLoader(child)
-
-        val bytes = run {
-            val wireTransaction = tx.toWireTransaction(serviceHub, context)
-            wireTransaction.serialize(context = context)
-        }
-        val copiedWireTransaction = bytes.deserialize(context = context)
-        assertEquals(1, copiedWireTransaction.outputs.size)
-        // Contracts need to be loaded by the same classloader as the ContractState itself
-        val contractClassloader = copiedWireTransaction.getOutput(0).javaClass.classLoader
-        val contract2 = contractClassloader.loadClass(copiedWireTransaction.outputs.first().contract).newInstance() as DummyContractBackdoor
-        assertEquals(contract2.javaClass.classLoader, copiedWireTransaction.outputs[0].data.javaClass.classLoader)
-        assertEquals(42, contract2.inspectState(copiedWireTransaction.outputs[0].data))
-    }
-
-    @Test
-    fun `test deserialize of WireTransaction where contract cannot be found`() {
-        kryoSpecific("Kryo verifies/loads attachments on deserialization, whereas AMQP currently does not") {
-            ClassLoaderForTests().use { child ->
-                val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child)
-                val contract = contractClass.newInstance() as DummyContractBackdoor
-                val tx = contract.generateInitial(MEGA_CORP.ref(0), 42, DUMMY_NOTARY)
-                val attachmentRef = attachmentId
-                val bytes = run {
-                    val outboundContext = SerializationFactory.defaultFactory.defaultContext
-                            .withServiceHub(serviceHub)
-                            .withClassLoader(child)
-                    val wireTransaction = tx.toWireTransaction(serviceHub, outboundContext)
-                    wireTransaction.serialize(context = outboundContext)
-                }
-                // use empty attachmentStorage
-
-                val e = assertFailsWith(MissingAttachmentsException::class) {
-                    val mockAttStorage = MockAttachmentStorage()
-                    val inboundContext = SerializationFactory.defaultFactory.defaultContext
-                            .withAttachmentStorage(mockAttStorage)
-                            .withAttachmentsClassLoader(listOf(attachmentRef))
-                    bytes.deserialize(context = inboundContext)
-
-                    if (mockAttStorage.openAttachment(attachmentRef) == null) {
-                        throw MissingAttachmentsException(listOf(attachmentRef))
-                    }
-                }
-                assertEquals(attachmentRef, e.ids.single())
-            }
-        }
-    }
-
-    @Test
-    fun `test loading a class from attachment during deserialization`() {
-        ClassLoaderForTests().use { child ->
-            val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child)
-            val contract = contractClass.newInstance() as DummyContractBackdoor
-            val outboundContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(child)
-            val attachmentRef = attachmentId
-            // We currently ignore annotations in attachments, so manually whitelist.
-            val inboundContext = SerializationFactory
-                    .defaultFactory
-                    .defaultContext
-                    .withWhitelisted(contract.javaClass)
-                    .withServiceHub(serviceHub)
-                    .withAttachmentsClassLoader(listOf(attachmentRef))
-
-            // Serialize with custom context to avoid populating the default context with the specially loaded class
-            val serialized = contract.serialize(context = outboundContext)
-            // Then deserialize with the attachment class loader associated with the attachment
-            serialized.deserialize(context = inboundContext)
-        }
-    }
-
-    @Test
-    fun `test loading a class with attachment missing during deserialization`() {
-        ClassLoaderForTests().use { child ->
-            val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child)
-            val contract = contractClass.newInstance() as DummyContractBackdoor
-            val attachmentRef = SecureHash.randomSHA256()
-            val outboundContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(child)
-            // Serialize with custom context to avoid populating the default context with the specially loaded class
-            val serialized = contract.serialize(context = outboundContext)
-
-            // Then deserialize with the attachment class loader associated with the attachment
-            val e = assertFailsWith(MissingAttachmentsException::class) {
-                // We currently ignore annotations in attachments, so manually whitelist.
-                val inboundContext = SerializationFactory
-                        .defaultFactory
-                        .defaultContext
-                        .withWhitelisted(contract.javaClass)
-                        .withServiceHub(serviceHub)
-                        .withAttachmentsClassLoader(listOf(attachmentRef))
-                serialized.deserialize(context = inboundContext)
-            }
-            assertEquals(attachmentRef, e.ids.single())
-        }
-    }
-}

serialization/src/test/kotlin/net/corda/serialization/internal/CordaClassResolverTests.kt

@@ -14,6 +14,7 @@ import net.corda.core.node.services.AttachmentStorage
 import net.corda.core.serialization.internal.CheckpointSerializationContext
 import net.corda.core.serialization.ClassWhitelist
 import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.internal.AttachmentsClassLoader
 import net.corda.node.serialization.kryo.CordaClassResolver
 import net.corda.node.serialization.kryo.CordaKryo
 import net.corda.testing.internal.rigorousMock
@@ -22,6 +23,7 @@ import org.junit.Rule
 import org.junit.Test
 import org.junit.rules.ExpectedException
 import java.lang.IllegalStateException
+import java.net.URL
 import java.sql.Connection
 import java.util.*
 import kotlin.test.assertEquals
@@ -112,6 +114,7 @@ class CordaClassResolverTests {
         val emptyListClass = listOf<Any>().javaClass
         val emptySetClass = setOf<Any>().javaClass
         val emptyMapClass = mapOf<Any, Any>().javaClass
+        val ISOLATED_CONTRACTS_JAR_PATH: URL = CordaClassResolverTests::class.java.getResource("isolated.jar")
     }
 
     private val emptyWhitelistContext: CheckpointSerializationContext = CheckpointSerializationContextImpl(this.javaClass.classLoader, EmptyWhitelist, emptyMap(), true, null)
@@ -201,7 +204,7 @@ class CordaClassResolverTests {
         CordaClassResolver(emptyWhitelistContext).getRegistration(DefaultSerializable::class.java)
     }
 
-    private fun importJar(storage: AttachmentStorage, uploader: String = DEPLOYED_CORDAPP_UPLOADER) = AttachmentsClassLoaderTests.ISOLATED_CONTRACTS_JAR_PATH.openStream().use { storage.importAttachment(it, uploader, "") }
+    private fun importJar(storage: AttachmentStorage, uploader: String = DEPLOYED_CORDAPP_UPLOADER) = ISOLATED_CONTRACTS_JAR_PATH.openStream().use { storage.importAttachment(it, uploader, "") }
 
     @Test(expected = KryoException::class)
     fun `Annotation does not work in conjunction with AttachmentClassLoader annotation`() {

testing/test-utils/src/main/kotlin/net/corda/testing/core/TestUtils.kt

@@ -161,3 +161,4 @@ fun NodeInfo.singleIdentityAndCert(): PartyAndCertificate = legalIdentitiesAndCe
  * Extract a single identity from the node info. Throws an error if the node has multiple identities.
  */
 fun NodeInfo.singleIdentity(): Party = singleIdentityAndCert().party
+

testing/test-utils/src/main/kotlin/net/corda/testing/internal/InternalTestUtils.kt

@@ -33,10 +33,13 @@ import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.nodeapi.internal.registerDevP2pCertificates
 import net.corda.serialization.internal.amqp.AMQP_ENABLED
 import net.corda.testing.internal.stubs.CertificateStoreStubs
+import java.io.ByteArrayOutputStream
 import java.nio.file.Files
 import java.nio.file.Path
 import java.security.KeyPair
 import java.util.*
+import java.util.jar.JarOutputStream
+import java.util.zip.ZipEntry
 import javax.security.auth.x500.X500Principal
 
 @Suppress("unused")
@@ -169,7 +172,20 @@ fun configureDatabase(hikariProperties: Properties,
                       schemaService: SchemaService = NodeSchemaService(),
                       internalSchemas: Set<MappedSchema> = NodeSchemaService().internalSchemas(),
                       cacheFactory: NamedCacheFactory = TestingNamedCacheFactory()): CordaPersistence {
-    val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService, hikariProperties, cacheFactory)
+    val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService, hikariProperties, cacheFactory, null)
     persistence.startHikariPool(hikariProperties, databaseConfig, internalSchemas)
     return persistence
 }
+
+/**
+ * Convenience method for creating a fake attachment containing a file with some content.
+ */
+fun fakeAttachment(filePath: String, content: String): ByteArray {
+    val bs = ByteArrayOutputStream()
+    JarOutputStream(bs).use { js ->
+        js.putNextEntry(ZipEntry(filePath))
+        js.writer().apply { append(content); flush() }
+        js.closeEntry()
+    }
+    return bs.toByteArray()
+}

testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt

@@ -39,7 +39,7 @@ class MockCordappProvider(
                 allFlows = emptyList(),
                 jarHash = SecureHash.allOnesHash)
         if (cordappRegistry.none { it.first.contractClassNames.contains(contractClassName) && it.second == contractHash }) {
-            cordappRegistry.add(Pair(cordapp, findOrImportAttachment(listOf(contractClassName), contractClassName.toByteArray(), attachments, contractHash, signers)))
+            cordappRegistry.add(Pair(cordapp, findOrImportAttachment(listOf(contractClassName), fakeAttachmentCached(contractClassName), attachments, contractHash, signers)))
         }
         return cordappRegistry.findLast { contractClassName in it.first.contractClassNames }?.second!!
     }
@@ -57,4 +57,9 @@ class MockCordappProvider(
             attachments.importContractAttachment(contractClassNames, DEPLOYED_CORDAPP_UPLOADER, data.inputStream(), contractHash, signers)
         }
     }
+
+    private val attachmentsCache = mutableMapOf<String, ByteArray>()
+    private fun fakeAttachmentCached(contractClass: String): ByteArray = attachmentsCache.computeIfAbsent(contractClass) {
+        fakeAttachment(contractClass, contractClass)
+    }
 }