diff --git a/core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt b/core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt index c509abdd4f..5b27d28ef5 100644 --- a/core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt +++ b/core/src/main/kotlin/net/corda/core/internal/AbstractAttachment.kt @@ -27,6 +27,11 @@ fun isUploaderTrusted(uploader: String?): Boolean = uploader in TRUSTED_UPLOADER @KeepForDJVM abstract class AbstractAttachment(dataLoader: () -> ByteArray) : Attachment { companion object { + /** + * Returns a function that knows how to load an attachment. + * + * TODO - this code together with the rest of the Attachment handling (including [FetchedAttachment]) needs some refactoring as it is really hard to follow. + */ @DeleteForDJVM fun SerializeAsTokenContext.attachmentDataLoader(id: SecureHash): () -> ByteArray { return { diff --git a/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt b/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt index 6595851d22..5c07f82117 100644 --- a/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt @@ -1,6 +1,10 @@ package net.corda.core.internal import net.corda.core.DeleteForDJVM +import net.corda.core.KeepForDJVM +import net.corda.core.contracts.ContractState +import net.corda.core.contracts.StateRef +import net.corda.core.contracts.TransactionState import net.corda.core.cordapp.Cordapp import net.corda.core.cordapp.CordappConfig import net.corda.core.cordapp.CordappContext @@ -8,11 +12,14 @@ import net.corda.core.crypto.SecureHash import net.corda.core.flows.FlowLogic import net.corda.core.node.ServicesForResolution import net.corda.core.node.ZoneVersionTooLowException +import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializationContext +import net.corda.core.serialization.SerializedBytes import net.corda.core.transactions.LedgerTransaction import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction +import net.corda.core.utilities.OpaqueBytes import org.slf4j.MDC // *Internal* Corda-specific utilities @@ -73,3 +80,11 @@ class LazyMappedList(val originalList: List, val transform: (T, Int) -> override fun get(index: Int) = partialResolvedList[index] ?: transform(originalList[index], index).also { computed -> partialResolvedList[index] = computed } } + +/** + * A SerializedStateAndRef is a pair (BinaryStateRepresentation, StateRef). + * The [serializedState] is the actual component from the original transaction. + */ +@KeepForDJVM +@CordaSerializable +data class SerializedStateAndRef(val serializedState: SerializedBytes>, val ref: StateRef) \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt index ac306fbdbb..1a0a1d5143 100644 --- a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt @@ -221,15 +221,16 @@ data class InputStreamAndHash(val inputStream: InputStream, val sha256: SecureHa * Note that a slightly bigger than numOfExpectedBytes size is expected. */ @DeleteForDJVM - fun createInMemoryTestZip(numOfExpectedBytes: Int, content: Byte): InputStreamAndHash { + fun createInMemoryTestZip(numOfExpectedBytes: Int, content: Byte, entryName: String = "z"): InputStreamAndHash { require(numOfExpectedBytes > 0){"Expected bytes must be greater than zero"} + require(numOfExpectedBytes > 0) val baos = ByteArrayOutputStream() ZipOutputStream(baos).use { zos -> val arraySize = 1024 val bytes = ByteArray(arraySize) { content } val n = (numOfExpectedBytes - 1) / arraySize + 1 // same as Math.ceil(numOfExpectedBytes/arraySize). zos.setLevel(Deflater.NO_COMPRESSION) - zos.putNextEntry(ZipEntry("z")) + zos.putNextEntry(ZipEntry(entryName)) for (i in 0 until n) { zos.write(bytes, 0, arraySize) } @@ -501,3 +502,18 @@ fun SerializedBytes.checkPayloadIs(type: Class): Untrustworthy return type.castIfPossible(payloadData)?.let { UntrustworthyData(it) } ?: throw IllegalArgumentException("We were expecting a ${type.name} but we instead got a ${payloadData.javaClass.name} ($payloadData)") } + +/** + * Simple Map structure that can be used as a cache in the DJVM. + */ +fun createSimpleCache(maxSize: Int, onEject: (MutableMap.MutableEntry) -> Unit = {}): MutableMap { + return object : LinkedHashMap() { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + val eject = size > maxSize + if (eject) onEject(eldest!!) + return eject + } + } +} + +fun MutableMap.toSynchronised(): MutableMap = Collections.synchronizedMap(this) \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt b/core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt index 3e774b4b6e..b8d574803e 100644 --- a/core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt @@ -1,15 +1,23 @@ package net.corda.core.internal -import net.corda.core.contracts.ContractClassName -import net.corda.core.contracts.PrivacySalt -import net.corda.core.contracts.StateRef +import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash +import net.corda.core.crypto.componentHash import net.corda.core.crypto.sha256 import net.corda.core.identity.Party +import net.corda.core.serialization.MissingAttachmentsException +import net.corda.core.serialization.SerializationContext +import net.corda.core.serialization.SerializationFactory import net.corda.core.serialization.serialize +import net.corda.core.transactions.ComponentGroup import net.corda.core.transactions.ContractUpgradeWireTransaction +import net.corda.core.transactions.FilteredComponentGroup import net.corda.core.transactions.NotaryChangeWireTransaction +import net.corda.core.utilities.OpaqueBytes +import net.corda.core.utilities.lazyMapped import java.io.ByteArrayOutputStream +import java.security.PublicKey +import kotlin.reflect.KClass /** Constructs a [NotaryChangeWireTransaction]. */ class NotaryChangeTransactionBuilder(val inputs: List, @@ -42,4 +50,75 @@ fun combinedHash(components: Iterable): SecureHash { stream.write(it.bytes) } return stream.toByteArray().sha256() +} + +/** + * This function knows how to deserialize a transaction component group. + * + * In case the [componentGroups] is an instance of [LazyMappedList], this function will just use the original deserialized version, and avoid an unnecessary deserialization. + * The [forceDeserialize] will force deserialization. In can be used in case the SerializationContext changes. + */ +fun deserialiseComponentGroup(componentGroups: List, + clazz: KClass, + groupEnum: ComponentGroupEnum, + forceDeserialize: Boolean = false, + factory: SerializationFactory = SerializationFactory.defaultFactory, + context: SerializationContext = factory.defaultContext): List { + val group = componentGroups.firstOrNull { it.groupIndex == groupEnum.ordinal } + + if (group == null || group.components.isEmpty()) { + return emptyList() + } + + // If the componentGroup is a [LazyMappedList] it means that the original deserialized version is already available. + val components = group.components + if (!forceDeserialize && components is LazyMappedList<*, OpaqueBytes>) { + return components.originalList as List + } + + return components.lazyMapped { component, internalIndex -> + try { + factory.deserialize(component, clazz.java, context) + } catch (e: MissingAttachmentsException) { + throw e + } catch (e: Exception) { + throw Exception("Malformed transaction, $groupEnum at index $internalIndex cannot be deserialised", e) + } + } +} + +/** + * Method to deserialise Commands from its two groups: + * * COMMANDS_GROUP which contains the CommandData part + * * and SIGNERS_GROUP which contains the Signers part. + * + * This method used the [deserialiseComponentGroup] method. + */ +fun deserialiseCommands(componentGroups: List, + forceDeserialize: Boolean = false, + factory: SerializationFactory = SerializationFactory.defaultFactory, + context: SerializationContext = factory.defaultContext): List> { + // TODO: we could avoid deserialising unrelated signers. + // However, current approach ensures the transaction is not malformed + // and it will throw if any of the signers objects is not List of public keys). + val signersList: List> = uncheckedCast(deserialiseComponentGroup(componentGroups, List::class, ComponentGroupEnum.SIGNERS_GROUP, forceDeserialize)) + val commandDataList: List = deserialiseComponentGroup(componentGroups, CommandData::class, ComponentGroupEnum.COMMANDS_GROUP, forceDeserialize) + val group = componentGroups.firstOrNull { it.groupIndex == ComponentGroupEnum.COMMANDS_GROUP.ordinal } + return if (group is FilteredComponentGroup) { + check(commandDataList.size <= signersList.size) { + "Invalid Transaction. Less Signers (${signersList.size}) than CommandData (${commandDataList.size}) objects" + } + val componentHashes = group.components.mapIndexed { index, component -> componentHash(group.nonces[index], component) } + val leafIndices = componentHashes.map { group.partialMerkleTree.leafIndex(it) } + if (leafIndices.isNotEmpty()) + check(leafIndices.max()!! < signersList.size) { "Invalid Transaction. A command with no corresponding signer detected" } + commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[leafIndices[index]]) } + } else { + // It is a WireTransaction + // or a FilteredTransaction with no Commands (in which case group is null). + check(commandDataList.size == signersList.size) { + "Invalid Transaction. Sizes of CommandData (${commandDataList.size}) and Signers (${signersList.size}) do not match" + } + commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[index]) } + } } \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt index 3a0ee16ce0..941617ba04 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt @@ -177,10 +177,10 @@ interface SerializationContext { fun withClassLoader(classLoader: ClassLoader): SerializationContext /** - * Helper method to return a new context based on this context with the appropriate class loader constructed from the passed attachment identifiers. - * (Requires the attachment storage to have been enabled). + * Does not do anything. */ @Throws(MissingAttachmentsException::class) + @Deprecated("There is no reason to call this. This method does not actually do anything.") fun withAttachmentsClassLoader(attachmentHashes: List): SerializationContext /** diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt new file mode 100644 index 0000000000..1a8d25437c --- /dev/null +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt @@ -0,0 +1,161 @@ +package net.corda.core.serialization.internal + +import net.corda.core.contracts.Attachment +import net.corda.core.contracts.ContractAttachment +import net.corda.core.crypto.SecureHash +import net.corda.core.internal.VisibleForTesting +import net.corda.core.internal.isUploaderTrusted +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializationFactory +import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory.toUrl +import net.corda.core.internal.createSimpleCache +import net.corda.core.internal.toSynchronised +import java.io.IOException +import java.io.InputStream +import java.net.* + +/** + * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only + * need to provide JAR streams, and so could be fetched from a database, local disk, etc. Constructing an + * AttachmentsClassLoader is somewhat expensive, as every attachment is scanned to ensure that there are no overlapping + * file paths. + */ +class AttachmentsClassLoader(attachments: List, parent: ClassLoader = ClassLoader.getSystemClassLoader()) : + URLClassLoader(attachments.map(::toUrl).toTypedArray(), parent) { + + companion object { + + init { + // This is required to register the AttachmentURLStreamHandlerFactory. + URL.setURLStreamHandlerFactory(AttachmentURLStreamHandlerFactory) + } + + private const val `META-INF` = "meta-inf" + private val excludeFromNoOverlapCheck = setOf( + "manifest.mf", + "license", + "license.txt", + "notice", + "notice.txt", + "index.list" + ) + + private fun shouldCheckForNoOverlap(path: String): Boolean { + if (!path.startsWith(`META-INF`)) return true + val p = path.substring(`META-INF`.length + 1) + if (p in excludeFromNoOverlapCheck) return false + if (p.endsWith(".sf") || p.endsWith(".dsa")) return false + return true + } + + @CordaSerializable + class OverlappingAttachments(val path: String) : Exception() { + override fun toString() = "Multiple attachments define a file at path $path" + } + + private fun requireNoDuplicates(attachments: List) { + val classLoaderEntries = mutableSetOf() + for (attachment in attachments) { + attachment.openAsJAR().use { jar -> + while (true) { + val entry = jar.nextJarEntry ?: break + + // We already verified that paths are not strange/game playing when we inserted the attachment + // into the storage service. So we don't need to repeat it here. + // + // We forbid files that differ only in case, or path separator to avoid issues for Windows/Mac developers where the + // filesystem tries to be case insensitive. This may break developers who attempt to use ProGuard. + // + // Also convert to Unix path separators as all resource/class lookups will expect this. + // If 2 entries have the same CRC, it means the same file is present in both attachments, so that is ok. TODO - Mike, wdyt? + val path = entry.name.toLowerCase().replace('\\', '/') + if (shouldCheckForNoOverlap(path)) { + if (path in classLoaderEntries) throw OverlappingAttachments(path) + classLoaderEntries.add(path) + } + } + } + } + } + } + + init { + require(attachments.mapNotNull { it as? ContractAttachment }.all { isUploaderTrusted(it.uploader) }) { + "Attempting to load Contract Attachments downloaded from the network" + } + + requireNoDuplicates(attachments) + } +} + +/** + * This is just a factory that provides a cache to avoid constructing expensive [AttachmentsClassLoader]s. + */ +@VisibleForTesting +internal object AttachmentsClassLoaderBuilder { + + private const val ATTACHMENT_CLASSLOADER_CACHE_SIZE = 1000 + + // This runs in the DJVM so it can't use caffeine. + private val cache: MutableMap, AttachmentsClassLoader> = createSimpleCache, AttachmentsClassLoader>(ATTACHMENT_CLASSLOADER_CACHE_SIZE) + .toSynchronised() + + fun build(attachments: List): AttachmentsClassLoader { + return cache.computeIfAbsent(attachments.map { it.id }.sorted()) { + AttachmentsClassLoader(attachments) + } + } + + fun withAttachmentsClassloaderContext(attachments: List, block: (ClassLoader) -> T): T { + + // Create classloader from the attachments. + val transactionClassLoader = AttachmentsClassLoaderBuilder.build(attachments) + + // Create a new serializationContext for the current Transaction. + val transactionSerializationContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(transactionClassLoader) + + // Deserialize all relevant classes in the transaction classloader. + return SerializationFactory.defaultFactory.withCurrentContext(transactionSerializationContext) { + block(transactionClassLoader) + } + } +} + +/** + * Registers a new internal "attachment" protocol. + * This will not be exposed as an API. + */ +object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory { + private const val attachmentScheme = "attachment" + + // TODO - what happens if this grows too large? + private val loadedAttachments = mutableMapOf().toSynchronised() + + override fun createURLStreamHandler(protocol: String): URLStreamHandler? { + return if (attachmentScheme == protocol) { + AttachmentURLStreamHandler + } else null + } + + fun toUrl(attachment: Attachment): URL { + val id = attachment.id.toString() + loadedAttachments[id] = attachment + return URL(attachmentScheme, "", -1, id, AttachmentURLStreamHandler) + } + + private object AttachmentURLStreamHandler : URLStreamHandler() { + override fun openConnection(url: URL): URLConnection { + if (url.protocol != attachmentScheme) throw IOException("Cannot handle protocol: ${url.protocol}") + val attachment = loadedAttachments[url.path] ?: throw IOException("Could not load url: $url .") + return AttachmentURLConnection(url, attachment) + } + } + + private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) { + override fun getContentLengthLong(): Long = attachment.size.toLong() + override fun getInputStream(): InputStream = attachment.open() + override fun connect() { + connected = true + } + } +} diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt index 6769b73b03..f870e9aa0e 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt @@ -73,13 +73,6 @@ interface CheckpointSerializationContext { */ fun withClassLoader(classLoader: ClassLoader): CheckpointSerializationContext - /** - * Helper method to return a new context based on this context with the appropriate class loader constructed from the passed attachment identifiers. - * (Requires the attachment storage to have been enabled). - */ - @Throws(MissingAttachmentsException::class) - fun withAttachmentsClassLoader(attachmentHashes: List): CheckpointSerializationContext - /** * Helper method to return a new context based on this context with the given class specifically whitelisted. */ diff --git a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt index ddadaefc0f..0ea2c661aa 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt @@ -1,5 +1,6 @@ package net.corda.core.transactions +import net.corda.core.CordaInternal import net.corda.core.KeepForDJVM import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash @@ -11,10 +12,12 @@ import net.corda.core.internal.AttachmentWithContext import net.corda.core.internal.combinedHash import net.corda.core.node.NetworkParameters import net.corda.core.node.ServicesForResolution -import net.corda.core.serialization.CordaSerializable -import net.corda.core.serialization.deserialize +import net.corda.core.serialization.* +import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder import net.corda.core.transactions.ContractUpgradeFilteredTransaction.FilteredComponent +import net.corda.core.transactions.ContractUpgradeWireTransaction.Companion.calculateUpgradedState import net.corda.core.transactions.ContractUpgradeWireTransaction.Component.* +import net.corda.core.transactions.WireTransaction.Companion.resolveStateRefBinaryComponent import net.corda.core.utilities.OpaqueBytes import net.corda.core.utilities.toBase58String import java.security.PublicKey @@ -35,6 +38,32 @@ data class ContractUpgradeWireTransaction( /** Required for hiding components in [ContractUpgradeFilteredTransaction]. */ val privacySalt: PrivacySalt = PrivacySalt() ) : CoreTransaction() { + + companion object { + /** + * Runs the explicit upgrade logic. + */ + @CordaInternal + internal fun calculateUpgradedState(state: TransactionState, upgradedContract: UpgradedContract, upgradedContractAttachment: Attachment): TransactionState { + // TODO: if there are encumbrance states in the inputs, just copy them across without modifying + val upgradedState: S = upgradedContract.upgrade(state.data) + val inputConstraint = state.constraint + val outputConstraint = when (inputConstraint) { + is HashAttachmentConstraint -> HashAttachmentConstraint(upgradedContractAttachment.id) + WhitelistedByZoneAttachmentConstraint -> WhitelistedByZoneAttachmentConstraint + else -> throw IllegalArgumentException("Unsupported input contract constraint $inputConstraint") + } + // TODO: re-map encumbrance pointers + return TransactionState( + data = upgradedState, + contract = upgradedContract::class.java.name, + constraint = outputConstraint, + notary = state.notary, + encumbrance = state.encumbrance + ) + } + } + override val inputs: List = serializedComponents[INPUTS.ordinal].deserialize() override val notary: Party by lazy { serializedComponents[NOTARY.ordinal].deserialize() } val legacyContractAttachmentId: SecureHash by lazy { serializedComponents[LEGACY_ATTACHMENT.ordinal].deserialize() } @@ -90,6 +119,32 @@ data class ContractUpgradeWireTransaction( ) } + private fun upgradedContract(className: ContractClassName, classLoader: ClassLoader): UpgradedContract = try { + classLoader.loadClass(className).asSubclass(UpgradedContract::class.java as Class>) + .newInstance() + } catch (e: Exception) { + throw TransactionVerificationException.ContractCreationError(id, className, e) + } + + /** + * Creates a binary serialized component for a virtual output state serialised and executed with the attachments from the transaction. + */ + @CordaInternal + internal fun resolveOutputComponent(services: ServicesForResolution, stateRef: StateRef): SerializedBytes> { + val binaryInput = resolveStateRefBinaryComponent(inputs[stateRef.index], services)!! + val legacyAttachment = services.attachments.openAttachment(legacyContractAttachmentId) + ?: throw MissingContractAttachments(emptyList()) + val upgradedAttachment = services.attachments.openAttachment(upgradedContractAttachmentId) + ?: throw MissingContractAttachments(emptyList()) + + return AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(listOf(legacyAttachment, upgradedAttachment)) { transactionClassLoader -> + val resolvedInput = binaryInput.deserialize>() + val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader) + val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment) + outputState.serialize() + } + } + /** Constructs a filtered transaction: the inputs and the notary party are always visible, while the rest are hidden. */ fun buildFilteredTransaction(): ContractUpgradeFilteredTransaction { val totalComponents = (0 until serializedComponents.size).toSet() @@ -222,22 +277,7 @@ data class ContractUpgradeLedgerTransaction( * Outputs are computed by running the contract upgrade logic on input states. This is done eagerly so that the * transaction is verified during construction. */ - override val outputs: List> = inputs.map { (state) -> - // TODO: if there are encumbrance states in the inputs, just copy them across without modifying - val upgradedState = upgradedContract.upgrade(state.data) - val inputConstraint = state.constraint - val outputConstraint = when (inputConstraint) { - is HashAttachmentConstraint -> HashAttachmentConstraint(upgradedContractAttachment.id) - WhitelistedByZoneAttachmentConstraint -> WhitelistedByZoneAttachmentConstraint - else -> throw IllegalArgumentException("Unsupported input contract constraint $inputConstraint") - } - // TODO: re-map encumbrance pointers - state.copy( - data = upgradedState, - contract = upgradedContractClassName, - constraint = outputConstraint - ) - } + override val outputs: List> = inputs.map { calculateUpgradedState(it.state, upgradedContract, upgradedContractAttachment) } /** The required signers are the set of all input states' participants. */ override val requiredSigningKeys: Set diff --git a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt index c72221a204..222a04897c 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt @@ -1,22 +1,21 @@ package net.corda.core.transactions +import net.corda.core.CordaInternal import net.corda.core.KeepForDJVM import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash import net.corda.core.crypto.isFulfilledBy import net.corda.core.identity.Party -import net.corda.core.internal.AttachmentWithContext -import net.corda.core.internal.castIfPossible -import net.corda.core.internal.checkMinimumPlatformVersion -import net.corda.core.internal.uncheckedCast +import net.corda.core.internal.* import net.corda.core.node.NetworkParameters import net.corda.core.serialization.CordaSerializable -import net.corda.core.utilities.Try +import net.corda.core.serialization.deserialize +import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder import net.corda.core.utilities.loggerFor +import net.corda.core.utilities.warnOnce import java.util.* import java.util.function.Predicate import kotlin.collections.HashSet -import net.corda.core.utilities.warnOnce /** * A LedgerTransaction is derived from a [WireTransaction]. It is the result of doing the following operations: @@ -34,7 +33,7 @@ import net.corda.core.utilities.warnOnce // DOCSTART 1 @KeepForDJVM @CordaSerializable -data class LedgerTransaction @JvmOverloads constructor( +data class LedgerTransaction private constructor( /** The resolved input states which will be consumed/invalidated by the execution of this transaction. */ override val inputs: List>, override val outputs: List>, @@ -47,9 +46,38 @@ data class LedgerTransaction @JvmOverloads constructor( override val notary: Party?, val timeWindow: TimeWindow?, val privacySalt: PrivacySalt, - private val networkParameters: NetworkParameters? = null, - override val references: List> = emptyList() + private val networkParameters: NetworkParameters?, + override val references: List>, + val componentGroups: List?, + val resolvedInputBytes: List?, + val resolvedReferenceBytes: List? ) : FullTransaction() { + + @Deprecated("Client code should not instantiate LedgerTransaction.") + constructor( + inputs: List>, + outputs: List>, + commands: List>, + attachments: List, + id: SecureHash, + notary: Party?, + timeWindow: TimeWindow?, + privacySalt: PrivacySalt + ) : this(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, null, emptyList(), null, null, null) + + @Deprecated("Client code should not instantiate LedgerTransaction.") + constructor( + inputs: List>, + outputs: List>, + commands: List>, + attachments: List, + id: SecureHash, + notary: Party?, + timeWindow: TimeWindow?, + privacySalt: PrivacySalt, + networkParameters: NetworkParameters? + ) : this(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, emptyList(), null, null, null) + //DOCEND 1 init { checkBaseInvariants() @@ -58,19 +86,25 @@ data class LedgerTransaction @JvmOverloads constructor( checkEncumbrancesValid() } - private companion object { - val logger = loggerFor() - private fun contractClassFor(className: ContractClassName, classLoader: ClassLoader?): Try> { - return Try.on { - (classLoader ?: this::class.java.classLoader) - .loadClass(className) - .asSubclass(Contract::class.java) - } - } + companion object { + private val logger = loggerFor() - private fun stateToContractClass(state: TransactionState): Try> { - return contractClassFor(state.contract, state.data::class.java.classLoader) - } + @CordaInternal + internal fun makeLedgerTransaction( + inputs: List>, + outputs: List>, + commands: List>, + attachments: List, + id: SecureHash, + notary: Party?, + timeWindow: TimeWindow?, + privacySalt: PrivacySalt, + networkParameters: NetworkParameters?, + references: List>, + componentGroups: List, + resolvedInputBytes: List, + resolvedReferenceBytes: List + ) = LedgerTransaction(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, resolvedInputBytes, resolvedReferenceBytes) } val inputStates: List get() = inputs.map { it.state.data } @@ -88,6 +122,12 @@ data class LedgerTransaction @JvmOverloads constructor( /** * Verifies this transaction and runs contract code. At this stage it is assumed that signatures have already been verified. + + * The contract verification logic is run in a custom [AttachmentsClassLoader] created for the current transaction. + * This classloader is only used during verification and does not leak to the client code. + * + * The reason for this is that classes (contract states) deserialized in this classloader would actually be a different type from what + * the calling code would expect. * * @throws TransactionVerificationException if anything goes wrong. */ @@ -95,12 +135,17 @@ data class LedgerTransaction @JvmOverloads constructor( fun verify() { val contractAttachmentsByContract: Map = getUniqueContractAttachmentsByContract() - // TODO - verify for version downgrade - validatePackageOwnership(contractAttachmentsByContract) - validateStatesAgainstContract() - verifyConstraintsValidity(contractAttachmentsByContract) - verifyConstraints(contractAttachmentsByContract) - verifyContracts() + AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(this.attachments) { transactionClassLoader -> + + val internalTx = createInternalLedgerTransaction() + + // TODO - verify for version downgrade + validatePackageOwnership(contractAttachmentsByContract) + validateStatesAgainstContract(internalTx) + verifyConstraintsValidity(internalTx, contractAttachmentsByContract, transactionClassLoader) + verifyConstraints(internalTx, contractAttachmentsByContract) + verifyContracts(internalTx) + } } /** @@ -133,7 +178,7 @@ data class LedgerTransaction @JvmOverloads constructor( * * A warning will be written to the log if any mismatch is detected. */ - private fun validateStatesAgainstContract() = allStates.forEach(::validateStateAgainstContract) + private fun validateStatesAgainstContract(internalTx: LedgerTransaction) = internalTx.allStates.forEach { validateStateAgainstContract(it) } private fun validateStateAgainstContract(state: TransactionState) { state.data.requiredContractClassName?.let { requiredContractClassName -> @@ -150,25 +195,25 @@ data class LedgerTransaction @JvmOverloads constructor( * * Constraints should be one of the valid supported ones. * * Constraints should propagate correctly if not marked otherwise. */ - private fun verifyConstraintsValidity(contractAttachmentsByContract: Map) { + private fun verifyConstraintsValidity(internalTx: LedgerTransaction, contractAttachmentsByContract: Map, transactionClassLoader: ClassLoader) { // First check that the constraints are valid. - for (state in allStates) { + for (state in internalTx.allStates) { checkConstraintValidity(state) } // Group the inputs and outputs by contract, and for each contract verify the constraints propagation logic. // This is not required for reference states as there is nothing to propagate. - val inputContractGroups = inputs.groupBy { it.state.contract } - val outputContractGroups = outputs.groupBy { it.contract } + val inputContractGroups = internalTx.inputs.groupBy { it.state.contract } + val outputContractGroups = internalTx.outputs.groupBy { it.contract } for (contractClassName in (inputContractGroups.keys + outputContractGroups.keys)) { - if (contractClassName.contractHasAutomaticConstraintPropagation()) { + if (contractClassName.contractHasAutomaticConstraintPropagation(transactionClassLoader)) { // Verify that the constraints of output states have at least the same level of restriction as the constraints of the corresponding input states. val inputConstraints = inputContractGroups[contractClassName]?.map { it.state.constraint }?.toSet() val outputConstraints = outputContractGroups[contractClassName]?.map { it.constraint }?.toSet() outputConstraints?.forEach { outputConstraint -> inputConstraints?.forEach { inputConstraint -> - if (!(outputConstraint.canBeTransitionedFrom(inputConstraint, contractAttachmentsByContract[contractClassName]!! ))) { + if (!(outputConstraint.canBeTransitionedFrom(inputConstraint, contractAttachmentsByContract[contractClassName]!!))) { throw TransactionVerificationException.ConstraintPropagationRejection(id, contractClassName, inputConstraint, outputConstraint) } } @@ -186,8 +231,8 @@ data class LedgerTransaction @JvmOverloads constructor( * * @throws TransactionVerificationException if the constraints fail to verify */ - private fun verifyConstraints(contractAttachmentsByContract: Map) { - for (state in allStates) { + private fun verifyConstraints(internalTx: LedgerTransaction, contractAttachmentsByContract: Map) { + for (state in internalTx.allStates) { val contractAttachment = contractAttachmentsByContract[state.contract] ?: throw TransactionVerificationException.MissingAttachmentRejection(id, state.contract) @@ -226,38 +271,64 @@ data class LedgerTransaction @JvmOverloads constructor( return result } + private fun contractClassFor(className: ContractClassName, classLoader: ClassLoader): Class = try { + classLoader.loadClass(className).asSubclass(Contract::class.java) + } catch (e: Exception) { + throw TransactionVerificationException.ContractCreationError(id, className, e) + } + + private fun createInternalLedgerTransaction(): LedgerTransaction { + return if (resolvedInputBytes != null && resolvedReferenceBytes != null && componentGroups != null) { + + // Deserialize all relevant classes in the transaction classloader. + val resolvedDeserializedInputs = resolvedInputBytes.map { StateAndRef(it.serializedState.deserialize(), it.ref) } + val resolvedDeserializedReferences = resolvedReferenceBytes.map { StateAndRef(it.serializedState.deserialize(), it.ref) } + val deserializedOutputs = deserialiseComponentGroup(componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true) + val deserializedCommands = deserialiseCommands(this.componentGroups, forceDeserialize = true) + val authenticatedArgs = deserializedCommands.map { cmd -> + val parties = commands.find { it.value.javaClass.name == cmd.value.javaClass.name }!!.signingParties + CommandWithParties(cmd.signers, parties, cmd.value) + } + + val ledgerTransactionToVerify = this.copy( + inputs = resolvedDeserializedInputs, + outputs = deserializedOutputs, + commands = authenticatedArgs, + references = resolvedDeserializedReferences) + + ledgerTransactionToVerify + } else { + // This branch is only present for backwards compatibility. + // TODO - it should be removed once the constructor of LedgerTransaction is no longer public api. + logger.warn("The LedgerTransaction should not be instantiated directly from client code. Please use WireTransaction.toLedgerTransaction. The result of the verify method might not be accurate.") + this + } + } + /** * Check the transaction is contract-valid by running the verify() for each input and output state contract. * If any contract fails to verify, the whole transaction is considered to be invalid. */ - private fun verifyContracts() = inputAndOutputStates.forEach { ts -> - val contractClass = getContractClass(ts) - val contract = createContractInstance(contractClass) + private fun verifyContracts(internalTx: LedgerTransaction) { + val contractClasses = (internalTx.inputs.map { it.state } + internalTx.outputs).toSet() + .map { it.contract to contractClassFor(it.contract, it.data.javaClass.classLoader) } - try { - contract.verify(this) - } catch (e: Exception) { - throw TransactionVerificationException.ContractRejection(id, contract, e) - } - } - - // Obtain the contract class from the class name, wrapping any exception as a [ContractCreationError] - private fun getContractClass(ts: TransactionState): Class = - try { - (ts.data::class.java.classLoader ?: this::class.java.classLoader) - .loadClass(ts.contract) - .asSubclass(Contract::class.java) - } catch (e: Exception) { - throw TransactionVerificationException.ContractCreationError(id, ts.contract, e) - } - - // Obtain an instance of the contract class, wrapping any exception as a [ContractCreationError] - private fun createContractInstance(contractClass: Class): Contract = + val contractInstances = contractClasses.map { (contractClassName, contractClass) -> try { contractClass.newInstance() } catch (e: Exception) { - throw TransactionVerificationException.ContractCreationError(id, contractClass.name, e) + throw TransactionVerificationException.ContractCreationError(id, contractClassName, e) } + } + + contractInstances.forEach { contract -> + try { + contract.verify(internalTx) + } catch (e: Exception) { + throw TransactionVerificationException.ContractRejection(id, contract, e) + } + } + } /** * Make sure the notary has stayed the same. As we can't tell how inputs and outputs connect, if there @@ -286,7 +357,8 @@ data class LedgerTransaction @JvmOverloads constructor( // b) the number of outputs can contain the encumbrance // c) the bi-directionality (full cycle) property is satisfied // d) encumbered output states are assigned to the same notary. - val statesAndEncumbrance = outputs.withIndex().filter { it.value.encumbrance != null }.map { Pair(it.index, it.value.encumbrance!!) } + val statesAndEncumbrance = outputs.withIndex().filter { it.value.encumbrance != null } + .map { Pair(it.index, it.value.encumbrance!!) } if (!statesAndEncumbrance.isEmpty()) { checkBidirectionalOutputEncumbrances(statesAndEncumbrance) checkNotariesOutputEncumbrance(statesAndEncumbrance) diff --git a/core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt index 811b3dfab6..e91f867067 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/MerkleTransaction.kt @@ -6,14 +6,14 @@ import net.corda.core.contracts.* import net.corda.core.contracts.ComponentGroupEnum.* import net.corda.core.crypto.* import net.corda.core.identity.Party -import net.corda.core.internal.LazyMappedList -import net.corda.core.internal.uncheckedCast -import net.corda.core.serialization.* +import net.corda.core.internal.deserialiseCommands +import net.corda.core.internal.deserialiseComponentGroup +import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.deserialize import net.corda.core.utilities.OpaqueBytes -import net.corda.core.utilities.lazyMapped import java.security.PublicKey import java.util.function.Predicate -import kotlin.reflect.KClass /** * Implemented by [WireTransaction] and [FilteredTransaction]. A TraversableTransaction allows you to iterate @@ -23,27 +23,27 @@ import kotlin.reflect.KClass */ abstract class TraversableTransaction(open val componentGroups: List) : CoreTransaction() { /** Hashes of the ZIP/JAR files that are needed to interpret the contents of this wire transaction. */ - val attachments: List = deserialiseComponentGroup(SecureHash::class, ATTACHMENTS_GROUP) + val attachments: List = deserialiseComponentGroup(componentGroups, SecureHash::class, ATTACHMENTS_GROUP) /** Pointers to the input states on the ledger, identified by (tx identity hash, output index). */ - override val inputs: List = deserialiseComponentGroup(StateRef::class, INPUTS_GROUP) + override val inputs: List = deserialiseComponentGroup(componentGroups, StateRef::class, INPUTS_GROUP) /** Pointers to reference states, identified by (tx identity hash, output index). */ - override val references: List = deserialiseComponentGroup(StateRef::class, REFERENCES_GROUP) + override val references: List = deserialiseComponentGroup(componentGroups, StateRef::class, REFERENCES_GROUP) - override val outputs: List> = deserialiseComponentGroup(TransactionState::class, OUTPUTS_GROUP, attachmentsContext = true) + override val outputs: List> = deserialiseComponentGroup(componentGroups, TransactionState::class, OUTPUTS_GROUP) /** Ordered list of ([CommandData], [PublicKey]) pairs that instruct the contracts what to do. */ - val commands: List> = deserialiseCommands() + val commands: List> = deserialiseCommands(componentGroups) override val notary: Party? = let { - val notaries: List = deserialiseComponentGroup(Party::class, NOTARY_GROUP) + val notaries: List = deserialiseComponentGroup(componentGroups, Party::class, NOTARY_GROUP) check(notaries.size <= 1) { "Invalid Transaction. More than 1 notary party detected." } notaries.firstOrNull() } val timeWindow: TimeWindow? = let { - val timeWindows: List = deserialiseComponentGroup(TimeWindow::class, TIMEWINDOW_GROUP) + val timeWindows: List = deserialiseComponentGroup(componentGroups, TimeWindow::class, TIMEWINDOW_GROUP) check(timeWindows.size <= 1) { "Invalid Transaction. More than 1 time-window detected." } timeWindows.firstOrNull() } @@ -66,65 +66,6 @@ abstract class TraversableTransaction(open val componentGroups: List deserialiseComponentGroup(clazz: KClass, - groupEnum: ComponentGroupEnum, - attachmentsContext: Boolean = false): List { - val group = componentGroups.firstOrNull { it.groupIndex == groupEnum.ordinal } - - if (group == null || group.components.isEmpty()) { - return emptyList() - } - - // If the componentGroup is a [LazyMappedList] it means that the original deserialized version is already available. - val components = group.components - if (components is LazyMappedList<*, OpaqueBytes>) { - return components.originalList as List - } - - val factory = SerializationFactory.defaultFactory - val context = factory.defaultContext.let { if (attachmentsContext) it.withAttachmentsClassLoader(attachments) else it } - - return components.lazyMapped { component, internalIndex -> - try { - factory.deserialize(component, clazz.java , context) - } catch (e: MissingAttachmentsException) { - throw e - } catch (e: Exception) { - throw Exception("Malformed transaction, $groupEnum at index $internalIndex cannot be deserialised", e) - } - } - } - - // Method to deserialise Commands from its two groups: - // COMMANDS_GROUP which contains the CommandData part - // and SIGNERS_GROUP which contains the Signers part. - private fun deserialiseCommands(): List> { - // TODO: we could avoid deserialising unrelated signers. - // However, current approach ensures the transaction is not malformed - // and it will throw if any of the signers objects is not List of public keys). - val signersList: List> = uncheckedCast(deserialiseComponentGroup(List::class, SIGNERS_GROUP)) - val commandDataList: List = deserialiseComponentGroup(CommandData::class, COMMANDS_GROUP, attachmentsContext = true) - val group = componentGroups.firstOrNull { it.groupIndex == COMMANDS_GROUP.ordinal } - return if (group is FilteredComponentGroup) { - check(commandDataList.size <= signersList.size) { - "Invalid Transaction. Less Signers (${signersList.size}) than CommandData (${commandDataList.size}) objects" - } - val componentHashes = group.components.mapIndexed { index, component -> componentHash(group.nonces[index], component) } - val leafIndices = componentHashes.map { group.partialMerkleTree.leafIndex(it) } - if (leafIndices.isNotEmpty()) - check(leafIndices.max()!! < signersList.size) { "Invalid Transaction. A command with no corresponding signer detected" } - commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[leafIndices[index]]) } - } else { - // It is a WireTransaction - // or a FilteredTransaction with no Commands (in which case group is null). - check(commandDataList.size == signersList.size) { - "Invalid Transaction. Sizes of CommandData (${commandDataList.size}) and Signers (${signersList.size}) do not match" - } - commandDataList.lazyMapped { commandData, index -> Command(commandData, signersList[index]) } - } - } } /** diff --git a/core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt b/core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt index 55f329540d..98c0ae4678 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt @@ -1,5 +1,6 @@ package net.corda.core.transactions +import net.corda.core.CordaInternal import net.corda.core.DeleteForDJVM import net.corda.core.KeepForDJVM import net.corda.core.contracts.* @@ -10,6 +11,7 @@ import net.corda.core.identity.Party import net.corda.core.node.ServiceHub import net.corda.core.node.ServicesForResolution import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.transactions.NotaryChangeWireTransaction.Component.* @@ -75,6 +77,20 @@ data class NotaryChangeWireTransaction( @DeleteForDJVM fun resolve(services: ServiceHub, sigs: List) = resolve(services as ServicesForResolution, sigs) + /** + * This should return a serialized virtual output state, that will be used to verify spending transactions. + * The binary output should not depend on the classpath of the node that is verifying the transaction. + * + * Ideally the serialization engine would support partial deserialization so that only the Notary ( and the encumbrance can be replaced from the binary input state) + * + * + * TODO - currently this uses the main classloader. + */ + @CordaInternal + internal fun resolveOutputComponent(services: ServicesForResolution, stateRef: StateRef): SerializedBytes> { + return services.loadState(stateRef).serialize() + } + enum class Component { INPUTS, NOTARY, NEW_NOTARY } diff --git a/core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt b/core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt index 9e6c0fceda..4013a6b6e8 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt @@ -270,7 +270,7 @@ open class TransactionBuilder @JvmOverloads constructor( } // The final step is to resolve AutomaticPlaceholderConstraint. - val automaticConstraintPropagation = contractClassName.contractHasAutomaticConstraintPropagation(serializationContext?.deserializationClassLoader) + val automaticConstraintPropagation = contractClassName.contractHasAutomaticConstraintPropagation(inputsAndOutputs.first().data::class.java.classLoader) // When automaticConstraintPropagation is disabled for a contract, output states must an explicit Constraint. require(automaticConstraintPropagation) { "Contract $contractClassName was marked with @NoConstraintPropagation, which means the constraint of the output states has to be set explicitly." } diff --git a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt index 6ff8983745..a7ecc86dce 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt @@ -7,11 +7,15 @@ import net.corda.core.contracts.* import net.corda.core.contracts.ComponentGroupEnum.* import net.corda.core.crypto.* import net.corda.core.identity.Party +import net.corda.core.internal.SerializedStateAndRef import net.corda.core.internal.Emoji import net.corda.core.node.NetworkParameters +import net.corda.core.node.ServiceHub import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.utilities.OpaqueBytes import net.corda.core.utilities.lazyMapped @@ -99,7 +103,7 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr return toLedgerTransactionInternal( resolveIdentity = { services.identityService.partyFromKey(it) }, resolveAttachment = { services.attachments.openAttachment(it) }, - resolveStateRef = { services.loadState(it) }, + resolveStateRefComponent = { resolveStateRefBinaryComponent(it, services) }, networkParameters = services.networkParameters ) } @@ -119,13 +123,14 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveStateRef: (StateRef) -> TransactionState<*>?, @Suppress("UNUSED_PARAMETER") resolveContractAttachment: (TransactionState) -> AttachmentId? ): LedgerTransaction { - return toLedgerTransactionInternal(resolveIdentity, resolveAttachment, resolveStateRef, null) + // This reverts to serializing the resolved transaction state. + return toLedgerTransactionInternal(resolveIdentity, resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, null) } private fun toLedgerTransactionInternal( resolveIdentity: (PublicKey) -> Party?, resolveAttachment: (SecureHash) -> Attachment?, - resolveStateRef: (StateRef) -> TransactionState<*>?, + resolveStateRefComponent: (StateRef) -> SerializedBytes>?, networkParameters: NetworkParameters? ): LedgerTransaction { // Look up public keys to authenticated identities. @@ -133,20 +138,38 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr val parties = cmd.signers.mapNotNull { pk -> resolveIdentity(pk) } CommandWithParties(cmd.signers, parties, cmd.value) } - val resolvedInputs = inputs.lazyMapped { ref, _ -> - resolveStateRef(ref)?.let { StateAndRef(it, ref) } ?: throw TransactionResolutionException(ref.txhash) + + val resolvedInputBytes = inputs.map { ref -> + SerializedStateAndRef(resolveStateRefComponent(ref) + ?: throw TransactionResolutionException(ref.txhash), ref) } - val resolvedReferences = references.lazyMapped { ref, _ -> - resolveStateRef(ref)?.let { StateAndRef(it, ref) } ?: throw TransactionResolutionException(ref.txhash) + val resolvedInputs = resolvedInputBytes.lazyMapped { (serialized, ref), _ -> + StateAndRef(serialized.deserialize(), ref) } + + val resolvedReferenceBytes = references.map { ref -> + SerializedStateAndRef(resolveStateRefComponent(ref) + ?: throw TransactionResolutionException(ref.txhash), ref) + } + val resolvedReferences = resolvedReferenceBytes.lazyMapped { (serialized, ref), _ -> + StateAndRef(serialized.deserialize(), ref) + } + val attachments = attachments.lazyMapped { att, _ -> resolveAttachment(att) ?: throw AttachmentResolutionException(att) } - val ltx = LedgerTransaction(resolvedInputs, outputs, authenticatedArgs, attachments, id, notary, timeWindow, privacySalt, networkParameters, resolvedReferences) - checkTransactionSize(ltx, networkParameters?.maxTransactionSize ?: 10485760) + + val ltx = LedgerTransaction.makeLedgerTransaction(resolvedInputs, outputs, authenticatedArgs, attachments, id, notary, timeWindow, privacySalt, networkParameters, resolvedReferences, componentGroups, resolvedInputBytes, resolvedReferenceBytes) + + checkTransactionSize(ltx, networkParameters?.maxTransactionSize ?: DEFAULT_MAX_TX_SIZE) + return ltx } + /** + * Deterministic function that checks if the transaction is below the maximum allowed size. + * It uses the binary representation of transactions. + */ private fun checkTransactionSize(ltx: LedgerTransaction, maxTransactionSize: Int) { var remainingTransactionSize = maxTransactionSize @@ -164,9 +187,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr // it's likely that the same underlying Attachment CorDapp will occur more than once so we dedup on the attachment id. ltx.attachments.distinctBy { it.id }.forEach { minus(it.size) } - // TODO - these can be optimized by creating a LazyStateAndRef class, that just stores (a pointer) the serialized output componentGroup from the previous transaction. - minus(ltx.references.serialize().size) - minus(ltx.inputs.serialize().size) + minus(ltx.resolvedInputBytes!!.sumBy { it.serializedState.size }) + minus(ltx.resolvedReferenceBytes!!.sumBy { it.serializedState.size }) // For Commands and outputs we can use the component groups as they are already serialized. minus(componentGroupSize(COMMANDS_GROUP)) @@ -253,6 +275,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr } companion object { + private const val DEFAULT_MAX_TX_SIZE = 10485760 + /** * Creating list of [ComponentGroup] used in one of the constructors of [WireTransaction] required * for backwards compatibility purposes. @@ -281,6 +305,28 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr if (commands.isNotEmpty()) componentGroupMap.add(ComponentGroup(SIGNERS_GROUP.ordinal, commands.map { it.signers }.lazyMapped(serialize))) return componentGroupMap } + + /** + * This is the main logic that knows how to retrieve the binary representation of [StateRef]s. + * + * For [ContractUpgradeWireTransaction] or [NotaryChangeWireTransaction] it knows how to recreate the output state in the correct classloader independent of the node's classpath. + */ + @CordaInternal + fun resolveStateRefBinaryComponent(stateRef: StateRef, services: ServicesForResolution): SerializedBytes>? { + return if (services is ServiceHub) { + val coreTransaction = services.validatedTransactions.getTransaction(stateRef.txhash)?.coreTransaction + ?: throw TransactionResolutionException(stateRef.txhash) + when (coreTransaction) { + is WireTransaction -> coreTransaction.componentGroups.firstOrNull { it.groupIndex == ComponentGroupEnum.OUTPUTS_GROUP.ordinal }?.components?.get(stateRef.index) as SerializedBytes>? + is ContractUpgradeWireTransaction -> coreTransaction.resolveOutputComponent(services, stateRef) + is NotaryChangeWireTransaction -> coreTransaction.resolveOutputComponent(services, stateRef) + else -> throw UnsupportedOperationException("Attempting to resolve input ${stateRef.index} of a ${coreTransaction.javaClass} transaction. This is not supported.") + } + } else { + // For backwards compatibility revert to using the node classloader. + services.loadState(stateRef).serialize() + } + } } @DeleteForDJVM diff --git a/core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt b/core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt index 8ef7904a75..56a30be9a5 100644 --- a/core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt +++ b/core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt @@ -6,6 +6,7 @@ import net.corda.core.DeleteForDJVM import net.corda.core.KeepForDJVM import net.corda.core.internal.LazyMappedList import net.corda.core.internal.concurrent.get +import net.corda.core.internal.createSimpleCache import net.corda.core.internal.uncheckedCast import net.corda.core.serialization.CordaSerializable import org.slf4j.Logger @@ -149,9 +150,7 @@ fun Future.getOrThrow(timeout: Duration? = null): V = try { fun List.lazyMapped(transform: (T, Int) -> U): List = LazyMappedList(this, transform) private const val MAX_SIZE = 100 -private val warnings = Collections.newSetFromMap(object : LinkedHashMap() { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?) = size > MAX_SIZE -}) +private val warnings = Collections.newSetFromMap(createSimpleCache(MAX_SIZE)) /** * Utility to help log a warning message only once. @@ -163,4 +162,4 @@ fun Logger.warnOnce(warning: String) { warnings.add(warning) this.warn(warning) } -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt b/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt index e2a2e22b71..b7c5b8a4cd 100644 --- a/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt +++ b/core/src/test/kotlin/net/corda/core/flows/AttachmentTests.kt @@ -14,18 +14,13 @@ import net.corda.core.internal.FetchAttachmentsFlow import net.corda.core.internal.FetchDataFlow import net.corda.core.internal.hash import net.corda.node.services.persistence.NodeAttachmentService -import net.corda.testing.core.ALICE_NAME -import net.corda.testing.core.BOB_NAME -import net.corda.testing.core.makeUnique -import net.corda.testing.core.singleIdentity +import net.corda.testing.core.* +import net.corda.testing.internal.fakeAttachment import net.corda.testing.node.internal.InternalMockNetwork import net.corda.testing.node.internal.InternalMockNodeParameters import net.corda.testing.node.internal.TestStartedNode import org.junit.AfterClass import org.junit.Test -import java.io.ByteArrayOutputStream -import java.util.jar.JarOutputStream -import java.util.zip.ZipEntry class AttachmentTests : WithMockNet { companion object { @@ -46,7 +41,7 @@ class AttachmentTests : WithMockNet { @Test fun `download and store`() { // Insert an attachment into node zero's store directly. - val id = aliceNode.importAttachment(fakeAttachment()) + val id = aliceNode.importAttachment(fakeAttachment("file1.txt", "Some useful content")) // Get node one to run a flow to fetch it and insert it. assert.that( @@ -87,7 +82,7 @@ class AttachmentTests : WithMockNet { val badAlice = badAliceNode.info.singleIdentity() // Insert an attachment into node zero's store directly. - val attachment = fakeAttachment() + val attachment = fakeAttachment("file1.txt", "Some useful content") val id = badAliceNode.importAttachment(attachment) // Corrupt its store. @@ -134,18 +129,6 @@ class AttachmentTests : WithMockNet { } }).apply { registerInitiatedFlow(FetchAttachmentsResponse::class.java) } - private fun fakeAttachment(): ByteArray = - ByteArrayOutputStream().use { baos -> - JarOutputStream(baos).use { jos -> - jos.putNextEntry(ZipEntry("file1.txt")) - jos.writer().apply { - append("Some useful content") - flush() - } - jos.closeEntry() - } - baos.toByteArray() - } //endregion //region Operations diff --git a/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderSerializationTests.kt b/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderSerializationTests.kt new file mode 100644 index 0000000000..c3364eeba3 --- /dev/null +++ b/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderSerializationTests.kt @@ -0,0 +1,98 @@ +package net.corda.core.transactions + +import net.corda.core.contracts.Contract +import net.corda.core.crypto.SecureHash +import net.corda.core.identity.CordaX500Name +import net.corda.core.internal.declaredField +import net.corda.core.serialization.deserialize +import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder +import net.corda.core.serialization.serialize +import net.corda.core.utilities.ByteSequence +import net.corda.core.utilities.OpaqueBytes +import net.corda.nodeapi.DummyContractBackdoor +import net.corda.testing.core.DUMMY_NOTARY_NAME +import net.corda.testing.core.SerializationEnvironmentRule +import net.corda.testing.core.TestIdentity +import net.corda.testing.internal.fakeAttachment +import net.corda.testing.services.MockAttachmentStorage +import org.apache.commons.io.IOUtils +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import java.io.NotSerializableException +import java.net.URL +import kotlin.test.assertFailsWith + +class AttachmentsClassLoaderSerializationTests { + + companion object { + val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderSerializationTests::class.java.getResource("isolated.jar") + private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract" + } + + @Rule + @JvmField + val testSerialization = SerializationEnvironmentRule() + + val storage = MockAttachmentStorage() + + @Test + fun `Can serialize and deserialize with an attachment classloader`() { + + val DUMMY_NOTARY = TestIdentity(DUMMY_NOTARY_NAME, 20).party + val MEGA_CORP = TestIdentity(CordaX500Name("MegaCorp", "London", "GB")).party + + val isolatedId = storage.importAttachment(ISOLATED_CONTRACTS_JAR_PATH.openStream(), "app", "isolated.jar") + val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar") + val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream(), "app", "file2.jar") + + val serialisedState = AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! }) { classLoader -> + val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader) + val contract = contractClass.newInstance() as Contract + assertEquals("helloworld", contract.declaredField("magicString").value) + + val txt = IOUtils.toString(classLoader.getResourceAsStream("file1.txt"), Charsets.UTF_8.name()) + assertEquals("some data", txt) + + val state = (contract as DummyContractBackdoor).generateInitial(MEGA_CORP.ref(1), 1, DUMMY_NOTARY).outputStates().first() + val serialisedState = state.serialize() + + val state1 = serialisedState.deserialize() + assertEquals(state, state1) + serialisedState + } + + assertFailsWith { + serialisedState.deserialize() + } + } + + // These tests are not Attachment specific. Should they be removed? + @Test + fun `test serialization of SecureHash`() { + val secureHash = SecureHash.randomSHA256() + val bytes = secureHash.serialize() + val copiedSecuredHash = bytes.deserialize() + + assertEquals(secureHash, copiedSecuredHash) + } + + @Test + fun `test serialization of OpaqueBytes`() { + val opaqueBytes = OpaqueBytes("0123456789".toByteArray()) + val bytes = opaqueBytes.serialize() + val copiedOpaqueBytes = bytes.deserialize() + + assertEquals(opaqueBytes, copiedOpaqueBytes) + } + + @Test + fun `test serialization of sub-sequence OpaqueBytes`() { + val bytesSequence = ByteSequence.of("0123456789".toByteArray(), 3, 2) + val bytes = bytesSequence.serialize() + val copiedBytesSequence = bytes.deserialize() + + assertEquals(bytesSequence, copiedBytesSequence) + } +} + diff --git a/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderTests.kt b/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderTests.kt new file mode 100644 index 0000000000..0fe3727df8 --- /dev/null +++ b/core/src/test/kotlin/net/corda/core/transactions/AttachmentsClassLoaderTests.kt @@ -0,0 +1,102 @@ +package net.corda.core.transactions + +import net.corda.core.contracts.Attachment +import net.corda.core.contracts.Contract +import net.corda.core.internal.declaredField +import net.corda.core.serialization.internal.AttachmentsClassLoader +import net.corda.testing.internal.fakeAttachment +import net.corda.testing.services.MockAttachmentStorage +import org.apache.commons.io.IOUtils +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Test +import java.io.ByteArrayOutputStream +import java.net.URL +import kotlin.test.assertFailsWith + +class AttachmentsClassLoaderTests { + + companion object { + val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderTests::class.java.getResource("isolated.jar") + private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract" + + private fun readAttachment(attachment: Attachment, filepath: String): ByteArray { + ByteArrayOutputStream().use { + attachment.extractFile(filepath, it) + return it.toByteArray() + } + } + } + + val storage = MockAttachmentStorage() + + @Test + fun `Loading AnotherDummyContract without using the AttachmentsClassLoader fails`() { + assertFailsWith { + Class.forName(ISOLATED_CONTRACT_CLASS_NAME) + } + } + + @Test + fun `Dynamically load AnotherDummyContract from isolated contracts jar using the AttachmentsClassLoader`() { + val isolatedId = storage.importAttachment(ISOLATED_CONTRACTS_JAR_PATH.openStream(), "app", "isolated.jar") + + val classloader = AttachmentsClassLoader(listOf(storage.openAttachment(isolatedId)!!)) + val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classloader) + val contract = contractClass.newInstance() as Contract + assertEquals("helloworld", contract.declaredField("magicString").value) + } + + @Test + fun `Load text resources from AttachmentsClassLoader`() { + val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar") + val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream(), "app", "file2.jar") + + val cl = AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! }) + val txt = IOUtils.toString(cl.getResourceAsStream("file1.txt"), Charsets.UTF_8.name()) + assertEquals("some data", txt) + + val txt1 = IOUtils.toString(cl.getResourceAsStream("file2.txt"), Charsets.UTF_8.name()) + assertEquals("some other data", txt1) + } + + @Test + fun `Test overlapping file exception`() { + val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream(), "app", "file1.jar") + val att2 = storage.importAttachment(fakeAttachment("file1.txt", "some other data").inputStream(), "app", "file2.jar") + + assertFailsWith(AttachmentsClassLoader.Companion.OverlappingAttachments::class) { + AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! }) + } + } + + @Test + fun `No overlapping exception thrown on certain META-INF files`() { + listOf("meta-inf/manifest.mf", "meta-inf/license", "meta-inf/test.dsa", "meta-inf/test.sf").forEach { path -> + val att1 = storage.importAttachment(fakeAttachment(path, "some data").inputStream(), "app", "file1.jar") + val att2 = storage.importAttachment(fakeAttachment(path, "some other data").inputStream(), "app", "file2.jar") + + AttachmentsClassLoader(arrayOf(att1, att2).map { storage.openAttachment(it)!! }) + } + } + + @Test + fun `Check platform independent path handling in attachment jars`() { + val storage = MockAttachmentStorage() + + val att1 = storage.importAttachment(fakeAttachment("/folder1/foldera/file1.txt", "some data").inputStream(), "app", "file1.jar") + val att2 = storage.importAttachment(fakeAttachment("\\folder1\\folderb\\file2.txt", "some other data").inputStream(), "app", "file2.jar") + + val data1a = readAttachment(storage.openAttachment(att1)!!, "/folder1/foldera/file1.txt") + assertArrayEquals("some data".toByteArray(), data1a) + + val data1b = readAttachment(storage.openAttachment(att1)!!, "\\folder1\\foldera\\file1.txt") + assertArrayEquals("some data".toByteArray(), data1b) + + val data2a = readAttachment(storage.openAttachment(att2)!!, "\\folder1\\folderb\\file2.txt") + assertArrayEquals("some other data".toByteArray(), data2a) + + val data2b = readAttachment(storage.openAttachment(att2)!!, "/folder1/folderb/file2.txt") + assertArrayEquals("some other data".toByteArray(), data2b) + } +} diff --git a/core/src/test/kotlin/net/corda/core/transactions/TransactionTests.kt b/core/src/test/kotlin/net/corda/core/transactions/TransactionTests.kt index 32f9105a1c..b37a34fc97 100644 --- a/core/src/test/kotlin/net/corda/core/transactions/TransactionTests.kt +++ b/core/src/test/kotlin/net/corda/core/transactions/TransactionTests.kt @@ -10,6 +10,7 @@ import net.corda.testing.common.internal.testNetworkParameters import net.corda.testing.contracts.DummyContract import net.corda.testing.core.* import net.corda.testing.internal.createWireTransaction +import net.corda.testing.internal.fakeAttachment import net.corda.testing.internal.rigorousMock import org.junit.Rule import org.junit.Test @@ -118,7 +119,8 @@ class TransactionTests { val commands = emptyList>() val attachments = listOf(ContractAttachment(rigorousMock().also { doReturn(SecureHash.zeroHash).whenever(it).id - }, DummyContract.PROGRAM_ID)) + doReturn(fakeAttachment("nothing", "nada").inputStream()).whenever(it).open() + }, DummyContract.PROGRAM_ID, uploader = "app")) val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() diff --git a/serialization/src/test/kotlin/net/corda/nodeapi/DummyContractBackdoor.kt b/core/src/test/kotlin/net/corda/nodeapi/DummyContractBackdoor.kt similarity index 100% rename from serialization/src/test/kotlin/net/corda/nodeapi/DummyContractBackdoor.kt rename to core/src/test/kotlin/net/corda/nodeapi/DummyContractBackdoor.kt diff --git a/core/src/test/resources/net/corda/core/transactions/isolated.jar b/core/src/test/resources/net/corda/core/transactions/isolated.jar new file mode 100644 index 0000000000..17bf0c2436 Binary files /dev/null and b/core/src/test/resources/net/corda/core/transactions/isolated.jar differ diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 534607d95a..602d57ab8a 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,11 @@ release, see :doc:`upgrade-notes`. Unreleased ---------- +* Deprecated `SerializationContext.withAttachmentsClassLoader`. This functionality has always been disabled by flags +and there is no reason for a CorDapp developer to use it. It is just an internal implementation detail of Corda. + +* Deprecated the `LedgerTransaction` constructor. No client code should call it directly. LedgerTransactions can be created from WireTransactions if required. + * Introduced new optional network bootstrapper command line options (--register-package-owner, --unregister-package-owner) to register/unregister a java package namespace with an associated owner in the network parameter packageOwnership whitelist. diff --git a/finance/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt b/finance/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt index e9e1ebf0d4..4b1bd358ef 100644 --- a/finance/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt +++ b/finance/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt @@ -25,6 +25,7 @@ import net.corda.testing.contracts.DummyState import net.corda.testing.core.* import net.corda.testing.dsl.* import net.corda.testing.internal.TEST_TX_TIME +import net.corda.testing.internal.fakeAttachment import net.corda.testing.internal.rigorousMock import net.corda.testing.internal.vault.CommodityState import net.corda.testing.node.MockServices @@ -565,7 +566,7 @@ class ObligationTests { @Test fun `commodity settlement`() { - val commodityContractBytes = "https://www.big-book-of-banking-law.gov/commodity-claims.html".toByteArray() + val commodityContractBytes = fakeAttachment("file1.txt", "https://www.big-book-of-banking-law.gov/commodity-claims.html") val defaultFcoj = Issued(defaultIssuer, Commodity.getInstance("FCOJ")!!) val oneUnitFcoj = Amount(1, defaultFcoj) val obligationDef = Obligation.Terms(NonEmptySet.of(commodityContractBytes.sha256() as SecureHash), NonEmptySet.of(defaultFcoj), TEST_TX_TIME) @@ -957,7 +958,7 @@ class ObligationTests { assertEquals(expected, actual) } - private val cashContractBytes = "https://www.big-book-of-banking-law.gov/cash-claims.html".toByteArray() + private val cashContractBytes = fakeAttachment("file1.txt", "https://www.big-book-of-banking-law.gov/cash-claims.html") private val Issued.OBLIGATION_DEF: Obligation.Terms get() = Obligation.Terms(NonEmptySet.of(cashContractBytes.sha256() as SecureHash), NonEmptySet.of(this), TEST_TX_TIME) private val Amount>.OBLIGATION: Obligation.State diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index ad86bee392..0ca39f664c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -61,7 +61,8 @@ class CordaPersistence( schemas: Set, val jdbcUrl: String, cacheFactory: NamedCacheFactory, - attributeConverters: Collection> = emptySet() + attributeConverters: Collection> = emptySet(), + customClassLoader: ClassLoader? = null ) : Closeable { companion object { private val log = contextLogger() @@ -70,7 +71,7 @@ class CordaPersistence( private val defaultIsolationLevel = databaseConfig.transactionIsolationLevel val hibernateConfig: HibernateConfiguration by lazy { transaction { - HibernateConfiguration(schemas, databaseConfig, attributeConverters, jdbcUrl, cacheFactory) + HibernateConfiguration(schemas, databaseConfig, attributeConverters, jdbcUrl, cacheFactory, customClassLoader) } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index ae371ca88e..fea52e35f5 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -32,7 +32,7 @@ class HibernateConfiguration( private val attributeConverters: Collection>, private val jdbcUrl: String, cacheFactory: NamedCacheFactory, - val cordappClassLoader: ClassLoader? = null + val customClassLoader: ClassLoader? = null ) { companion object { private val logger = contextLogger() @@ -86,7 +86,7 @@ class HibernateConfiguration( schema.mappedTypes.forEach { config.addAnnotatedClass(it) } } - val sessionFactory = buildSessionFactory(config, metadataSources, cordappClassLoader) + val sessionFactory = buildSessionFactory(config, metadataSources, customClassLoader) logger.info("Created session factory for schemas: $schemas") // export Hibernate JMX statistics @@ -112,13 +112,13 @@ class HibernateConfiguration( } } - private fun buildSessionFactory(config: Configuration, metadataSources: MetadataSources, cordappClassLoader: ClassLoader?): SessionFactory { + private fun buildSessionFactory(config: Configuration, metadataSources: MetadataSources, customClassLoader: ClassLoader?): SessionFactory { config.standardServiceRegistryBuilder.applySettings(config.properties) - if (cordappClassLoader != null) { + if (customClassLoader != null) { config.standardServiceRegistryBuilder.addService( ClassLoaderService::class.java, - ClassLoaderServiceImpl(cordappClassLoader)) + ClassLoaderServiceImpl(customClassLoader)) } val metadataBuilder = metadataSources.getMetadataBuilder(config.standardServiceRegistryBuilder.build()) diff --git a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/LargeTransactionsTest.kt b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/LargeTransactionsTest.kt index 1116ed8c29..59d1d457ce 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/statemachine/LargeTransactionsTest.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/statemachine/LargeTransactionsTest.kt @@ -69,10 +69,10 @@ class LargeTransactionsTest { fun checkCanSendLargeTransactions() { // These 4 attachments yield a transaction that's got >10mb attached, so it'd push us over the Artemis // max message size. - val bigFile1 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 0) - val bigFile2 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 1) - val bigFile3 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 2) - val bigFile4 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 3) + val bigFile1 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 0, "a") + val bigFile2 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 1, "b") + val bigFile3 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 2, "c") + val bigFile4 = InputStreamAndHash.createInMemoryTestZip(3.MB.toInt(), 3, "d") driver(DriverParameters( startNodesInProcess = true, extraCordappPackagesToScan = listOf("net.corda.testing.contracts"), diff --git a/node/src/main/java/CordaCaplet.java b/node/src/main/java/CordaCaplet.java index 78ccce99af..129b1e7dee 100644 --- a/node/src/main/java/CordaCaplet.java +++ b/node/src/main/java/CordaCaplet.java @@ -112,8 +112,6 @@ public class CordaCaplet extends Capsule { // If it fails, just return the existing class path. The main Corda jar will detect the error and fail gracefully. return cp; } - // Add additional directories of JARs to the classpath (at the end), e.g., for JDBC drivers. - augmentClasspath((List) cp, cordappsDir); try { List jarDirs = nodeConfig.getStringList("jarDirs"); log(LOG_VERBOSE, "Configured JAR directories = " + jarDirs); diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index e6669e14ec..bb788f2a72 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -155,7 +155,8 @@ abstract class AbstractNode(val configuration: NodeConfiguration, identityService::wellKnownPartyFromAnonymous, schemaService, configuration.dataSourceProperties, - cacheFactory) + cacheFactory, + this.cordappLoader.appClassLoader) init { // TODO Break cyclic dependency @@ -748,7 +749,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, protected open fun startDatabase() { val props = configuration.dataSourceProperties if (props.isEmpty) throw DatabaseConfigurationException("There must be a database configured.") - database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry) + database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry, this.cordappLoader.appClassLoader) // Now log the vendor string as this will also cause a connection to be tested eagerly. logVendorString(database, log) } @@ -1061,7 +1062,8 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, schemaService: SchemaService, hikariProperties: Properties, - cacheFactory: NamedCacheFactory): CordaPersistence { + cacheFactory: NamedCacheFactory, + customClassLoader: ClassLoader?): CordaPersistence { // Register the AbstractPartyDescriptor so Hibernate doesn't warn when encountering AbstractParty. Unfortunately // Hibernate warns about not being able to find a descriptor if we don't provide one, but won't use it by default // so we end up providing both descriptor and converter. We should re-examine this in later versions to see if @@ -1069,13 +1071,13 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig, JavaTypeDescriptorRegistry.INSTANCE.addDescriptor(AbstractPartyDescriptor(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) val attributeConverters = listOf(PublicKeyToTextConverter(), AbstractPartyToX500NameAsStringConverter(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) val jdbcUrl = hikariProperties.getProperty("dataSource.url", "") - return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters) + return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters, customClassLoader) } -fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set, metricRegistry: MetricRegistry? = null) { +fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set, metricRegistry: MetricRegistry? = null, classloader: ClassLoader = Thread.currentThread().contextClassLoader) { try { val dataSource = DataSourceFactory.createDataSource(hikariProperties, metricRegistry = metricRegistry) - val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig) + val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig, classloader) schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) start(dataSource) } catch (ex: Exception) { diff --git a/node/src/main/kotlin/net/corda/node/serialization/kryo/CordaClassResolver.kt b/node/src/main/kotlin/net/corda/node/serialization/kryo/CordaClassResolver.kt index e3ff2584f7..87183504e8 100644 --- a/node/src/main/kotlin/net/corda/node/serialization/kryo/CordaClassResolver.kt +++ b/node/src/main/kotlin/net/corda/node/serialization/kryo/CordaClassResolver.kt @@ -11,7 +11,7 @@ import net.corda.core.internal.writer import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.ClassWhitelist import net.corda.core.utilities.contextLogger -import net.corda.serialization.internal.AttachmentsClassLoader +import net.corda.core.serialization.internal.AttachmentsClassLoader import net.corda.serialization.internal.MutableClassWhitelist import net.corda.serialization.internal.TransientClassWhiteList import net.corda.serialization.internal.amqp.hasCordaSerializable diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt b/node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt index 5f2b1a71be..88be7c6e8b 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt @@ -229,7 +229,7 @@ class NodeAttachmentService( val attachmentImpl = AttachmentImpl(id, { attachment.content }, checkAttachmentsOnLoad).let { val contracts = attachment.contractClassNames if (contracts != null && contracts.isNotEmpty()) { - ContractAttachment(it, contracts.first(), contracts.drop(1).toSet(), attachment.uploader, attachment.signers + ContractAttachment(it, contracts.first(), contracts.drop(1).toSet(), attachment.uploader, attachment.signers?.toList() ?: emptyList()) } else { it diff --git a/node/src/test/kotlin/net/corda/node/services/transactions/MaxTransactionSizeTests.kt b/node/src/test/kotlin/net/corda/node/services/transactions/MaxTransactionSizeTests.kt index 28db2d7d13..19d8693539 100644 --- a/node/src/test/kotlin/net/corda/node/services/transactions/MaxTransactionSizeTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/transactions/MaxTransactionSizeTests.kt @@ -54,10 +54,10 @@ class MaxTransactionSizeTests { @Test fun `check transaction will fail when exceed max transaction size limit`() { // These 4 attachments yield a transaction that's got ~ 4mb, which will exceed the 3mb max transaction size limit - val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0) - val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1) - val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2) - val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3) + val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0, "a") + val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1, "b") + val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2, "c") + val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3, "d") val flow = aliceNode.transaction { val hash1 = aliceNode.importAttachment(bigFile1.inputStream) val hash2 = aliceNode.importAttachment(bigFile2.inputStream) @@ -77,10 +77,10 @@ class MaxTransactionSizeTests { @Test fun `check transaction will be rejected by counterparty when exceed max transaction size limit`() { // These 4 attachments yield a transaction that's got ~ 4mb, which will exceed the 3mb max transaction size limit - val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0) - val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1) - val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2) - val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3) + val bigFile1 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 0, "a") + val bigFile2 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 1, "b") + val bigFile3 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 2, "c") + val bigFile4 = InputStreamAndHash.createInMemoryTestZip(1024 * 1024, 3, "c") val flow = aliceNode.transaction { val hash1 = aliceNode.importAttachment(bigFile1.inputStream) val hash2 = aliceNode.importAttachment(bigFile2.inputStream) diff --git a/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderBuilder.kt b/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderBuilder.kt deleted file mode 100644 index 7f77351952..0000000000 --- a/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderBuilder.kt +++ /dev/null @@ -1,12 +0,0 @@ -package net.corda.serialization.internal - -import net.corda.core.crypto.SecureHash - -/** - * Drop-in replacement for [AttachmentsClassLoaderBuilder] in the serialization module. - * This version is not strongly-coupled to [net.corda.core.node.ServiceHub]. - */ -@Suppress("UNUSED", "UNUSED_PARAMETER") -internal class AttachmentsClassLoaderBuilder() { - fun build(attachmentHashes: List, properties: Map, deserializationClassLoader: ClassLoader): AttachmentsClassLoader? = null -} \ No newline at end of file diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoader.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoader.kt deleted file mode 100644 index 79de2f342b..0000000000 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/AttachmentsClassLoader.kt +++ /dev/null @@ -1,115 +0,0 @@ -package net.corda.serialization.internal - -import net.corda.core.KeepForDJVM -import net.corda.core.contracts.Attachment -import net.corda.core.contracts.ContractAttachment -import net.corda.core.crypto.SecureHash -import net.corda.core.internal.isUploaderTrusted -import net.corda.core.serialization.CordaSerializable -import java.io.ByteArrayOutputStream -import java.io.FileNotFoundException -import java.io.InputStream -import java.net.URL -import java.net.URLConnection -import java.net.URLStreamHandler -import java.security.CodeSigner -import java.security.CodeSource -import java.security.SecureClassLoader -import java.util.* - -/** - * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only - * need to provide JAR streams, and so could be fetched from a database, local disk, etc. Constructing an - * AttachmentsClassLoader is somewhat expensive, as every attachment is scanned to ensure that there are no overlapping - * file paths. - */ -@KeepForDJVM -class AttachmentsClassLoader(attachments: List, parent: ClassLoader = ClassLoader.getSystemClassLoader()) : SecureClassLoader(parent) { - private val pathsToAttachments = HashMap() - private val idsToAttachments = HashMap() - - @CordaSerializable - class OverlappingAttachments(val path: String) : Exception() { - override fun toString() = "Multiple attachments define a file at path $path" - } - - init { - require(attachments.mapNotNull { it as? ContractAttachment }.all { isUploaderTrusted(it.uploader) }) { - "Attempting to load Contract Attachments downloaded from the network" - } - - for (attachment in attachments) { - attachment.openAsJAR().use { jar -> - while (true) { - val entry = jar.nextJarEntry ?: break - - // We already verified that paths are not strange/game playing when we inserted the attachment - // into the storage service. So we don't need to repeat it here. - // - // We forbid files that differ only in case, or path separator to avoid issues for Windows/Mac developers where the - // filesystem tries to be case insensitive. This may break developers who attempt to use ProGuard. - // - // Also convert to Unix path separators as all resource/class lookups will expect this. - val path = entry.name.toLowerCase().replace('\\', '/') - if (path in pathsToAttachments) - throw OverlappingAttachments(path) - pathsToAttachments[path] = attachment - } - } - idsToAttachments[attachment.id] = attachment - } - } - - // Example: attachment://0b4fc1327f3bbebf1bfe98330ea402ae035936c3cb6da9bd3e26eeaa9584e74d/some/file.txt - // - // We have to provide a fake stream handler to satisfy the URL class that the scheme is known. But it's not - // a real scheme and we don't register it. It's just here to ensure that there aren't codepaths that could - // lead to data loading that we don't control right here in this class (URLs can have evil security properties!) - private val fakeStreamHandler = object : URLStreamHandler() { - override fun openConnection(u: URL?): URLConnection? { - throw UnsupportedOperationException() - } - } - - private fun Attachment.toURL(path: String?) = URL(null, "attachment://$id/" + (path ?: ""), fakeStreamHandler) - - override fun findClass(name: String): Class<*> { - val path = name.replace('.', '/').toLowerCase() + ".class" - val attachment = pathsToAttachments[path] ?: throw ClassNotFoundException(name) - val stream = ByteArrayOutputStream() - try { - attachment.extractFile(path, stream) - } catch (e: FileNotFoundException) { - throw ClassNotFoundException(name) - } - val bytes = stream.toByteArray() - // We don't attempt to propagate signatures from the JAR into the codesource, because our sandbox does not - // depend on external policy files to specify what it can do, so the data wouldn't be useful. - val codesource = CodeSource(attachment.toURL(null), emptyArray()) - // TODO: Define an empty ProtectionDomain to start enforcing the standard Java sandbox. - // The standard Java sandbox is insufficient for our needs and a much more sophisticated sandboxing - // ClassLoader will appear here in future, but it can't hurt to use the default one too: defence in depth! - return defineClass(name, bytes, 0, bytes.size, codesource) - } - - override fun findResource(name: String): URL? { - val attachment = pathsToAttachments[name.toLowerCase()] ?: return null - return attachment.toURL(name) - } - - override fun getResourceAsStream(name: String): InputStream? { - val url = getResource(name) ?: return null // May check parent classloaders, for example. - if (url.protocol != "attachment") return null - val attachment = idsToAttachments[SecureHash.parse(url.host)] ?: return null - val path = url.path?.substring(1) ?: return null // Chop off the leading slash. - return try { - val stream = ByteArrayOutputStream() - attachment.extractFile(path, stream) - stream.toByteArray().inputStream() - } catch (e: FileNotFoundException) { - null - } - } -} - - diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt index 12519312e9..83ec6971ce 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/CheckpointSerializationScheme.kt @@ -16,15 +16,6 @@ data class CheckpointSerializationContextImpl @JvmOverloads constructor( override val objectReferencesEnabled: Boolean, override val encoding: SerializationEncoding?, override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist) : CheckpointSerializationContext { - /** - * {@inheritDoc} - * - * Unsupported for checkpoints. - */ - override fun withAttachmentsClassLoader(attachmentHashes: List): CheckpointSerializationContext { - throw UnsupportedOperationException() - } - override fun withProperty(property: Any, value: Any): CheckpointSerializationContext { return copy(properties = properties + (property to value)) } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt index ed5aecd987..5f4b1127c3 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt @@ -8,6 +8,7 @@ import net.corda.core.contracts.Attachment import net.corda.core.crypto.SecureHash import net.corda.core.internal.copyBytes import net.corda.core.serialization.* +import net.corda.core.serialization.internal.AttachmentsClassLoader import net.corda.core.utilities.ByteSequence import net.corda.serialization.internal.amqp.amqpMagic import org.slf4j.LoggerFactory @@ -31,20 +32,12 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe override val useCase: SerializationContext.UseCase, override val encoding: SerializationEncoding?, override val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist, - override val lenientCarpenterEnabled: Boolean = false, - private val builder: AttachmentsClassLoaderBuilder = AttachmentsClassLoaderBuilder() -) : SerializationContext { - - + override val lenientCarpenterEnabled: Boolean = false) : SerializationContext { /** * {@inheritDoc} - * - * We need to cache the AttachmentClassLoaders to avoid too many contexts, since the class loader is part of cache key for the context. */ override fun withAttachmentsClassLoader(attachmentHashes: List): SerializationContext { - properties[attachmentsClassLoaderEnabledPropertyName] as? Boolean == true || return this - val classLoader = builder.build(attachmentHashes, properties, deserializationClassLoader) ?: return this - return withClassLoader(classLoader) + return this } override fun withProperty(property: Any, value: Any): SerializationContext { @@ -72,34 +65,6 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe override fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist) = copy(encodingWhitelist = encodingWhitelist) } -/* - * This class is internal rather than private so that serialization-deterministic - * can replace it with an alternative version. - */ -@DeleteForDJVM -class AttachmentsClassLoaderBuilder() { - private val cache: Cache, ClassLoader>, AttachmentsClassLoader> = Caffeine.newBuilder().weakValues().maximumSize(1024).build() - - fun build(attachmentHashes: List, properties: Map, deserializationClassLoader: ClassLoader): AttachmentsClassLoader? { - val serializationContext = properties[serializationContextKey] as? SerializeAsTokenContext ?: return null // Some tests don't set one. - try { - return cache.get(Pair(attachmentHashes, deserializationClassLoader)) { - val missing = ArrayList() - val attachments = ArrayList() - attachmentHashes.forEach { id -> - serializationContext.serviceHub.attachments.openAttachment(id)?.let { attachments += it } - ?: run { missing += id } - } - missing.isNotEmpty() && throw MissingAttachmentsException(missing) - AttachmentsClassLoader(attachments, parent = deserializationClassLoader) - }!! - } catch (e: ExecutionException) { - // Caught from within the cache get, so unwrap. - throw e.cause!! - } - } -} - @KeepForDJVM open class SerializationFactoryImpl( // TODO: This is read-mostly. Probably a faster implementation to be found. diff --git a/serialization/src/test/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderTests.kt b/serialization/src/test/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderTests.kt deleted file mode 100644 index b428fdd811..0000000000 --- a/serialization/src/test/kotlin/net/corda/serialization/internal/AttachmentsClassLoaderTests.kt +++ /dev/null @@ -1,382 +0,0 @@ -package net.corda.serialization.internal - -import com.nhaarman.mockito_kotlin.doReturn -import com.nhaarman.mockito_kotlin.whenever -import net.corda.core.contracts.Attachment -import net.corda.core.contracts.Contract -import net.corda.core.crypto.SecureHash -import net.corda.core.identity.CordaX500Name -import net.corda.core.internal.declaredField -import net.corda.core.internal.toWireTransaction -import net.corda.core.node.ServiceHub -import net.corda.core.node.services.AttachmentStorage -import net.corda.core.serialization.* -import net.corda.core.utilities.ByteSequence -import net.corda.core.utilities.OpaqueBytes -import net.corda.node.internal.cordapp.JarScanningCordappLoader -import net.corda.node.internal.cordapp.CordappProviderImpl -import net.corda.nodeapi.DummyContractBackdoor -import net.corda.testing.common.internal.testNetworkParameters -import net.corda.testing.core.DUMMY_NOTARY_NAME -import net.corda.testing.core.SerializationEnvironmentRule -import net.corda.testing.core.TestIdentity -import net.corda.testing.internal.MockCordappConfigProvider -import net.corda.testing.internal.kryoSpecific -import net.corda.testing.internal.rigorousMock -import net.corda.testing.services.MockAttachmentStorage -import org.apache.commons.io.IOUtils -import org.junit.Assert.* -import org.junit.Rule -import org.junit.Test -import java.io.ByteArrayOutputStream -import java.net.URL -import java.net.URLClassLoader -import java.util.jar.JarOutputStream -import java.util.zip.ZipEntry -import kotlin.test.assertFailsWith - -class AttachmentsClassLoaderTests { - companion object { - val ISOLATED_CONTRACTS_JAR_PATH: URL = AttachmentsClassLoaderTests::class.java.getResource("isolated.jar") - private const val ISOLATED_CONTRACT_CLASS_NAME = "net.corda.finance.contracts.isolated.AnotherDummyContract" - private val DUMMY_NOTARY = TestIdentity(DUMMY_NOTARY_NAME, 20).party - private val MEGA_CORP = TestIdentity(CordaX500Name("MegaCorp", "London", "GB")).party - private fun SerializationContext.withAttachmentStorage(attachmentStorage: AttachmentStorage): SerializationContext { - val serviceHub = rigorousMock() - doReturn(attachmentStorage).whenever(serviceHub).attachments - return this.withServiceHub(serviceHub) - } - - private fun SerializationContext.withServiceHub(serviceHub: ServiceHub): SerializationContext { - return this.withTokenContext(SerializeAsTokenContextImpl(serviceHub) {}).withProperty(attachmentsClassLoaderEnabledPropertyName, true) - } - } - - @Rule - @JvmField - val testSerialization = SerializationEnvironmentRule() - private val attachments = MockAttachmentStorage() - private val networkParameters = testNetworkParameters() - private val cordappProvider = CordappProviderImpl(JarScanningCordappLoader.fromJarUrls(listOf(ISOLATED_CONTRACTS_JAR_PATH)), MockCordappConfigProvider(), attachments).apply { - start(networkParameters.whitelistedContractImplementations) - } - private val cordapp get() = cordappProvider.cordapps.first() - private val attachmentId get() = cordappProvider.getCordappAttachmentId(cordapp)!! - private val appContext get() = cordappProvider.getAppContext(cordapp) - private val serviceHub = rigorousMock().also { - doReturn(attachments).whenever(it).attachments - doReturn(cordappProvider).whenever(it).cordappProvider - doReturn(networkParameters).whenever(it).networkParameters - } - - // These ClassLoaders work together to load 'AnotherDummyContract' in a disposable way, such that even though - // the class may be on the unit test class path (due to default IDE settings, etc), it won't be loaded into the - // regular app classloader but rather than ClassLoaderForTests. This helps keep our environment clean and - // ensures we have precise control over where it's loaded. - object FilteringClassLoader : ClassLoader() { - @Throws(ClassNotFoundException::class) - override fun loadClass(name: String, resolve: Boolean): Class<*> { - if ("AnotherDummyContract" in name) { - throw ClassNotFoundException(name) - } - return super.loadClass(name, resolve) - } - } - - class ClassLoaderForTests : URLClassLoader(arrayOf(ISOLATED_CONTRACTS_JAR_PATH), FilteringClassLoader) - @Test - fun `dynamically load AnotherDummyContract from isolated contracts jar`() { - ClassLoaderForTests().use { child -> - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child) - val contract = contractClass.newInstance() as Contract - - assertEquals("helloworld", contract.declaredField("magicString").value) - } - } - - private fun fakeAttachment(filepath: String, content: String): ByteArray { - val bs = ByteArrayOutputStream() - JarOutputStream(bs).use { js -> - js.putNextEntry(ZipEntry(filepath)) - js.writer().apply { append(content); flush() } - js.closeEntry() - } - return bs.toByteArray() - } - - private fun readAttachment(attachment: Attachment, filepath: String): ByteArray { - ByteArrayOutputStream().use { - attachment.extractFile(filepath, it) - return it.toByteArray() - } - } - - @Test - fun `test MockAttachmentStorage open as jar`() { - val storage = attachments - val key = attachmentId - val attachment = storage.openAttachment(key)!! - - val jar = attachment.openAsJAR() - - assertNotNull(jar.nextEntry) - } - - @Test - @Suppress("DEPRECATION") - fun `test overlapping file exception`() { - val storage = attachments - val att0 = attachmentId - val att1 = storage.importAttachment(fakeAttachment("file.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("file.txt", "some other data").inputStream()) - - assertFailsWith(AttachmentsClassLoader.OverlappingAttachments::class) { - AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }) - } - } - - @Test - @Suppress("DEPRECATION") - fun basic() { - val storage = attachments - val att0 = attachmentId - val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream()) - - val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }) - val txt = IOUtils.toString(cl.getResourceAsStream("file1.txt"), Charsets.UTF_8.name()) - assertEquals("some data", txt) - } - - @Test - @Suppress("DEPRECATION") - fun `Check platform independent path handling in attachment jars`() { - val storage = MockAttachmentStorage() - - val att1 = storage.importAttachment(fakeAttachment("/folder1/foldera/file1.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("\\folder1\\folderb\\file2.txt", "some other data").inputStream()) - - val data1a = readAttachment(storage.openAttachment(att1)!!, "/folder1/foldera/file1.txt") - assertArrayEquals("some data".toByteArray(), data1a) - - val data1b = readAttachment(storage.openAttachment(att1)!!, "\\folder1\\foldera\\file1.txt") - assertArrayEquals("some data".toByteArray(), data1b) - - val data2a = readAttachment(storage.openAttachment(att2)!!, "\\folder1\\folderb\\file2.txt") - assertArrayEquals("some other data".toByteArray(), data2a) - - val data2b = readAttachment(storage.openAttachment(att2)!!, "/folder1/folderb/file2.txt") - assertArrayEquals("some other data".toByteArray(), data2b) - } - - @Test - @Suppress("DEPRECATION") - fun `loading class AnotherDummyContract`() { - val storage = attachments - val att0 = attachmentId - val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream()) - - val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader) - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl) - val contract = contractClass.newInstance() as Contract - assertEquals(cl, contract.javaClass.classLoader) - assertEquals("helloworld", contract.declaredField("magicString").value) - } - - private fun createContract2Cash(): Contract { - ClassLoaderForTests().use { cl -> - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl) - return contractClass.newInstance() as Contract - } - } - - @Test - @Suppress("DEPRECATION") - fun `testing Kryo with ClassLoader (with top level class name)`() { - val contract = createContract2Cash() - - val bytes = contract.serialize() - val storage = attachments - val att0 = attachmentId - val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream()) - - val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader) - - val context = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl).withWhitelisted(contract.javaClass) - val state2 = bytes.deserialize(context = context) - assertTrue(state2.javaClass.classLoader is AttachmentsClassLoader) - assertNotNull(state2) - } - - // top level wrapper - @CordaSerializable - class Data(val contract: Contract) - - @Test - @Suppress("DEPRECATION") - fun `testing Kryo with ClassLoader (without top level class name)`() { - val data = Data(createContract2Cash()) - - assertNotNull(data.contract) - - val context2 = SerializationFactory.defaultFactory.defaultContext.withWhitelisted(data.contract.javaClass) - - val bytes = data.serialize(context = context2) - val storage = attachments - val att0 = attachmentId - val att1 = storage.importAttachment(fakeAttachment("file1.txt", "some data").inputStream()) - val att2 = storage.importAttachment(fakeAttachment("file2.txt", "some other data").inputStream()) - - val cl = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader) - - val context = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl).withWhitelisted(Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl)) - - val state2 = bytes.deserialize(context = context) - assertEquals(cl, state2.contract.javaClass.classLoader) - assertNotNull(state2) - - // We should be able to load same class from a different class loader and have them be distinct. - val cl2 = AttachmentsClassLoader(arrayOf(att0, att1, att2).map { storage.openAttachment(it)!! }, FilteringClassLoader) - - val context3 = SerializationFactory.defaultFactory.defaultContext.withClassLoader(cl2).withWhitelisted(Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, cl2)) - - val state3 = bytes.deserialize(context = context3) - assertEquals(cl2, state3.contract.javaClass.classLoader) - assertNotNull(state3) - } - - @Test - fun `test serialization of SecureHash`() { - val secureHash = SecureHash.randomSHA256() - val bytes = secureHash.serialize() - val copiedSecuredHash = bytes.deserialize() - - assertEquals(secureHash, copiedSecuredHash) - } - - @Test - fun `test serialization of OpaqueBytes`() { - val opaqueBytes = OpaqueBytes("0123456789".toByteArray()) - val bytes = opaqueBytes.serialize() - val copiedOpaqueBytes = bytes.deserialize() - - assertEquals(opaqueBytes, copiedOpaqueBytes) - } - - @Test - fun `test serialization of sub-sequence OpaqueBytes`() { - val bytesSequence = ByteSequence.of("0123456789".toByteArray(), 3, 2) - val bytes = bytesSequence.serialize() - val copiedBytesSequence = bytes.deserialize() - - assertEquals(bytesSequence, copiedBytesSequence) - } - - @Test - fun `test serialization of WireTransaction with dynamically loaded contract`() { - val child = appContext.classLoader - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child) - val contract = contractClass.newInstance() as DummyContractBackdoor - val tx = contract.generateInitial(MEGA_CORP.ref(0), 42, DUMMY_NOTARY) - val context = SerializationFactory.defaultFactory.defaultContext - .withWhitelisted(contract.javaClass) - .withWhitelisted(Class.forName("$ISOLATED_CONTRACT_CLASS_NAME\$State", true, child)) - .withWhitelisted(Class.forName("$ISOLATED_CONTRACT_CLASS_NAME\$Commands\$Create", true, child)) - .withServiceHub(serviceHub) - .withClassLoader(child) - - val bytes = run { - val wireTransaction = tx.toWireTransaction(serviceHub, context) - wireTransaction.serialize(context = context) - } - val copiedWireTransaction = bytes.deserialize(context = context) - assertEquals(1, copiedWireTransaction.outputs.size) - // Contracts need to be loaded by the same classloader as the ContractState itself - val contractClassloader = copiedWireTransaction.getOutput(0).javaClass.classLoader - val contract2 = contractClassloader.loadClass(copiedWireTransaction.outputs.first().contract).newInstance() as DummyContractBackdoor - assertEquals(contract2.javaClass.classLoader, copiedWireTransaction.outputs[0].data.javaClass.classLoader) - assertEquals(42, contract2.inspectState(copiedWireTransaction.outputs[0].data)) - } - - @Test - fun `test deserialize of WireTransaction where contract cannot be found`() { - kryoSpecific("Kryo verifies/loads attachments on deserialization, whereas AMQP currently does not") { - ClassLoaderForTests().use { child -> - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child) - val contract = contractClass.newInstance() as DummyContractBackdoor - val tx = contract.generateInitial(MEGA_CORP.ref(0), 42, DUMMY_NOTARY) - val attachmentRef = attachmentId - val bytes = run { - val outboundContext = SerializationFactory.defaultFactory.defaultContext - .withServiceHub(serviceHub) - .withClassLoader(child) - val wireTransaction = tx.toWireTransaction(serviceHub, outboundContext) - wireTransaction.serialize(context = outboundContext) - } - // use empty attachmentStorage - - val e = assertFailsWith(MissingAttachmentsException::class) { - val mockAttStorage = MockAttachmentStorage() - val inboundContext = SerializationFactory.defaultFactory.defaultContext - .withAttachmentStorage(mockAttStorage) - .withAttachmentsClassLoader(listOf(attachmentRef)) - bytes.deserialize(context = inboundContext) - - if (mockAttStorage.openAttachment(attachmentRef) == null) { - throw MissingAttachmentsException(listOf(attachmentRef)) - } - } - assertEquals(attachmentRef, e.ids.single()) - } - } - } - - @Test - fun `test loading a class from attachment during deserialization`() { - ClassLoaderForTests().use { child -> - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child) - val contract = contractClass.newInstance() as DummyContractBackdoor - val outboundContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(child) - val attachmentRef = attachmentId - // We currently ignore annotations in attachments, so manually whitelist. - val inboundContext = SerializationFactory - .defaultFactory - .defaultContext - .withWhitelisted(contract.javaClass) - .withServiceHub(serviceHub) - .withAttachmentsClassLoader(listOf(attachmentRef)) - - // Serialize with custom context to avoid populating the default context with the specially loaded class - val serialized = contract.serialize(context = outboundContext) - // Then deserialize with the attachment class loader associated with the attachment - serialized.deserialize(context = inboundContext) - } - } - - @Test - fun `test loading a class with attachment missing during deserialization`() { - ClassLoaderForTests().use { child -> - val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, child) - val contract = contractClass.newInstance() as DummyContractBackdoor - val attachmentRef = SecureHash.randomSHA256() - val outboundContext = SerializationFactory.defaultFactory.defaultContext.withClassLoader(child) - // Serialize with custom context to avoid populating the default context with the specially loaded class - val serialized = contract.serialize(context = outboundContext) - - // Then deserialize with the attachment class loader associated with the attachment - val e = assertFailsWith(MissingAttachmentsException::class) { - // We currently ignore annotations in attachments, so manually whitelist. - val inboundContext = SerializationFactory - .defaultFactory - .defaultContext - .withWhitelisted(contract.javaClass) - .withServiceHub(serviceHub) - .withAttachmentsClassLoader(listOf(attachmentRef)) - serialized.deserialize(context = inboundContext) - } - assertEquals(attachmentRef, e.ids.single()) - } - } -} diff --git a/serialization/src/test/kotlin/net/corda/serialization/internal/CordaClassResolverTests.kt b/serialization/src/test/kotlin/net/corda/serialization/internal/CordaClassResolverTests.kt index 860a04a81c..700b8b0560 100644 --- a/serialization/src/test/kotlin/net/corda/serialization/internal/CordaClassResolverTests.kt +++ b/serialization/src/test/kotlin/net/corda/serialization/internal/CordaClassResolverTests.kt @@ -14,6 +14,7 @@ import net.corda.core.node.services.AttachmentStorage import net.corda.core.serialization.internal.CheckpointSerializationContext import net.corda.core.serialization.ClassWhitelist import net.corda.core.serialization.CordaSerializable +import net.corda.core.serialization.internal.AttachmentsClassLoader import net.corda.node.serialization.kryo.CordaClassResolver import net.corda.node.serialization.kryo.CordaKryo import net.corda.testing.internal.rigorousMock @@ -22,6 +23,7 @@ import org.junit.Rule import org.junit.Test import org.junit.rules.ExpectedException import java.lang.IllegalStateException +import java.net.URL import java.sql.Connection import java.util.* import kotlin.test.assertEquals @@ -112,6 +114,7 @@ class CordaClassResolverTests { val emptyListClass = listOf().javaClass val emptySetClass = setOf().javaClass val emptyMapClass = mapOf().javaClass + val ISOLATED_CONTRACTS_JAR_PATH: URL = CordaClassResolverTests::class.java.getResource("isolated.jar") } private val emptyWhitelistContext: CheckpointSerializationContext = CheckpointSerializationContextImpl(this.javaClass.classLoader, EmptyWhitelist, emptyMap(), true, null) @@ -201,7 +204,7 @@ class CordaClassResolverTests { CordaClassResolver(emptyWhitelistContext).getRegistration(DefaultSerializable::class.java) } - private fun importJar(storage: AttachmentStorage, uploader: String = DEPLOYED_CORDAPP_UPLOADER) = AttachmentsClassLoaderTests.ISOLATED_CONTRACTS_JAR_PATH.openStream().use { storage.importAttachment(it, uploader, "") } + private fun importJar(storage: AttachmentStorage, uploader: String = DEPLOYED_CORDAPP_UPLOADER) = ISOLATED_CONTRACTS_JAR_PATH.openStream().use { storage.importAttachment(it, uploader, "") } @Test(expected = KryoException::class) fun `Annotation does not work in conjunction with AttachmentClassLoader annotation`() { diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/core/TestUtils.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/core/TestUtils.kt index 485e7671c2..54b6e7e605 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/core/TestUtils.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/core/TestUtils.kt @@ -161,3 +161,4 @@ fun NodeInfo.singleIdentityAndCert(): PartyAndCertificate = legalIdentitiesAndCe * Extract a single identity from the node info. Throws an error if the node has multiple identities. */ fun NodeInfo.singleIdentity(): Party = singleIdentityAndCert().party + diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/InternalTestUtils.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/InternalTestUtils.kt index 8f82cf3792..5f044339b6 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/InternalTestUtils.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/InternalTestUtils.kt @@ -33,10 +33,13 @@ import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.registerDevP2pCertificates import net.corda.serialization.internal.amqp.AMQP_ENABLED import net.corda.testing.internal.stubs.CertificateStoreStubs +import java.io.ByteArrayOutputStream import java.nio.file.Files import java.nio.file.Path import java.security.KeyPair import java.util.* +import java.util.jar.JarOutputStream +import java.util.zip.ZipEntry import javax.security.auth.x500.X500Principal @Suppress("unused") @@ -169,7 +172,20 @@ fun configureDatabase(hikariProperties: Properties, schemaService: SchemaService = NodeSchemaService(), internalSchemas: Set = NodeSchemaService().internalSchemas(), cacheFactory: NamedCacheFactory = TestingNamedCacheFactory()): CordaPersistence { - val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService, hikariProperties, cacheFactory) + val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService, hikariProperties, cacheFactory, null) persistence.startHikariPool(hikariProperties, databaseConfig, internalSchemas) return persistence +} + +/** + * Convenience method for creating a fake attachment containing a file with some content. + */ +fun fakeAttachment(filePath: String, content: String): ByteArray { + val bs = ByteArrayOutputStream() + JarOutputStream(bs).use { js -> + js.putNextEntry(ZipEntry(filePath)) + js.writer().apply { append(content); flush() } + js.closeEntry() + } + return bs.toByteArray() } \ No newline at end of file diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt index a6847b8fa2..3b87150155 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt @@ -39,7 +39,7 @@ class MockCordappProvider( allFlows = emptyList(), jarHash = SecureHash.allOnesHash) if (cordappRegistry.none { it.first.contractClassNames.contains(contractClassName) && it.second == contractHash }) { - cordappRegistry.add(Pair(cordapp, findOrImportAttachment(listOf(contractClassName), contractClassName.toByteArray(), attachments, contractHash, signers))) + cordappRegistry.add(Pair(cordapp, findOrImportAttachment(listOf(contractClassName), fakeAttachmentCached(contractClassName), attachments, contractHash, signers))) } return cordappRegistry.findLast { contractClassName in it.first.contractClassNames }?.second!! } @@ -57,4 +57,9 @@ class MockCordappProvider( attachments.importContractAttachment(contractClassNames, DEPLOYED_CORDAPP_UPLOADER, data.inputStream(), contractHash, signers) } } + + private val attachmentsCache = mutableMapOf() + private fun fakeAttachmentCached(contractClass: String): ByteArray = attachmentsCache.computeIfAbsent(contractClass) { + fakeAttachment(contractClass, contractClass) + } }