diff --git a/.ci/api-current.txt b/.ci/api-current.txt index 28b29044d8..359b09e523 100644 --- a/.ci/api-current.txt +++ b/.ci/api-current.txt @@ -6160,7 +6160,7 @@ public abstract class net.corda.core.transactions.FullTransaction extends net.co public final class net.corda.core.transactions.LedgerTransaction extends net.corda.core.transactions.FullTransaction public (java.util.List>, java.util.List>, java.util.List>, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt) public (java.util.List>, java.util.List>, java.util.List>, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters) - public (java.util.List, java.util.List, java.util.List, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters, java.util.List, java.util.List, java.util.List, java.util.List, kotlin.jvm.functions.Function1, kotlin.jvm.functions.Function2, kotlin.jvm.internal.DefaultConstructorMarker) + public (java.util.List, java.util.List, java.util.List, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters, java.util.List, java.util.List, java.util.List, java.util.List, kotlin.jvm.functions.Function1, kotlin.jvm.functions.Function2, net.corda.core.serialization.internal.AttachmentsClassLoaderCache, kotlin.jvm.internal.DefaultConstructorMarker) @NotNull public final java.util.List> commandsOfType(Class) @NotNull diff --git a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt index 6c50e4e2a2..2a45e2105e 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt @@ -53,7 +53,7 @@ class ReceiveFinalityFlowTest { val paymentReceiverId = paymentReceiverFuture.getOrThrow() assertThat(bob.services.vaultService.queryBy>().states).isEmpty() - bob.assertFlowSentForObservationDueToConstraintError(paymentReceiverId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(paymentReceiverId) // Restart Bob with the contracts CorDapp so that it can recover from the error bob = mockNet.restartNode(bob, parameters = InternalMockNodeParameters(additionalCordapps = listOf(FINANCE_CONTRACTS_CORDAPP))) @@ -69,7 +69,7 @@ class ReceiveFinalityFlowTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -77,6 +77,6 @@ class ReceiveFinalityFlowTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt index 5cfaf252cb..4ca58d6b46 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt @@ -55,7 +55,7 @@ class AttachmentsClassLoaderSerializationTests { arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! }, testNetworkParameters(), SecureHash.zeroHash, - { attachmentTrustCalculator.calculate(it) }) { classLoader -> + { attachmentTrustCalculator.calculate(it) }, attachmentsClassLoaderCache = null) { classLoader -> val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader) val contract = contractClass.getDeclaredConstructor().newInstance() as Contract assertEquals("helloworld", contract.declaredField("magicString").value) diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt index bb198fe8b0..565641a096 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt @@ -23,6 +23,7 @@ import net.corda.core.internal.inputStream import net.corda.core.node.NetworkParameters import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.internal.AttachmentsClassLoader +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.testing.common.internal.testNetworkParameters import net.corda.node.services.attachments.NodeAttachmentTrustCalculator import net.corda.testing.contracts.DummyContract @@ -500,6 +501,7 @@ class AttachmentsClassLoaderTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) val transaction = createLedgerTransaction( inputs, outputs, @@ -511,7 +513,8 @@ class AttachmentsClassLoaderTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() } diff --git a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt index cec21d0f08..1fe9805681 100644 --- a/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt +++ b/core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt @@ -10,6 +10,7 @@ import net.corda.core.internal.AbstractAttachment import net.corda.core.internal.TESTDSL_UPLOADER import net.corda.core.internal.createLedgerTransaction import net.corda.core.node.NotaryInfo +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.WireTransaction import net.corda.testing.common.internal.testNetworkParameters @@ -18,6 +19,7 @@ import net.corda.testing.core.* import net.corda.testing.internal.createWireTransaction import net.corda.testing.internal.fakeAttachment import net.corda.testing.internal.rigorousMock +import net.corda.testing.internal.TestingNamedCacheFactory import org.junit.Rule import org.junit.Test import java.math.BigInteger @@ -131,6 +133,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) val transaction = createLedgerTransaction( inputs, outputs, @@ -142,7 +145,8 @@ class TransactionTests { privacySalt, testNetworkParameters(), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) transaction.verify() @@ -183,6 +187,7 @@ class TransactionTests { val id = SecureHash.randomSHA256() val timeWindow: TimeWindow? = null val privacySalt = PrivacySalt() + val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) fun buildTransaction() = createLedgerTransaction( inputs, @@ -195,7 +200,8 @@ class TransactionTests { privacySalt, testNetworkParameters(notaries = listOf(NotaryInfo(DUMMY_NOTARY, true))), emptyList(), - isAttachmentTrusted = { true } + isAttachmentTrusted = { true }, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) assertFailsWith { buildTransaction().verify() } diff --git a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt index 04cc543b81..e0e0cccf72 100644 --- a/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt +++ b/core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt @@ -4,6 +4,7 @@ import co.paralleluniverse.fibers.Suspendable import net.corda.core.DeleteForDJVM import net.corda.core.node.ServiceHub import net.corda.core.node.StatesToRecord +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import java.util.concurrent.ExecutorService // TODO: This should really be called ServiceHubInternal but that name is already taken by net.corda.node.services.api.ServiceHubInternal. @@ -15,6 +16,8 @@ interface ServiceHubCoreInternal : ServiceHub { val attachmentTrustCalculator: AttachmentTrustCalculator fun createTransactionsResolver(flow: ResolveTransactionsFlow): TransactionsResolver + + val attachmentsClassLoaderCache: AttachmentsClassLoaderCache } interface TransactionsResolver { diff --git a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt index e712ff10af..a5077cb0bf 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt @@ -1,5 +1,8 @@ package net.corda.core.serialization.internal +import com.github.benmanes.caffeine.cache.Cache +import com.github.benmanes.caffeine.cache.Caffeine +import net.corda.core.DeleteForDJVM import net.corda.core.contracts.Attachment import net.corda.core.contracts.ContractAttachment import net.corda.core.contracts.TransactionVerificationException @@ -19,7 +22,9 @@ import java.io.IOException import java.io.InputStream import java.lang.ref.WeakReference import java.net.* +import java.security.Permission import java.util.* +import java.util.function.Function /** * A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only @@ -288,31 +293,27 @@ class AttachmentsClassLoader(attachments: List, */ @VisibleForTesting object AttachmentsClassLoaderBuilder { - private const val CACHE_SIZE = 1000 + const val CACHE_SIZE = 16 - // We use a set here because the ordering of attachments doesn't affect code execution, due to the no - // overlap rule, and attachments don't have any particular ordering enforced by the builders. So we - // can just do unordered comparisons here. But the same attachments run with different network parameters - // may behave differently, so that has to be a part of the cache key. - private data class Key(val hashes: Set, val params: NetworkParameters) - - // This runs in the DJVM so it can't use caffeine. - private val cache: MutableMap = createSimpleCache(CACHE_SIZE).toSynchronised() + private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE) /** * Runs the given block with serialization execution context set up with a (possibly cached) attachments classloader. * * @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup. */ + @Suppress("LongParameterList") fun withAttachmentsClassloaderContext(attachments: List, params: NetworkParameters, txId: SecureHash, isAttachmentTrusted: (Attachment) -> Boolean, parent: ClassLoader = ClassLoader.getSystemClassLoader(), + attachmentsClassLoaderCache: AttachmentsClassLoaderCache?, block: (ClassLoader) -> T): T { val attachmentIds = attachments.map(Attachment::id).toSet() - val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) { + val cache = attachmentsClassLoaderCache ?: fallBackCache + val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function { // Create classloader and load serializers, whitelisted classes val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, isAttachmentTrusted, parent) val serializers = createInstancesOfClassesImplementing(transactionClassLoader, SerializationCustomSerializer::class.java) @@ -329,7 +330,7 @@ object AttachmentsClassLoaderBuilder { .withWhitelist(whitelistedClasses) .withCustomSerializers(serializers) .withoutCarpenter() - } + }) // Deserialize all relevant classes in the transaction classloader. return SerializationFactory.defaultFactory.withCurrentContext(serializationContext) { @@ -385,14 +386,6 @@ object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory { if (url.protocol != attachmentScheme) throw IllegalArgumentException("Cannot handle protocol: ${url.protocol}") return url.file.hashCode() } - - private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) { - override fun getContentLengthLong(): Long = attachment.size.toLong() - override fun getInputStream(): InputStream = attachment.open() - override fun connect() { - connected = true - } - } } } @@ -420,3 +413,51 @@ private class AttachmentsHolderImpl : AttachmentsHolder { attachments[key] = WeakReference(key) to value } } + +interface AttachmentsClassLoaderCache { + fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext +} + +@DeleteForDJVM +class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache { + + private val cache: Cache = cacheFactory.buildNamed(Caffeine.newBuilder(), "AttachmentsClassLoader_cache") + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function") + } +} + +class AttachmentsClassLoaderSimpleCacheImpl(cacheSize: Int) : AttachmentsClassLoaderCache { + + private val cache: MutableMap + = createSimpleCache(cacheSize).toSynchronised() + + override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function): SerializationContext { + return cache.computeIfAbsent(key, mappingFunction) + } +} + +// We use a set here because the ordering of attachments doesn't affect code execution, due to the no +// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we +// can just do unordered comparisons here. But the same attachments run with different network parameters +// may behave differently, so that has to be a part of the cache key. +data class AttachmentsClassLoaderKey(val hashes: Set, val params: NetworkParameters) + +private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) { + override fun getContentLengthLong(): Long = attachment.size.toLong() + override fun getInputStream(): InputStream = attachment.open() + /** + * Define the permissions that [AttachmentsClassLoader] will need to + * use this [URL]. The attachment is stored in memory, and so we + * don't need any extra permissions here. But if we don't override + * [getPermission] then [AttachmentsClassLoader] will assign the + * default permission of ALL_PERMISSION to these classes' + * [java.security.ProtectionDomain]. This would be a security hole! + */ + override fun getPermission(): Permission? = null + override fun connect() { + connected = true + } +} + diff --git a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt index c590047267..277dccc1d2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt @@ -153,7 +153,8 @@ data class ContractUpgradeWireTransaction( listOf(legacyAttachment, upgradedAttachment), params, id, - { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }) { transactionClassLoader -> + { (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }, + attachmentsClassLoaderCache = (services as ServiceHubCoreInternal).attachmentsClassLoaderCache) { transactionClassLoader -> val resolvedInput = binaryInput.deserialize() val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader) val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment) diff --git a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt index 3dffc8182c..6c73c299c2 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt @@ -26,6 +26,7 @@ import net.corda.core.internal.deserialiseComponentGroup import net.corda.core.internal.isUploaderTrusted import net.corda.core.internal.uncheckedCast import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder import net.corda.core.utilities.contextLogger import java.util.Collections.unmodifiableList @@ -87,7 +88,8 @@ private constructor( private val serializedInputs: List?, private val serializedReferences: List?, private val isAttachmentTrusted: (Attachment) -> Boolean, - private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier + private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier, + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ) : FullTransaction() { init { @@ -124,7 +126,8 @@ private constructor( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { return LedgerTransaction( inputs = inputs, @@ -141,7 +144,8 @@ private constructor( serializedInputs = protect(serializedInputs), serializedReferences = protect(serializedReferences), isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -176,7 +180,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { true }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) } } @@ -218,7 +223,8 @@ private constructor( txAttachments, getParamsWithGoo(), id, - isAttachmentTrusted = isAttachmentTrusted) { transactionClassLoader -> + isAttachmentTrusted = isAttachmentTrusted, + attachmentsClassLoaderCache = attachmentsClassLoaderCache) { transactionClassLoader -> // Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader]. // Only the copy will be used for verification, and the outer shell will be discarded. // This artifice is required to preserve backwards compatibility. @@ -254,7 +260,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = alternateVerifier + verifierFactory = alternateVerifier, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) // Read network parameters with backwards compatibility goo. @@ -320,7 +327,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } else { // This branch is only present for backwards compatibility. @@ -704,7 +712,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransaction should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -733,7 +742,8 @@ private constructor( serializedInputs = null, serializedReferences = null, isAttachmentTrusted = { it.isUploaderTrusted() }, - verifierFactory = ::BasicVerifier + verifierFactory = ::BasicVerifier, + attachmentsClassLoaderCache = null ) @Deprecated("LedgerTransactions should not be created directly, use WireTransaction.toLedgerTransaction instead.") @@ -761,7 +771,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } @@ -791,7 +802,8 @@ private constructor( serializedInputs = serializedInputs, serializedReferences = serializedReferences, isAttachmentTrusted = isAttachmentTrusted, - verifierFactory = verifierFactory + verifierFactory = verifierFactory, + attachmentsClassLoaderCache = attachmentsClassLoaderCache ) } } diff --git a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt index 89e81f39be..a4468771c1 100644 --- a/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt +++ b/core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt @@ -15,6 +15,7 @@ import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.serialization.serialize import net.corda.core.utilities.OpaqueBytes import java.security.PublicKey @@ -109,7 +110,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr services.networkParametersService.lookup(hashToResolve) }, // `as?` is used due to [MockServices] not implementing [ServiceHubCoreInternal] - isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true } + isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true }, + attachmentsClassLoaderCache = (services as? ServiceHubCoreInternal)?.attachmentsClassLoaderCache ) ) } @@ -145,7 +147,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, { null }, - { it.isUploaderTrusted() } + { it.isUploaderTrusted() }, + null ) } @@ -161,16 +164,19 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr resolveAttachment, { stateRef -> resolveStateRef(stateRef)?.serialize() }, resolveParameters, - { true } // Any attachment loaded through the DJVM should be trusted + { true }, // Any attachment loaded through the DJVM should be trusted + null ) } + @Suppress("LongParameterList", "ThrowsCount") private fun toLedgerTransactionInternal( resolveIdentity: (PublicKey) -> Party?, resolveAttachment: (SecureHash) -> Attachment?, resolveStateRefAsSerialized: (StateRef) -> SerializedBytes>?, resolveParameters: (SecureHash?) -> NetworkParameters?, - isAttachmentTrusted: (Attachment) -> Boolean + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache? ): LedgerTransaction { // Look up public keys to authenticated identities. val authenticatedCommands = commands.lazyMapped { cmd, _ -> @@ -206,7 +212,8 @@ class WireTransaction(componentGroups: List, val privacySalt: Pr componentGroups, serializedResolvedInputs, serializedResolvedReferences, - isAttachmentTrusted + isAttachmentTrusted, + attachmentsClassLoaderCache ) checkTransactionSize(ltx, resolvedNetworkParameters.maxTransactionSize, serializedResolvedInputs, serializedResolvedReferences) diff --git a/core/src/test/kotlin/net/corda/core/internal/ClassLoadingUtilsTest.kt b/core/src/test/kotlin/net/corda/core/internal/ClassLoadingUtilsTest.kt index 3e19491b94..68bd3ba625 100644 --- a/core/src/test/kotlin/net/corda/core/internal/ClassLoadingUtilsTest.kt +++ b/core/src/test/kotlin/net/corda/core/internal/ClassLoadingUtilsTest.kt @@ -5,13 +5,20 @@ import net.corda.core.contracts.ContractAttachment import net.corda.core.contracts.ContractClassName import net.corda.core.crypto.SecureHash import net.corda.core.identity.Party +import net.corda.core.node.services.AttachmentId import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory import net.corda.core.serialization.internal.AttachmentsClassLoader import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatThrownBy import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Ignore import org.junit.Test import java.io.ByteArrayOutputStream +import java.lang.ref.ReferenceQueue +import java.lang.ref.WeakReference +import java.net.URL import java.net.URLClassLoader import java.security.PublicKey import java.util.jar.JarOutputStream @@ -120,9 +127,125 @@ class ClassLoadingUtilsTest { } } - private fun signedAttachment(data: ByteArray, vararg parties: Party) = ContractAttachment.create( + @Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.") + @Test(timeout=300_000) + @Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE") + fun `test weak reference removed from map`() { + val jarData = with(ByteArrayOutputStream()) { + val internalName = STANDALONE_CLASS_NAME.asInternalName + JarOutputStream(this, Manifest()).use { + it.setLevel(NO_COMPRESSION) + it.setMethod(DEFLATED) + it.putNextEntry(directoryEntry("com")) + it.putNextEntry(directoryEntry("com/example")) + it.putNextEntry(classEntry(internalName)) + it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName)) + } + toByteArray() + } + val attachment = signedAttachment(jarData) + var url: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment) + + val referenceQueue: ReferenceQueue = ReferenceQueue() + val weakReference = WeakReference(url, referenceQueue) + + assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + // Clear strong reference + url = null + System.gc() + val ref = referenceQueue.remove(100000) + assertSame(weakReference, ref) + assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + } + + @Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.") + @Test(timeout=300_000) + @Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE") + fun `test adding same attachment twice then removing`() { + val jarData = with(ByteArrayOutputStream()) { + val internalName = STANDALONE_CLASS_NAME.asInternalName + JarOutputStream(this, Manifest()).use { + it.setLevel(NO_COMPRESSION) + it.setMethod(DEFLATED) + it.putNextEntry(directoryEntry("com")) + it.putNextEntry(directoryEntry("com/example")) + it.putNextEntry(classEntry(internalName)) + it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName)) + } + toByteArray() + } + val attachment1 = signedAttachment(jarData) + val attachment2 = signedAttachment(jarData) + var url1: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment1) + var url2: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment2) + + val referenceQueue1: ReferenceQueue = ReferenceQueue() + val weakReference1 = WeakReference(url1, referenceQueue1) + + val referenceQueue2: ReferenceQueue = ReferenceQueue() + val weakReference2 = WeakReference(url2, referenceQueue2) + + assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + url1 = null + System.gc() + val ref1 = referenceQueue1.remove(500) + assertNull(ref1) + assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + + url2 = null + System.gc() + val ref2 = referenceQueue2.remove(100000) + assertSame(weakReference2, ref2) + assertSame(weakReference1, referenceQueue1.poll()) + assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + } + + @Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.") + @Test(timeout=300_000) + @Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE") + fun `test adding two different attachments then removing`() { + val jarData1 = with(ByteArrayOutputStream()) { + val internalName = STANDALONE_CLASS_NAME.asInternalName + JarOutputStream(this, Manifest()).use { + it.setLevel(NO_COMPRESSION) + it.setMethod(DEFLATED) + it.putNextEntry(directoryEntry("com")) + it.putNextEntry(directoryEntry("com/example")) + it.putNextEntry(classEntry(internalName)) + it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName)) + } + toByteArray() + } + + val attachment1 = signedAttachment(jarData1) + val attachment2 = signedAttachment(jarData1, id = SecureHash.randomSHA256()) + var url1: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment1) + var url2: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment2) + + val referenceQueue1: ReferenceQueue = ReferenceQueue() + val weakReference1 = WeakReference(url1, referenceQueue1) + + val referenceQueue2: ReferenceQueue = ReferenceQueue() + val weakReference2 = WeakReference(url2, referenceQueue2) + + assertEquals(2, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + url1 = null + System.gc() + val ref1 = referenceQueue1.remove(100000) + assertSame(weakReference1, ref1) + assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + + url2 = null + System.gc() + val ref2 = referenceQueue2.remove(100000) + assertSame(weakReference2, ref2) + assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize()) + } + + private fun signedAttachment(data: ByteArray, id: AttachmentId = contractAttachmentId, + vararg parties: Party) = ContractAttachment.create( object : AbstractAttachment({ data }, "test") { - override val id: SecureHash get() = contractAttachmentId + override val id: SecureHash get() = id override val signerKeys: List get() = parties.map(Party::owningKey) }, PROGRAM_ID, signerKeys = parties.map(Party::owningKey) diff --git a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt index 148aaff7bf..c325c805e3 100644 --- a/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt +++ b/core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt @@ -4,6 +4,7 @@ import net.corda.core.contracts.* import net.corda.core.crypto.SecureHash import net.corda.core.identity.Party import net.corda.core.node.NetworkParameters +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache import net.corda.core.transactions.ComponentGroup import net.corda.core.transactions.LedgerTransaction import net.corda.core.transactions.WireTransaction @@ -17,6 +18,7 @@ fun WireTransaction.accessGroupHashes() = this.groupHashes fun WireTransaction.accessGroupMerkleRoots() = this.groupsMerkleRoots fun WireTransaction.accessAvailableComponentHashes() = this.availableComponentHashes +@Suppress("LongParameterList") fun createLedgerTransaction( inputs: List>, outputs: List>, @@ -31,8 +33,9 @@ fun createLedgerTransaction( componentGroups: List? = null, serializedInputs: List? = null, serializedReferences: List? = null, - isAttachmentTrusted: (Attachment) -> Boolean -): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted) + isAttachmentTrusted: (Attachment) -> Boolean, + attachmentsClassLoaderCache: AttachmentsClassLoaderCache +): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted, attachmentsClassLoaderCache) fun createContractCreationError(txId: SecureHash, contractClass: String, cause: Throwable) = TransactionVerificationException.ContractCreationError(txId, contractClass, cause) fun createContractRejection(txId: SecureHash, contract: Contract, cause: Throwable) = TransactionVerificationException.ContractRejection(txId, contract, cause) diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 83e35807a6..43857548c1 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -57,6 +57,8 @@ import net.corda.core.schemas.MappedSchema import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SingletonSerializeAsToken +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.LedgerTransaction import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.days @@ -315,6 +317,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, } else { BasicVerifierFactoryService() } + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory).tokenize() val contractUpgradeService = ContractUpgradeServiceImpl(cacheFactory).tokenize() val auditService = DummyAuditService().tokenize() @Suppress("LeakingThis") @@ -1144,6 +1147,8 @@ abstract class AbstractNode(val configuration: NodeConfiguration, private lateinit var _myInfo: NodeInfo override val myInfo: NodeInfo get() = _myInfo + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache get() = this@AbstractNode.attachmentsClassLoaderCache + private lateinit var _networkParameters: NetworkParameters override val networkParameters: NetworkParameters get() = _networkParameters diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt index 70bb911106..0e2538e8ac 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt @@ -37,6 +37,7 @@ class MigrationNamedCacheFactory(private val metricRegistry: MetricRegistry?, "NodeAttachmentService_contractAttachmentVersions" -> caffeine.maximumSize(defaultCacheSize) "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name.") } } diff --git a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt index ec8b5d315d..0186b9659c 100644 --- a/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt +++ b/node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt @@ -15,6 +15,8 @@ import net.corda.core.node.services.NetworkParametersService import net.corda.core.node.services.TransactionStorage import net.corda.core.serialization.deserialize import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.ContractUpgradeLedgerTransaction import net.corda.core.transactions.NotaryChangeLedgerTransaction import net.corda.core.transactions.WireTransaction @@ -62,6 +64,8 @@ class MigrationServicesForResolution( cacheFactory ) + private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory) + private fun defaultNetworkParameters(): NetworkParameters { logger.warn("Using a dummy set of network parameters for migration.") val clock = Clock.systemUTC() @@ -124,7 +128,8 @@ class MigrationServicesForResolution( networkParameters, tx.id, attachmentTrustCalculator::calculate, - cordappLoader.appClassLoader) { + cordappLoader.appClassLoader, + attachmentsClassLoaderCache) { deserialiseComponentGroup(tx.componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true) } states.filterIndexed {index, _ -> stateIndices.contains(index)}.toList() diff --git a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt index 4a514e0172..4d6dd05b19 100644 --- a/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt +++ b/node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt @@ -63,6 +63,7 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi name == "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize) name == "PublicKeyToOwningIdentityCache_cache" -> caffeine.maximumSize(defaultCacheSize) name == "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize) + name == "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultAttachmentsClassLoaderCacheSize) else -> throw IllegalArgumentException("Unexpected cache name $name. Did you add a new cache?") } } @@ -85,4 +86,6 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi } open protected val defaultCacheSize = 1024L -} \ No newline at end of file + private val defaultAttachmentsClassLoaderCacheSize = defaultCacheSize / CACHE_SIZE_DENOMINATOR +} +private const val CACHE_SIZE_DENOMINATOR = 4L \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt index 594932f5c0..b6bb0817b2 100644 --- a/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt @@ -50,13 +50,13 @@ class FinalityHandlerTest { getOrThrow() } - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() bob = mockNet.restartNode(bob) // Since we've not done anything to fix the orignal error, we expect the finality handler to be sent to the hospital // again on restart - bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId) + bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId) assertThat(bob.getTransaction(stx.id)).isNull() } @@ -96,7 +96,7 @@ class FinalityHandlerTest { .ofType(R::class.java) } - private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) { + private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) { val observation = medicalRecordsOfType() .filter { it.flowId == runId } .toBlocking() @@ -104,7 +104,7 @@ class FinalityHandlerTest { assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION) assertThat(observation.by).contains(FinalityDoctor) val error = observation.errors.single() - assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java) + assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java) } private fun TestStartedNode.getTransaction(id: SecureHash): SignedTransaction? { diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt index fc12fedc19..386545886a 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt @@ -13,6 +13,8 @@ import net.corda.core.node.ServiceHub import net.corda.core.node.ServicesForResolution import net.corda.core.node.services.AttachmentId import net.corda.core.node.services.TransactionStorage +import net.corda.core.serialization.internal.AttachmentsClassLoaderCache +import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl import net.corda.core.transactions.SignedTransaction import net.corda.core.transactions.TransactionBuilder import net.corda.core.transactions.WireTransaction @@ -127,6 +129,8 @@ data class TestTransactionDSLInterpreter private constructor( override val cordappProvider: CordappProvider = ledgerInterpreter.services.cordappProvider + + override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory()) } private fun copy(): TestTransactionDSLInterpreter = diff --git a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt index 6802fd042e..402b3757c0 100644 --- a/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt +++ b/testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt @@ -26,6 +26,7 @@ class TestingNamedCacheFactory private constructor(private val sizeOverride: Lon val configuredCaffeine = when (name) { "DBTransactionStorage_transactions" -> caffeine.maximumWeight(1.MB) "NodeAttachmentService_attachmentContent" -> caffeine.maximumWeight(1.MB) + "AttachmentsClassLoader_cache" -> caffeine.maximumSize(sizeOverride) else -> caffeine.maximumSize(sizeOverride) } return configuredCaffeine.build(loader)