ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2025-01-31 00:24:59 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

CORDA-4125: Backport AttachmentClassLoader updates to 4.4 to enable ENT-6152 backport (#6878)

Browse Source 
* CORDA-3755: Switched attachments map to a WeakHashMap (#6214)

* Bump OS release version 4.6

* CORDA-3755: Switched attachments map to a WeakHashMap

* CORDA-3755: Added explicit strong references to map key.

* CORDA-3755: Keeping detekt happy.

* CORDA-3755: Test a gc in verify.

* CORDA-3755: Making detekt happy.

* CORDA-3755: Suppress warnings for weak reference test.

* CORDA-3755: Fixing build failure with attachments.

* CORDA-3755: Rewrite based on Ricks input - now handles attachment already existing in map!

* CORDA-3755: Refactor WeakReference behaviour into AttachmentsHolderImpl and provide alternate version of this class for core-deterministic.

* CORDA-3755: Added more tests for WeakHashMap.

* CORDA-3755: Ignore the tests using System.gc keep for local testing only

* CORDA-3755: Adding comment to explain the ignored tests.

* Make AttachmentsHolderImpl package-private inside core-deterministic, just like it is inside core.

* CORDA-3755: Update assertions following review comments.

* CORDA-3755: Removing import

* CORDA-3755: Removed unused var.

* CORDA-3755: Reverting files that somehow got changed in rebase.

Co-authored-by: nargas-ritu <ritu.gupta@r3.com>
Co-authored-by: Chris Rankin <chris.rankin@r3.com>

* CORDA-3769: Switched attachments class loader cache to use caffeine (#6326)

* CORDA-3769: Switched attachments class loader cache to use caffeine with original implementation used by determinstic core.

* CORDA-3769: Removed default ctor arguments.

* CORDA-3769: Switched mapping function to Function type to avoid synthetic method being generated.

* CORDA-3769: Now using a cache created from NamedCacheFactory for the attachments class loader cache.

* CORDA-3769: Making detekt happy.

* CORDA-3769: The finality tests now check for UntrustedAttachmentsException which will actually happen in reality.

* CORDA-3769: Refactored after review comments.

* CORDA-3769: Removed the AttachmentsClassLoaderSimpleCacheImpl as DJVM does not need it. Also updated due to review comments.

* CORDA-3769: Removed the generic parameters from AttachmentsClassLoader.

* CORDA-3769: Removed unused imports.

* CORDA-3769: Updates from review comments.

* CORDA-3769: Updated following review comments. MigrationServicesForResolution now uses cache factory. Ctor updated for AttachmentsClassLoaderSimpleCacheImpl.

* CORDA-3769: Reduced max class loader cache size

* CORDA-3769: Fixed the attachments class loader cache size to a fixed default

* CORDA-3769: Switched attachments class loader size to be reduced by fixed value.

* CORDA-4125: Parameter has been added to a private ctor.

Co-authored-by: nargas-ritu <ritu.gupta@r3.com>
Co-authored-by: Chris Rankin <chris.rankin@r3.com>
This commit is contained in:
Adel El-Beik 2021-02-19 05:38:22 -08:00 committed by GitHub
parent 90238f22a0
commit 360b3f8d0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 272 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.ci/api-current.txt
View File

@ -6160,7 +6160,7 @@ public abstract class net.corda.core.transactions.FullTransaction extends net.co
public final class net.corda.core.transactions.LedgerTransaction extends net.corda.core.transactions.FullTransaction
public <init>(java.util.List<? extends net.corda.core.contracts.StateAndRef<? extends net.corda.core.contracts.ContractState>>, java.util.List<? extends net.corda.core.contracts.TransactionState<? extends net.corda.core.contracts.ContractState>>, java.util.List<? extends net.corda.core.contracts.CommandWithParties<? extends net.corda.core.contracts.CommandData>>, java.util.List<? extends net.corda.core.contracts.Attachment>, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt)
public <init>(java.util.List<? extends net.corda.core.contracts.StateAndRef<? extends net.corda.core.contracts.ContractState>>, java.util.List<? extends net.corda.core.contracts.TransactionState<? extends net.corda.core.contracts.ContractState>>, java.util.List<? extends net.corda.core.contracts.CommandWithParties<? extends net.corda.core.contracts.CommandData>>, java.util.List<? extends net.corda.core.contracts.Attachment>, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters)
public <init>(java.util.List, java.util.List, java.util.List, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters, java.util.List, java.util.List, java.util.List, java.util.List, kotlin.jvm.functions.Function1, kotlin.jvm.functions.Function2, kotlin.jvm.internal.DefaultConstructorMarker)
public <init>(java.util.List, java.util.List, java.util.List, java.util.List, net.corda.core.crypto.SecureHash, net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt, net.corda.core.node.NetworkParameters, java.util.List, java.util.List, java.util.List, java.util.List, kotlin.jvm.functions.Function1, kotlin.jvm.functions.Function2, net.corda.core.serialization.internal.AttachmentsClassLoaderCache, kotlin.jvm.internal.DefaultConstructorMarker)
@NotNull
public final java.util.List<net.corda.core.contracts.Command<T>> commandsOfType(Class<T>)
@NotNull

6
core-tests/src/test/kotlin/net/corda/coretests/flows/ReceiveFinalityFlowTest.kt
View File

@ -53,7 +53,7 @@ class ReceiveFinalityFlowTest {
val paymentReceiverId = paymentReceiverFuture.getOrThrow()
assertThat(bob.services.vaultService.queryBy<FungibleAsset<*>>().states).isEmpty()
bob.assertFlowSentForObservationDueToConstraintError(paymentReceiverId)
bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(paymentReceiverId)
// Restart Bob with the contracts CorDapp so that it can recover from the error
bob = mockNet.restartNode(bob, parameters = InternalMockNodeParameters(additionalCordapps = listOf(FINANCE_CONTRACTS_CORDAPP)))
@ -69,7 +69,7 @@ class ReceiveFinalityFlowTest {
.ofType(R::class.java)
}
private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) {
private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) {
val observation = medicalRecordsOfType<Flow>()
.filter { it.flowId == runId }
.toBlocking()
@ -77,6 +77,6 @@ class ReceiveFinalityFlowTest {
assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION)
assertThat(observation.by).contains(FinalityDoctor)
val error = observation.errors.single()
assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java)
assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java)
}
}

2
core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderSerializationTests.kt
View File

@ -55,7 +55,7 @@ class AttachmentsClassLoaderSerializationTests {
arrayOf(isolatedId, att1, att2).map { storage.openAttachment(it)!! },
testNetworkParameters(),
SecureHash.zeroHash,
{ attachmentTrustCalculator.calculate(it) }) { classLoader ->
{ attachmentTrustCalculator.calculate(it) }, attachmentsClassLoaderCache = null) { classLoader ->
val contractClass = Class.forName(ISOLATED_CONTRACT_CLASS_NAME, true, classLoader)
val contract = contractClass.getDeclaredConstructor().newInstance() as Contract
assertEquals("helloworld", contract.declaredField<Any?>("magicString").value)

5
core-tests/src/test/kotlin/net/corda/coretests/transactions/AttachmentsClassLoaderTests.kt
View File

@ -23,6 +23,7 @@ import net.corda.core.internal.inputStream
import net.corda.core.node.NetworkParameters
import net.corda.core.node.services.AttachmentId
import net.corda.core.serialization.internal.AttachmentsClassLoader
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
import net.corda.testing.common.internal.testNetworkParameters
import net.corda.node.services.attachments.NodeAttachmentTrustCalculator
import net.corda.testing.contracts.DummyContract
@ -500,6 +501,7 @@ class AttachmentsClassLoaderTests {
val id = SecureHash.randomSHA256()
val timeWindow: TimeWindow? = null
val privacySalt = PrivacySalt()
val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory)
val transaction = createLedgerTransaction(
inputs,
outputs,
@ -511,7 +513,8 @@ class AttachmentsClassLoaderTests {
privacySalt,
testNetworkParameters(),
emptyList(),
isAttachmentTrusted = { true }
isAttachmentTrusted = { true },
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
transaction.verify()
}

10
core-tests/src/test/kotlin/net/corda/coretests/transactions/TransactionTests.kt
View File

@ -10,6 +10,7 @@ import net.corda.core.internal.AbstractAttachment
import net.corda.core.internal.TESTDSL_UPLOADER
import net.corda.core.internal.createLedgerTransaction
import net.corda.core.node.NotaryInfo
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
import net.corda.core.transactions.SignedTransaction
import net.corda.core.transactions.WireTransaction
import net.corda.testing.common.internal.testNetworkParameters
@ -18,6 +19,7 @@ import net.corda.testing.core.*
import net.corda.testing.internal.createWireTransaction
import net.corda.testing.internal.fakeAttachment
import net.corda.testing.internal.rigorousMock
import net.corda.testing.internal.TestingNamedCacheFactory
import org.junit.Rule
import org.junit.Test
import java.math.BigInteger
@ -131,6 +133,7 @@ class TransactionTests {
val id = SecureHash.randomSHA256()
val timeWindow: TimeWindow? = null
val privacySalt = PrivacySalt()
val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory())
val transaction = createLedgerTransaction(
inputs,
outputs,
@ -142,7 +145,8 @@ class TransactionTests {
privacySalt,
testNetworkParameters(),
emptyList(),
isAttachmentTrusted = { true }
isAttachmentTrusted = { true },
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
transaction.verify()
@ -183,6 +187,7 @@ class TransactionTests {
val id = SecureHash.randomSHA256()
val timeWindow: TimeWindow? = null
val privacySalt = PrivacySalt()
val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory())
fun buildTransaction() = createLedgerTransaction(
inputs,
@ -195,7 +200,8 @@ class TransactionTests {
privacySalt,
testNetworkParameters(notaries = listOf(NotaryInfo(DUMMY_NOTARY, true))),
emptyList(),
isAttachmentTrusted = { true }
isAttachmentTrusted = { true },
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
assertFailsWith<TransactionVerificationException.NotaryChangeInWrongTransactionType> { buildTransaction().verify() }

3
core/src/main/kotlin/net/corda/core/internal/ServiceHubCoreInternal.kt
View File

@ -4,6 +4,7 @@ import co.paralleluniverse.fibers.Suspendable
import net.corda.core.DeleteForDJVM
import net.corda.core.node.ServiceHub
import net.corda.core.node.StatesToRecord
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import java.util.concurrent.ExecutorService
// TODO: This should really be called ServiceHubInternal but that name is already taken by net.corda.node.services.api.ServiceHubInternal.
@ -15,6 +16,8 @@ interface ServiceHubCoreInternal : ServiceHub {
val attachmentTrustCalculator: AttachmentTrustCalculator
fun createTransactionsResolver(flow: ResolveTransactionsFlow): TransactionsResolver
val attachmentsClassLoaderCache: AttachmentsClassLoaderCache
}
interface TransactionsResolver {

79
core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt
View File

@ -1,5 +1,8 @@
package net.corda.core.serialization.internal
import com.github.benmanes.caffeine.cache.Cache
import com.github.benmanes.caffeine.cache.Caffeine
import net.corda.core.DeleteForDJVM
import net.corda.core.contracts.Attachment
import net.corda.core.contracts.ContractAttachment
import net.corda.core.contracts.TransactionVerificationException
@ -19,7 +22,9 @@ import java.io.IOException
import java.io.InputStream
import java.lang.ref.WeakReference
import java.net.*
import java.security.Permission
import java.util.*
import java.util.function.Function
/**
* A custom ClassLoader that knows how to load classes from a set of attachments. The attachments themselves only
@ -288,31 +293,27 @@ class AttachmentsClassLoader(attachments: List<Attachment>,
*/
@VisibleForTesting
object AttachmentsClassLoaderBuilder {
private const val CACHE_SIZE = 1000
const val CACHE_SIZE = 16
// We use a set here because the ordering of attachments doesn't affect code execution, due to the no
// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we
// can just do unordered comparisons here. But the same attachments run with different network parameters
// may behave differently, so that has to be a part of the cache key.
private data class Key(val hashes: Set<SecureHash>, val params: NetworkParameters)
// This runs in the DJVM so it can't use caffeine.
private val cache: MutableMap<Key, SerializationContext> = createSimpleCache<Key, SerializationContext>(CACHE_SIZE).toSynchronised()
private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE)
/**
* Runs the given block with serialization execution context set up with a (possibly cached) attachments classloader.
*
* @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup.
*/
@Suppress("LongParameterList")
fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>,
params: NetworkParameters,
txId: SecureHash,
isAttachmentTrusted: (Attachment) -> Boolean,
parent: ClassLoader = ClassLoader.getSystemClassLoader(),
attachmentsClassLoaderCache: AttachmentsClassLoaderCache?,
block: (ClassLoader) -> T): T {
val attachmentIds = attachments.map(Attachment::id).toSet()
val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) {
val cache = attachmentsClassLoaderCache ?: fallBackCache
val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function {
// Create classloader and load serializers, whitelisted classes
val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, isAttachmentTrusted, parent)
val serializers = createInstancesOfClassesImplementing(transactionClassLoader, SerializationCustomSerializer::class.java)
@ -329,7 +330,7 @@ object AttachmentsClassLoaderBuilder {
.withWhitelist(whitelistedClasses)
.withCustomSerializers(serializers)
.withoutCarpenter()
}
})
// Deserialize all relevant classes in the transaction classloader.
return SerializationFactory.defaultFactory.withCurrentContext(serializationContext) {
@ -385,14 +386,6 @@ object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory {
if (url.protocol != attachmentScheme) throw IllegalArgumentException("Cannot handle protocol: ${url.protocol}")
return url.file.hashCode()
}
private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) {
override fun getContentLengthLong(): Long = attachment.size.toLong()
override fun getInputStream(): InputStream = attachment.open()
override fun connect() {
connected = true
}
}
}
}
@ -420,3 +413,51 @@ private class AttachmentsHolderImpl : AttachmentsHolder {
attachments[key] = WeakReference(key) to value
}
}
interface AttachmentsClassLoaderCache {
fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext
}
@DeleteForDJVM
class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache {
private val cache: Cache<AttachmentsClassLoaderKey, SerializationContext> = cacheFactory.buildNamed(Caffeine.newBuilder(), "AttachmentsClassLoader_cache")
override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function")
}
}
class AttachmentsClassLoaderSimpleCacheImpl(cacheSize: Int) : AttachmentsClassLoaderCache {
private val cache: MutableMap<AttachmentsClassLoaderKey, SerializationContext>
= createSimpleCache<AttachmentsClassLoaderKey, SerializationContext>(cacheSize).toSynchronised()
override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
return cache.computeIfAbsent(key, mappingFunction)
}
}
// We use a set here because the ordering of attachments doesn't affect code execution, due to the no
// overlap rule, and attachments don't have any particular ordering enforced by the builders. So we
// can just do unordered comparisons here. But the same attachments run with different network parameters
// may behave differently, so that has to be a part of the cache key.
data class AttachmentsClassLoaderKey(val hashes: Set<SecureHash>, val params: NetworkParameters)
private class AttachmentURLConnection(url: URL, private val attachment: Attachment) : URLConnection(url) {
override fun getContentLengthLong(): Long = attachment.size.toLong()
override fun getInputStream(): InputStream = attachment.open()
/**
* Define the permissions that [AttachmentsClassLoader] will need to
* use this [URL]. The attachment is stored in memory, and so we
* don't need any extra permissions here. But if we don't override
* [getPermission] then [AttachmentsClassLoader] will assign the
* default permission of ALL_PERMISSION to these classes'
* [java.security.ProtectionDomain]. This would be a security hole!
*/
override fun getPermission(): Permission? = null
override fun connect() {
connected = true
}
}

3
core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt
View File

@ -153,7 +153,8 @@ data class ContractUpgradeWireTransaction(
listOf(legacyAttachment, upgradedAttachment),
params,
id,
{ (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) }) { transactionClassLoader ->
{ (services as ServiceHubCoreInternal).attachmentTrustCalculator.calculate(it) },
attachmentsClassLoaderCache = (services as ServiceHubCoreInternal).attachmentsClassLoaderCache) { transactionClassLoader ->
val resolvedInput = binaryInput.deserialize()
val upgradedContract = upgradedContract(upgradedContractClassName, transactionClassLoader)
val outputState = calculateUpgradedState(resolvedInput, upgradedContract, upgradedAttachment)

34
core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt
View File

@ -26,6 +26,7 @@ import net.corda.core.internal.deserialiseComponentGroup
import net.corda.core.internal.isUploaderTrusted
import net.corda.core.internal.uncheckedCast
import net.corda.core.node.NetworkParameters
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
import net.corda.core.utilities.contextLogger
import java.util.Collections.unmodifiableList
@ -87,7 +88,8 @@ private constructor(
private val serializedInputs: List<SerializedStateAndRef>?,
private val serializedReferences: List<SerializedStateAndRef>?,
private val isAttachmentTrusted: (Attachment) -> Boolean,
private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier
private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier,
private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
) : FullTransaction() {
init {
@ -124,7 +126,8 @@ private constructor(
componentGroups: List<ComponentGroup>? = null,
serializedInputs: List<SerializedStateAndRef>? = null,
serializedReferences: List<SerializedStateAndRef>? = null,
isAttachmentTrusted: (Attachment) -> Boolean
isAttachmentTrusted: (Attachment) -> Boolean,
attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
): LedgerTransaction {
return LedgerTransaction(
inputs = inputs,
@ -141,7 +144,8 @@ private constructor(
serializedInputs = protect(serializedInputs),
serializedReferences = protect(serializedReferences),
isAttachmentTrusted = isAttachmentTrusted,
verifierFactory = ::BasicVerifier
verifierFactory = ::BasicVerifier,
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
}
@ -176,7 +180,8 @@ private constructor(
serializedInputs = null,
serializedReferences = null,
isAttachmentTrusted = { true },
verifierFactory = ::BasicVerifier
verifierFactory = ::BasicVerifier,
attachmentsClassLoaderCache = null
)
}
}
@ -218,7 +223,8 @@ private constructor(
txAttachments,
getParamsWithGoo(),
id,
isAttachmentTrusted = isAttachmentTrusted) { transactionClassLoader ->
isAttachmentTrusted = isAttachmentTrusted,
attachmentsClassLoaderCache = attachmentsClassLoaderCache) { transactionClassLoader ->
// Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader].
// Only the copy will be used for verification, and the outer shell will be discarded.
// This artifice is required to preserve backwards compatibility.
@ -254,7 +260,8 @@ private constructor(
serializedInputs = serializedInputs,
serializedReferences = serializedReferences,
isAttachmentTrusted = isAttachmentTrusted,
verifierFactory = alternateVerifier
verifierFactory = alternateVerifier,
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
// Read network parameters with backwards compatibility goo.
@ -320,7 +327,8 @@ private constructor(
serializedInputs = serializedInputs,
serializedReferences = serializedReferences,
isAttachmentTrusted = isAttachmentTrusted,
verifierFactory = verifierFactory
verifierFactory = verifierFactory,
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
} else {
// This branch is only present for backwards compatibility.
@ -704,7 +712,8 @@ private constructor(
serializedInputs = null,
serializedReferences = null,
isAttachmentTrusted = { it.isUploaderTrusted() },
verifierFactory = ::BasicVerifier
verifierFactory = ::BasicVerifier,
attachmentsClassLoaderCache = null
)
@Deprecated("LedgerTransaction should not be created directly, use WireTransaction.toLedgerTransaction instead.")
@ -733,7 +742,8 @@ private constructor(
serializedInputs = null,
serializedReferences = null,
isAttachmentTrusted = { it.isUploaderTrusted() },
verifierFactory = ::BasicVerifier
verifierFactory = ::BasicVerifier,
attachmentsClassLoaderCache = null
)
@Deprecated("LedgerTransactions should not be created directly, use WireTransaction.toLedgerTransaction instead.")
@ -761,7 +771,8 @@ private constructor(
serializedInputs = serializedInputs,
serializedReferences = serializedReferences,
isAttachmentTrusted = isAttachmentTrusted,
verifierFactory = verifierFactory
verifierFactory = verifierFactory,
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
}
@ -791,7 +802,8 @@ private constructor(
serializedInputs = serializedInputs,
serializedReferences = serializedReferences,
isAttachmentTrusted = isAttachmentTrusted,
verifierFactory = verifierFactory
verifierFactory = verifierFactory,
attachmentsClassLoaderCache = attachmentsClassLoaderCache
)
}
}

17
core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt
View File

@ -15,6 +15,7 @@ import net.corda.core.node.ServicesForResolution
import net.corda.core.node.services.AttachmentId
import net.corda.core.serialization.CordaSerializable
import net.corda.core.serialization.SerializedBytes
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.serialization.serialize
import net.corda.core.utilities.OpaqueBytes
import java.security.PublicKey
@ -109,7 +110,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
services.networkParametersService.lookup(hashToResolve)
},
// `as?` is used due to [MockServices] not implementing [ServiceHubCoreInternal]
isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true }
isAttachmentTrusted = { (services as? ServiceHubCoreInternal)?.attachmentTrustCalculator?.calculate(it) ?: true },
attachmentsClassLoaderCache = (services as? ServiceHubCoreInternal)?.attachmentsClassLoaderCache
)
)
}
@ -145,7 +147,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
resolveAttachment,
{ stateRef -> resolveStateRef(stateRef)?.serialize() },
{ null },
{ it.isUploaderTrusted() }
{ it.isUploaderTrusted() },
null
)
}
@ -161,16 +164,19 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
resolveAttachment,
{ stateRef -> resolveStateRef(stateRef)?.serialize() },
resolveParameters,
{ true } // Any attachment loaded through the DJVM should be trusted
{ true }, // Any attachment loaded through the DJVM should be trusted
null
)
}
@Suppress("LongParameterList", "ThrowsCount")
private fun toLedgerTransactionInternal(
resolveIdentity: (PublicKey) -> Party?,
resolveAttachment: (SecureHash) -> Attachment?,
resolveStateRefAsSerialized: (StateRef) -> SerializedBytes<TransactionState<ContractState>>?,
resolveParameters: (SecureHash?) -> NetworkParameters?,
isAttachmentTrusted: (Attachment) -> Boolean
isAttachmentTrusted: (Attachment) -> Boolean,
attachmentsClassLoaderCache: AttachmentsClassLoaderCache?
): LedgerTransaction {
// Look up public keys to authenticated identities.
val authenticatedCommands = commands.lazyMapped { cmd, _ ->
@ -206,7 +212,8 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
componentGroups,
serializedResolvedInputs,
serializedResolvedReferences,
isAttachmentTrusted
isAttachmentTrusted,
attachmentsClassLoaderCache
)
checkTransactionSize(ltx, resolvedNetworkParameters.maxTransactionSize, serializedResolvedInputs, serializedResolvedReferences)

127
core/src/test/kotlin/net/corda/core/internal/ClassLoadingUtilsTest.kt
View File

@ -5,13 +5,20 @@ import net.corda.core.contracts.ContractAttachment
import net.corda.core.contracts.ContractClassName
import net.corda.core.crypto.SecureHash
import net.corda.core.identity.Party
import net.corda.core.node.services.AttachmentId
import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory
import net.corda.core.serialization.internal.AttachmentsClassLoader
import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.api.Assertions.assertThatThrownBy
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertSame
import org.junit.Ignore
import org.junit.Test
import java.io.ByteArrayOutputStream
import java.lang.ref.ReferenceQueue
import java.lang.ref.WeakReference
import java.net.URL
import java.net.URLClassLoader
import java.security.PublicKey
import java.util.jar.JarOutputStream
@ -120,9 +127,125 @@ class ClassLoadingUtilsTest {
}
}
private fun signedAttachment(data: ByteArray, vararg parties: Party) = ContractAttachment.create(
@Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.")
@Test(timeout=300_000)
@Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE")
fun `test weak reference removed from map`() {
val jarData = with(ByteArrayOutputStream()) {
val internalName = STANDALONE_CLASS_NAME.asInternalName
JarOutputStream(this, Manifest()).use {
it.setLevel(NO_COMPRESSION)
it.setMethod(DEFLATED)
it.putNextEntry(directoryEntry("com"))
it.putNextEntry(directoryEntry("com/example"))
it.putNextEntry(classEntry(internalName))
it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName))
}
toByteArray()
}
val attachment = signedAttachment(jarData)
var url: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment)
val referenceQueue: ReferenceQueue<URL> = ReferenceQueue()
val weakReference = WeakReference<URL>(url, referenceQueue)
assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
// Clear strong reference
url = null
System.gc()
val ref = referenceQueue.remove(100000)
assertSame(weakReference, ref)
assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
}
@Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.")
@Test(timeout=300_000)
@Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE")
fun `test adding same attachment twice then removing`() {
val jarData = with(ByteArrayOutputStream()) {
val internalName = STANDALONE_CLASS_NAME.asInternalName
JarOutputStream(this, Manifest()).use {
it.setLevel(NO_COMPRESSION)
it.setMethod(DEFLATED)
it.putNextEntry(directoryEntry("com"))
it.putNextEntry(directoryEntry("com/example"))
it.putNextEntry(classEntry(internalName))
it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName))
}
toByteArray()
}
val attachment1 = signedAttachment(jarData)
val attachment2 = signedAttachment(jarData)
var url1: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment1)
var url2: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment2)
val referenceQueue1: ReferenceQueue<URL> = ReferenceQueue()
val weakReference1 = WeakReference<URL>(url1, referenceQueue1)
val referenceQueue2: ReferenceQueue<URL> = ReferenceQueue()
val weakReference2 = WeakReference<URL>(url2, referenceQueue2)
assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
url1 = null
System.gc()
val ref1 = referenceQueue1.remove(500)
assertNull(ref1)
assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
url2 = null
System.gc()
val ref2 = referenceQueue2.remove(100000)
assertSame(weakReference2, ref2)
assertSame(weakReference1, referenceQueue1.poll())
assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
}
@Ignore("Using System.gc in this test which has no guarantees when/if gc occurs.")
@Test(timeout=300_000)
@Suppress("ExplicitGarbageCollectionCall", "UNUSED_VALUE")
fun `test adding two different attachments then removing`() {
val jarData1 = with(ByteArrayOutputStream()) {
val internalName = STANDALONE_CLASS_NAME.asInternalName
JarOutputStream(this, Manifest()).use {
it.setLevel(NO_COMPRESSION)
it.setMethod(DEFLATED)
it.putNextEntry(directoryEntry("com"))
it.putNextEntry(directoryEntry("com/example"))
it.putNextEntry(classEntry(internalName))
it.write(TemplateClassWithEmptyConstructor::class.java.renameTo(internalName))
}
toByteArray()
}
val attachment1 = signedAttachment(jarData1)
val attachment2 = signedAttachment(jarData1, id = SecureHash.randomSHA256())
var url1: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment1)
var url2: URL? = AttachmentURLStreamHandlerFactory.toUrl(attachment2)
val referenceQueue1: ReferenceQueue<URL> = ReferenceQueue()
val weakReference1 = WeakReference<URL>(url1, referenceQueue1)
val referenceQueue2: ReferenceQueue<URL> = ReferenceQueue()
val weakReference2 = WeakReference<URL>(url2, referenceQueue2)
assertEquals(2, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
url1 = null
System.gc()
val ref1 = referenceQueue1.remove(100000)
assertSame(weakReference1, ref1)
assertEquals(1, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
url2 = null
System.gc()
val ref2 = referenceQueue2.remove(100000)
assertSame(weakReference2, ref2)
assertEquals(0, AttachmentURLStreamHandlerFactory.loadedAttachmentsSize())
}
private fun signedAttachment(data: ByteArray, id: AttachmentId = contractAttachmentId,
vararg parties: Party) = ContractAttachment.create(
object : AbstractAttachment({ data }, "test") {
override val id: SecureHash get() = contractAttachmentId
override val id: SecureHash get() = id
override val signerKeys: List<PublicKey> get() = parties.map(Party::owningKey)
}, PROGRAM_ID, signerKeys = parties.map(Party::owningKey)

7
core/src/test/kotlin/net/corda/core/internal/internalAccessTestHelpers.kt
View File

@ -4,6 +4,7 @@ import net.corda.core.contracts.*
import net.corda.core.crypto.SecureHash
import net.corda.core.identity.Party
import net.corda.core.node.NetworkParameters
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.transactions.ComponentGroup
import net.corda.core.transactions.LedgerTransaction
import net.corda.core.transactions.WireTransaction
@ -17,6 +18,7 @@ fun WireTransaction.accessGroupHashes() = this.groupHashes
fun WireTransaction.accessGroupMerkleRoots() = this.groupsMerkleRoots
fun WireTransaction.accessAvailableComponentHashes() = this.availableComponentHashes
@Suppress("LongParameterList")
fun createLedgerTransaction(
inputs: List<StateAndRef<ContractState>>,
outputs: List<TransactionState<ContractState>>,
@ -31,8 +33,9 @@ fun createLedgerTransaction(
componentGroups: List<ComponentGroup>? = null,
serializedInputs: List<SerializedStateAndRef>? = null,
serializedReferences: List<SerializedStateAndRef>? = null,
isAttachmentTrusted: (Attachment) -> Boolean
): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted)
isAttachmentTrusted: (Attachment) -> Boolean,
attachmentsClassLoaderCache: AttachmentsClassLoaderCache
): LedgerTransaction = LedgerTransaction.create(inputs, outputs, commands, attachments, id, notary, timeWindow, privacySalt, networkParameters, references, componentGroups, serializedInputs, serializedReferences, isAttachmentTrusted, attachmentsClassLoaderCache)
fun createContractCreationError(txId: SecureHash, contractClass: String, cause: Throwable) = TransactionVerificationException.ContractCreationError(txId, contractClass, cause)
fun createContractRejection(txId: SecureHash, contract: Contract, cause: Throwable) = TransactionVerificationException.ContractRejection(txId, contract, cause)

5
node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt
View File

@ -57,6 +57,8 @@ import net.corda.core.schemas.MappedSchema
import net.corda.core.serialization.SerializationWhitelist
import net.corda.core.serialization.SerializeAsToken
import net.corda.core.serialization.SingletonSerializeAsToken
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
import net.corda.core.transactions.LedgerTransaction
import net.corda.core.utilities.NetworkHostAndPort
import net.corda.core.utilities.days
@ -315,6 +317,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
} else {
BasicVerifierFactoryService()
}
private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory).tokenize()
val contractUpgradeService = ContractUpgradeServiceImpl(cacheFactory).tokenize()
val auditService = DummyAuditService().tokenize()
@Suppress("LeakingThis")
@ -1144,6 +1147,8 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
private lateinit var _myInfo: NodeInfo
override val myInfo: NodeInfo get() = _myInfo
override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache get() = this@AbstractNode.attachmentsClassLoaderCache
private lateinit var _networkParameters: NetworkParameters
override val networkParameters: NetworkParameters get() = _networkParameters

1
node/src/main/kotlin/net/corda/node/migration/MigrationNamedCacheFactory.kt
View File

@ -37,6 +37,7 @@ class MigrationNamedCacheFactory(private val metricRegistry: MetricRegistry?,
"NodeAttachmentService_contractAttachmentVersions" -> caffeine.maximumSize(defaultCacheSize)
"NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize)
"NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize)
"AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultCacheSize)
else -> throw IllegalArgumentException("Unexpected cache name $name.")
}
}

7
node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt
View File

@ -15,6 +15,8 @@ import net.corda.core.node.services.NetworkParametersService
import net.corda.core.node.services.TransactionStorage
import net.corda.core.serialization.deserialize
import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
import net.corda.core.transactions.ContractUpgradeLedgerTransaction
import net.corda.core.transactions.NotaryChangeLedgerTransaction
import net.corda.core.transactions.WireTransaction
@ -62,6 +64,8 @@ class MigrationServicesForResolution(
cacheFactory
)
private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory)
private fun defaultNetworkParameters(): NetworkParameters {
logger.warn("Using a dummy set of network parameters for migration.")
val clock = Clock.systemUTC()
@ -124,7 +128,8 @@ class MigrationServicesForResolution(
networkParameters,
tx.id,
attachmentTrustCalculator::calculate,
cordappLoader.appClassLoader) {
cordappLoader.appClassLoader,
attachmentsClassLoaderCache) {
deserialiseComponentGroup(tx.componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true)
}
states.filterIndexed {index, _ -> stateIndices.contains(index)}.toList()

5
node/src/main/kotlin/net/corda/node/utilities/NodeNamedCache.kt
View File

@ -63,6 +63,7 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi
name == "NodeParametersStorage_networkParametersByHash" -> caffeine.maximumSize(defaultCacheSize)
name == "PublicKeyToOwningIdentityCache_cache" -> caffeine.maximumSize(defaultCacheSize)
name == "NodeAttachmentTrustCalculator_trustedKeysCache" -> caffeine.maximumSize(defaultCacheSize)
name == "AttachmentsClassLoader_cache" -> caffeine.maximumSize(defaultAttachmentsClassLoaderCacheSize)
else -> throw IllegalArgumentException("Unexpected cache name $name. Did you add a new cache?")
}
}
@ -85,4 +86,6 @@ open class DefaultNamedCacheFactory protected constructor(private val metricRegi
}
open protected val defaultCacheSize = 1024L
}
private val defaultAttachmentsClassLoaderCacheSize = defaultCacheSize / CACHE_SIZE_DENOMINATOR
}
private const val CACHE_SIZE_DENOMINATOR = 4L

8
node/src/test/kotlin/net/corda/node/services/FinalityHandlerTest.kt
View File

@ -50,13 +50,13 @@ class FinalityHandlerTest {
getOrThrow()
}
bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId)
bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId)
assertThat(bob.getTransaction(stx.id)).isNull()
bob = mockNet.restartNode(bob)
// Since we've not done anything to fix the orignal error, we expect the finality handler to be sent to the hospital
// again on restart
bob.assertFlowSentForObservationDueToConstraintError(finalityHandlerId)
bob.assertFlowSentForObservationDueToUntrustedAttachmentsException(finalityHandlerId)
assertThat(bob.getTransaction(stx.id)).isNull()
}
@ -96,7 +96,7 @@ class FinalityHandlerTest {
.ofType(R::class.java)
}
private fun TestStartedNode.assertFlowSentForObservationDueToConstraintError(runId: StateMachineRunId) {
private fun TestStartedNode.assertFlowSentForObservationDueToUntrustedAttachmentsException(runId: StateMachineRunId) {
val observation = medicalRecordsOfType<MedicalRecord.Flow>()
.filter { it.flowId == runId }
.toBlocking()
@ -104,7 +104,7 @@ class FinalityHandlerTest {
assertThat(observation.outcome).isEqualTo(Outcome.OVERNIGHT_OBSERVATION)
assertThat(observation.by).contains(FinalityDoctor)
val error = observation.errors.single()
assertThat(error).isInstanceOf(TransactionVerificationException.ContractConstraintRejection::class.java)
assertThat(error).isInstanceOf(TransactionVerificationException.UntrustedAttachmentsException::class.java)
}
private fun TestStartedNode.getTransaction(id: SecureHash): SignedTransaction? {

4
testing/test-utils/src/main/kotlin/net/corda/testing/dsl/TestDSL.kt
View File

@ -13,6 +13,8 @@ import net.corda.core.node.ServiceHub
import net.corda.core.node.ServicesForResolution
import net.corda.core.node.services.AttachmentId
import net.corda.core.node.services.TransactionStorage
import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
import net.corda.core.transactions.SignedTransaction
import net.corda.core.transactions.TransactionBuilder
import net.corda.core.transactions.WireTransaction
@ -127,6 +129,8 @@ data class TestTransactionDSLInterpreter private constructor(
override val cordappProvider: CordappProvider =
ledgerInterpreter.services.cordappProvider
override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(TestingNamedCacheFactory())
}
private fun copy(): TestTransactionDSLInterpreter =

1
testing/test-utils/src/main/kotlin/net/corda/testing/internal/TestingNamedCacheFactory.kt
View File

@ -26,6 +26,7 @@ class TestingNamedCacheFactory private constructor(private val sizeOverride: Lon
val configuredCaffeine = when (name) {
"DBTransactionStorage_transactions" -> caffeine.maximumWeight(1.MB)
"NodeAttachmentService_attachmentContent" -> caffeine.maximumWeight(1.MB)
"AttachmentsClassLoader_cache" -> caffeine.maximumSize(sizeOverride)
else -> caffeine.maximumSize(sizeOverride)
}
return configuredCaffeine.build<K, V>(loader)