mirror of
https://github.com/corda/corda.git
synced 2024-12-24 07:06:44 +00:00
Merge pull request #7270 from corda/cc/merge/os-4.9/os-4.10/21-Nov-2022
ENT-8796 - cc/merge/os 4.9/os 4.10/21 nov 2022
This commit is contained in:
commit
4d14d718d5
2
.ci/dev/regression/Jenkinsfile
vendored
2
.ci/dev/regression/Jenkinsfile
vendored
@ -194,7 +194,7 @@ pipeline {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
stage('Snyk Security') {
|
stage('Snyk Security') {
|
||||||
when {
|
when {
|
||||||
expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
|
expression { isReleaseTag || isReleaseCandidate || isReleaseBranch }
|
||||||
}
|
}
|
||||||
|
@ -24,8 +24,9 @@ import net.corda.core.node.NetworkParameters
|
|||||||
import net.corda.core.node.services.AttachmentId
|
import net.corda.core.node.services.AttachmentId
|
||||||
import net.corda.core.serialization.internal.AttachmentsClassLoader
|
import net.corda.core.serialization.internal.AttachmentsClassLoader
|
||||||
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
|
import net.corda.core.serialization.internal.AttachmentsClassLoaderCacheImpl
|
||||||
import net.corda.testing.common.internal.testNetworkParameters
|
import net.corda.core.transactions.LedgerTransaction
|
||||||
import net.corda.node.services.attachments.NodeAttachmentTrustCalculator
|
import net.corda.node.services.attachments.NodeAttachmentTrustCalculator
|
||||||
|
import net.corda.testing.common.internal.testNetworkParameters
|
||||||
import net.corda.testing.contracts.DummyContract
|
import net.corda.testing.contracts.DummyContract
|
||||||
import net.corda.testing.core.ALICE_NAME
|
import net.corda.testing.core.ALICE_NAME
|
||||||
import net.corda.testing.core.BOB_NAME
|
import net.corda.testing.core.BOB_NAME
|
||||||
@ -74,7 +75,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
val BOB = TestIdentity(BOB_NAME, 80).party
|
val BOB = TestIdentity(BOB_NAME, 80).party
|
||||||
val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20)
|
val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20)
|
||||||
val DUMMY_NOTARY get() = dummyNotary.party
|
val DUMMY_NOTARY get() = dummyNotary.party
|
||||||
val PROGRAM_ID: String = "net.corda.testing.contracts.MyDummyContract"
|
const val PROGRAM_ID = "net.corda.testing.contracts.MyDummyContract"
|
||||||
}
|
}
|
||||||
|
|
||||||
@Rule
|
@Rule
|
||||||
@ -89,7 +90,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
private lateinit var internalStorage: InternalMockAttachmentStorage
|
private lateinit var internalStorage: InternalMockAttachmentStorage
|
||||||
private lateinit var attachmentTrustCalculator: AttachmentTrustCalculator
|
private lateinit var attachmentTrustCalculator: AttachmentTrustCalculator
|
||||||
private val networkParameters = testNetworkParameters()
|
private val networkParameters = testNetworkParameters()
|
||||||
private val cacheFactory = TestingNamedCacheFactory()
|
private val cacheFactory = TestingNamedCacheFactory(1)
|
||||||
|
|
||||||
private fun createClassloader(
|
private fun createClassloader(
|
||||||
attachment: AttachmentId,
|
attachment: AttachmentId,
|
||||||
@ -541,6 +542,50 @@ class AttachmentsClassLoaderTests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test(timeout=300_000)
|
||||||
|
fun `class loader not closed after cache starts evicting`() {
|
||||||
|
tempFolder.root.toPath().let { path ->
|
||||||
|
val transactions = mutableListOf<LedgerTransaction>()
|
||||||
|
val iterations = 10
|
||||||
|
|
||||||
|
val baseOutState = TransactionState(DummyContract.SingleOwnerState(0, ALICE), PROGRAM_ID, DUMMY_NOTARY, constraint = AlwaysAcceptAttachmentConstraint)
|
||||||
|
val inputs = emptyList<StateAndRef<*>>()
|
||||||
|
val outputs = listOf(baseOutState, baseOutState.copy(notary = ALICE), baseOutState.copy(notary = BOB))
|
||||||
|
val commands = emptyList<CommandWithParties<CommandData>>()
|
||||||
|
val content = createContractString(PROGRAM_ID)
|
||||||
|
val timeWindow: TimeWindow? = null
|
||||||
|
val attachmentsClassLoaderCache = AttachmentsClassLoaderCacheImpl(cacheFactory)
|
||||||
|
val contractJarPath = ContractJarTestUtils.makeTestContractJar(path, PROGRAM_ID, content = content)
|
||||||
|
val attachments = createAttachments(contractJarPath)
|
||||||
|
|
||||||
|
for(i in 1 .. iterations) {
|
||||||
|
val id = SecureHash.randomSHA256()
|
||||||
|
val privacySalt = PrivacySalt()
|
||||||
|
val transaction = createLedgerTransaction(
|
||||||
|
inputs,
|
||||||
|
outputs,
|
||||||
|
commands,
|
||||||
|
attachments,
|
||||||
|
id,
|
||||||
|
null,
|
||||||
|
timeWindow,
|
||||||
|
privacySalt,
|
||||||
|
testNetworkParameters(),
|
||||||
|
emptyList(),
|
||||||
|
isAttachmentTrusted = { true },
|
||||||
|
attachmentsClassLoaderCache = attachmentsClassLoaderCache
|
||||||
|
)
|
||||||
|
transactions.add(transaction)
|
||||||
|
System.gc()
|
||||||
|
Thread.sleep(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
transactions.forEach {
|
||||||
|
it.verify()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private fun createContractString(contractName: String, versionSeed: Int = 0): String {
|
private fun createContractString(contractName: String, versionSeed: Int = 0): String {
|
||||||
val pkgs = contractName.split(".")
|
val pkgs = contractName.split(".")
|
||||||
val className = pkgs.last()
|
val className = pkgs.last()
|
||||||
@ -563,7 +608,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
}
|
}
|
||||||
""".trimIndent()
|
""".trimIndent()
|
||||||
|
|
||||||
System.out.println(output)
|
println(output)
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -571,6 +616,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
|
|
||||||
val attachment = object : AbstractAttachment({contractJarPath.inputStream().readBytes()}, uploader = "app") {
|
val attachment = object : AbstractAttachment({contractJarPath.inputStream().readBytes()}, uploader = "app") {
|
||||||
@Suppress("OverridingDeprecatedMember")
|
@Suppress("OverridingDeprecatedMember")
|
||||||
|
@Deprecated("Use signerKeys. There is no requirement that attachment signers are Corda parties.")
|
||||||
override val signers: List<Party> = emptyList()
|
override val signers: List<Party> = emptyList()
|
||||||
override val signerKeys: List<PublicKey> = emptyList()
|
override val signerKeys: List<PublicKey> = emptyList()
|
||||||
override val size: Int = 1234
|
override val size: Int = 1234
|
||||||
@ -581,6 +627,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
return listOf(
|
return listOf(
|
||||||
object : AbstractAttachment({ISOLATED_CONTRACTS_JAR_PATH.openStream().readBytes()}, uploader = "app") {
|
object : AbstractAttachment({ISOLATED_CONTRACTS_JAR_PATH.openStream().readBytes()}, uploader = "app") {
|
||||||
@Suppress("OverridingDeprecatedMember")
|
@Suppress("OverridingDeprecatedMember")
|
||||||
|
@Deprecated("Use signerKeys. There is no requirement that attachment signers are Corda parties.")
|
||||||
override val signers: List<Party> = emptyList()
|
override val signers: List<Party> = emptyList()
|
||||||
override val signerKeys: List<PublicKey> = emptyList()
|
override val signerKeys: List<PublicKey> = emptyList()
|
||||||
override val size: Int = 1234
|
override val size: Int = 1234
|
||||||
@ -589,6 +636,7 @@ class AttachmentsClassLoaderTests {
|
|||||||
object : AbstractAttachment({fakeAttachment("importantDoc.pdf", "I am a pdf!").inputStream().readBytes()
|
object : AbstractAttachment({fakeAttachment("importantDoc.pdf", "I am a pdf!").inputStream().readBytes()
|
||||||
}, uploader = "app") {
|
}, uploader = "app") {
|
||||||
@Suppress("OverridingDeprecatedMember")
|
@Suppress("OverridingDeprecatedMember")
|
||||||
|
@Deprecated("Use signerKeys. There is no requirement that attachment signers are Corda parties.")
|
||||||
override val signers: List<Party> = emptyList()
|
override val signers: List<Party> = emptyList()
|
||||||
override val signerKeys: List<PublicKey> = emptyList()
|
override val signerKeys: List<PublicKey> = emptyList()
|
||||||
override val size: Int = 1234
|
override val size: Int = 1234
|
||||||
|
@ -16,9 +16,9 @@ import net.corda.core.contracts.TransactionState
|
|||||||
import net.corda.core.contracts.TransactionVerificationException
|
import net.corda.core.contracts.TransactionVerificationException
|
||||||
import net.corda.core.contracts.TransactionVerificationException.ConflictingAttachmentsRejection
|
import net.corda.core.contracts.TransactionVerificationException.ConflictingAttachmentsRejection
|
||||||
import net.corda.core.contracts.TransactionVerificationException.ConstraintPropagationRejection
|
import net.corda.core.contracts.TransactionVerificationException.ConstraintPropagationRejection
|
||||||
|
import net.corda.core.contracts.TransactionVerificationException.ContractConstraintRejection
|
||||||
import net.corda.core.contracts.TransactionVerificationException.ContractCreationError
|
import net.corda.core.contracts.TransactionVerificationException.ContractCreationError
|
||||||
import net.corda.core.contracts.TransactionVerificationException.ContractRejection
|
import net.corda.core.contracts.TransactionVerificationException.ContractRejection
|
||||||
import net.corda.core.contracts.TransactionVerificationException.ContractConstraintRejection
|
|
||||||
import net.corda.core.contracts.TransactionVerificationException.Direction
|
import net.corda.core.contracts.TransactionVerificationException.Direction
|
||||||
import net.corda.core.contracts.TransactionVerificationException.DuplicateAttachmentsRejection
|
import net.corda.core.contracts.TransactionVerificationException.DuplicateAttachmentsRejection
|
||||||
import net.corda.core.contracts.TransactionVerificationException.InvalidConstraintRejection
|
import net.corda.core.contracts.TransactionVerificationException.InvalidConstraintRejection
|
||||||
|
@ -35,6 +35,7 @@ import net.corda.core.utilities.debug
|
|||||||
import net.corda.core.utilities.loggerFor
|
import net.corda.core.utilities.loggerFor
|
||||||
import java.io.IOException
|
import java.io.IOException
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
|
import java.lang.ref.ReferenceQueue
|
||||||
import java.lang.ref.WeakReference
|
import java.lang.ref.WeakReference
|
||||||
import java.net.URL
|
import java.net.URL
|
||||||
import java.net.URLClassLoader
|
import java.net.URLClassLoader
|
||||||
@ -46,6 +47,8 @@ import java.security.Permission
|
|||||||
import java.util.Locale
|
import java.util.Locale
|
||||||
import java.util.ServiceLoader
|
import java.util.ServiceLoader
|
||||||
import java.util.WeakHashMap
|
import java.util.WeakHashMap
|
||||||
|
import java.util.concurrent.ConcurrentHashMap
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
import java.util.function.Function
|
import java.util.function.Function
|
||||||
import kotlin.collections.component1
|
import kotlin.collections.component1
|
||||||
import kotlin.collections.component2
|
import kotlin.collections.component2
|
||||||
@ -336,6 +339,7 @@ class AttachmentsClassLoader(attachments: List<Attachment>,
|
|||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
object AttachmentsClassLoaderBuilder {
|
object AttachmentsClassLoaderBuilder {
|
||||||
private const val CACHE_SIZE = 16
|
private const val CACHE_SIZE = 16
|
||||||
|
private const val STRONG_REFERENCE_TO_CACHED_SERIALIZATION_CONTEXT = "cachedSerializationContext"
|
||||||
|
|
||||||
private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE)
|
private val fallBackCache: AttachmentsClassLoaderCache = AttachmentsClassLoaderSimpleCacheImpl(CACHE_SIZE)
|
||||||
|
|
||||||
@ -355,16 +359,15 @@ object AttachmentsClassLoaderBuilder {
|
|||||||
val attachmentIds = attachments.mapTo(LinkedHashSet(), Attachment::id)
|
val attachmentIds = attachments.mapTo(LinkedHashSet(), Attachment::id)
|
||||||
|
|
||||||
val cache = attachmentsClassLoaderCache ?: fallBackCache
|
val cache = attachmentsClassLoaderCache ?: fallBackCache
|
||||||
val serializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function { key ->
|
val cachedSerializationContext = cache.computeIfAbsent(AttachmentsClassLoaderKey(attachmentIds, params), Function { key ->
|
||||||
// Create classloader and load serializers, whitelisted classes
|
// Create classloader and load serializers, whitelisted classes
|
||||||
val transactionClassLoader = AttachmentsClassLoader(attachments, key.params, txId, isAttachmentTrusted, parent)
|
val transactionClassLoader = AttachmentsClassLoader(attachments, key.params, txId, isAttachmentTrusted, parent)
|
||||||
val serializers = try {
|
val serializers = try {
|
||||||
createInstancesOfClassesImplementing(transactionClassLoader, SerializationCustomSerializer::class.java,
|
createInstancesOfClassesImplementing(transactionClassLoader, SerializationCustomSerializer::class.java,
|
||||||
JDK1_2_CLASS_FILE_FORMAT_MAJOR_VERSION..JDK8_CLASS_FILE_FORMAT_MAJOR_VERSION)
|
JDK1_2_CLASS_FILE_FORMAT_MAJOR_VERSION..JDK8_CLASS_FILE_FORMAT_MAJOR_VERSION)
|
||||||
}
|
} catch (ex: UnsupportedClassVersionError) {
|
||||||
catch(ex: UnsupportedClassVersionError) {
|
throw TransactionVerificationException.UnsupportedClassVersionError(txId, ex.message!!, ex)
|
||||||
throw TransactionVerificationException.UnsupportedClassVersionError(txId, ex.message!!, ex)
|
}
|
||||||
}
|
|
||||||
val whitelistedClasses = ServiceLoader.load(SerializationWhitelist::class.java, transactionClassLoader)
|
val whitelistedClasses = ServiceLoader.load(SerializationWhitelist::class.java, transactionClassLoader)
|
||||||
.flatMap(SerializationWhitelist::whitelist)
|
.flatMap(SerializationWhitelist::whitelist)
|
||||||
|
|
||||||
@ -378,11 +381,17 @@ object AttachmentsClassLoaderBuilder {
|
|||||||
.withWhitelist(whitelistedClasses)
|
.withWhitelist(whitelistedClasses)
|
||||||
.withCustomSerializers(serializers)
|
.withCustomSerializers(serializers)
|
||||||
.withoutCarpenter()
|
.withoutCarpenter()
|
||||||
}).withProperties(mapOf<Any, Any>(
|
})
|
||||||
// Duplicate the SerializationContext from the cache and give
|
|
||||||
// it these extra properties, just for this transaction.
|
val serializationContext = cachedSerializationContext.withProperties(mapOf<Any, Any>(
|
||||||
AMQP_ENVELOPE_CACHE_PROPERTY to HashMap<Any, Any>(AMQP_ENVELOPE_CACHE_INITIAL_CAPACITY),
|
// Duplicate the SerializationContext from the cache and give
|
||||||
DESERIALIZATION_CACHE_PROPERTY to HashMap<Any, Any>()
|
// it these extra properties, just for this transaction.
|
||||||
|
// However, keep a strong reference to the cached SerializationContext so we can
|
||||||
|
// leverage the power of WeakReferences in the AttachmentsClassLoaderCacheImpl to figure
|
||||||
|
// out when all these have gone out of scope by the BasicVerifier going out of scope.
|
||||||
|
AMQP_ENVELOPE_CACHE_PROPERTY to HashMap<Any, Any>(AMQP_ENVELOPE_CACHE_INITIAL_CAPACITY),
|
||||||
|
DESERIALIZATION_CACHE_PROPERTY to HashMap<Any, Any>(),
|
||||||
|
STRONG_REFERENCE_TO_CACHED_SERIALIZATION_CONTEXT to cachedSerializationContext
|
||||||
))
|
))
|
||||||
|
|
||||||
// Deserialize all relevant classes in the transaction classloader.
|
// Deserialize all relevant classes in the transaction classloader.
|
||||||
@ -399,6 +408,8 @@ object AttachmentsClassLoaderBuilder {
|
|||||||
object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory {
|
object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory {
|
||||||
internal const val attachmentScheme = "attachment"
|
internal const val attachmentScheme = "attachment"
|
||||||
|
|
||||||
|
private val uniqueness = AtomicLong(0)
|
||||||
|
|
||||||
private val loadedAttachments: AttachmentsHolder = AttachmentsHolderImpl()
|
private val loadedAttachments: AttachmentsHolder = AttachmentsHolderImpl()
|
||||||
|
|
||||||
override fun createURLStreamHandler(protocol: String): URLStreamHandler? {
|
override fun createURLStreamHandler(protocol: String): URLStreamHandler? {
|
||||||
@ -409,14 +420,9 @@ object AttachmentURLStreamHandlerFactory : URLStreamHandlerFactory {
|
|||||||
|
|
||||||
@Synchronized
|
@Synchronized
|
||||||
fun toUrl(attachment: Attachment): URL {
|
fun toUrl(attachment: Attachment): URL {
|
||||||
val proposedURL = URL(attachmentScheme, "", -1, attachment.id.toString(), AttachmentURLStreamHandler)
|
val uniqueURL = URL(attachmentScheme, "", -1, attachment.id.toString()+ "?" + uniqueness.getAndIncrement(), AttachmentURLStreamHandler)
|
||||||
val existingURL = loadedAttachments.getKey(proposedURL)
|
loadedAttachments[uniqueURL] = attachment
|
||||||
return if (existingURL == null) {
|
return uniqueURL
|
||||||
loadedAttachments[proposedURL] = attachment
|
|
||||||
proposedURL
|
|
||||||
} else {
|
|
||||||
existingURL
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@VisibleForTesting
|
@VisibleForTesting
|
||||||
@ -474,20 +480,52 @@ interface AttachmentsClassLoaderCache {
|
|||||||
@DeleteForDJVM
|
@DeleteForDJVM
|
||||||
class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache {
|
class AttachmentsClassLoaderCacheImpl(cacheFactory: NamedCacheFactory) : SingletonSerializeAsToken(), AttachmentsClassLoaderCache {
|
||||||
|
|
||||||
private val cache: Cache<AttachmentsClassLoaderKey, SerializationContext> = cacheFactory.buildNamed(
|
private class ToBeClosed(
|
||||||
// Close deserialization classloaders when we evict them
|
serializationContext: SerializationContext,
|
||||||
// to release any resources they may be holding.
|
val classLoaderToClose: AutoCloseable,
|
||||||
@Suppress("TooGenericExceptionCaught")
|
val cacheKey: AttachmentsClassLoaderKey,
|
||||||
Caffeine.newBuilder().removalListener { key, context, _ ->
|
queue: ReferenceQueue<SerializationContext>
|
||||||
try {
|
) : WeakReference<SerializationContext>(serializationContext, queue)
|
||||||
(context?.deserializationClassLoader as? AutoCloseable)?.close()
|
|
||||||
} catch (e: Exception) {
|
private val logger = loggerFor<AttachmentsClassLoaderCacheImpl>()
|
||||||
loggerFor<AttachmentsClassLoaderCacheImpl>().warn("Error destroying serialization context for $key", e)
|
private val toBeClosed = ConcurrentHashMap.newKeySet<ToBeClosed>()
|
||||||
|
private val expiryQueue = ReferenceQueue<SerializationContext>()
|
||||||
|
|
||||||
|
@Suppress("TooGenericExceptionCaught")
|
||||||
|
private fun purgeExpiryQueue() {
|
||||||
|
// Close the AttachmentsClassLoader for every SerializationContext
|
||||||
|
// that has already been garbage-collected.
|
||||||
|
while (true) {
|
||||||
|
val head = expiryQueue.poll() as? ToBeClosed ?: break
|
||||||
|
if (!toBeClosed.remove(head)) {
|
||||||
|
logger.warn("Reaped unexpected serialization context for {}", head.cacheKey)
|
||||||
}
|
}
|
||||||
}, "AttachmentsClassLoader_cache"
|
|
||||||
|
try {
|
||||||
|
head.classLoaderToClose.close()
|
||||||
|
} catch (e: Exception) {
|
||||||
|
logger.warn("Error destroying serialization context for ${head.cacheKey}", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val cache: Cache<AttachmentsClassLoaderKey, SerializationContext> = cacheFactory.buildNamed(
|
||||||
|
// Schedule for closing the deserialization classloaders when we evict them
|
||||||
|
// to release any resources they may be holding.
|
||||||
|
Caffeine.newBuilder().removalListener { key, context, _ ->
|
||||||
|
(context?.deserializationClassLoader as? AutoCloseable)?.also { autoCloseable ->
|
||||||
|
// ClassLoader to be closed once the BasicVerifier, which has a strong
|
||||||
|
// reference chain to this SerializationContext, has gone out of scope.
|
||||||
|
toBeClosed += ToBeClosed(context, autoCloseable, key!!, expiryQueue)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reap any entries which have been garbage-collected.
|
||||||
|
purgeExpiryQueue()
|
||||||
|
}, "AttachmentsClassLoader_cache"
|
||||||
)
|
)
|
||||||
|
|
||||||
override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
|
override fun computeIfAbsent(key: AttachmentsClassLoaderKey, mappingFunction: Function<in AttachmentsClassLoaderKey, out SerializationContext>): SerializationContext {
|
||||||
|
purgeExpiryQueue()
|
||||||
return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function")
|
return cache.get(key, mappingFunction) ?: throw NullPointerException("null returned from cache mapping function")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user