mirror of
https://github.com/corda/corda.git
synced 2024-12-18 20:47:57 +00:00
ENT-3000 Start pooling classpath scanning and related fixes (#4664)
* Start pooling classpath scanning Quickly patch synchronisation of attachment class loader cache. Needs a revisit but more complicated due to DJVM. Annotate away for DJVM Take ClassGraph utils into their own file so we can exclude for DJVM Clean up a little * Daemonize the threads * Seems to be some concurrency problems with use of ClassGraph. Using a mutex for now to work around.
This commit is contained in:
parent
9ade410109
commit
7dc7313fb7
@ -0,0 +1,19 @@
|
|||||||
|
@file:DeleteForDJVM
|
||||||
|
|
||||||
|
package net.corda.core.internal
|
||||||
|
|
||||||
|
import co.paralleluniverse.strands.concurrent.ReentrantLock
|
||||||
|
import io.github.classgraph.ClassGraph
|
||||||
|
import io.github.classgraph.ScanResult
|
||||||
|
import net.corda.core.DeleteForDJVM
|
||||||
|
import kotlin.concurrent.withLock
|
||||||
|
|
||||||
|
private val pooledScanMutex = ReentrantLock()
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Use this rather than the built in implementation of [scan] on [ClassGraph]. The built in implementation of [scan] creates
|
||||||
|
* a thread pool every time resulting in too many threads. This one uses a mutex to restrict concurrency.
|
||||||
|
*/
|
||||||
|
fun ClassGraph.pooledScan(): ScanResult {
|
||||||
|
return pooledScanMutex.withLock { this@pooledScan.scan() }
|
||||||
|
}
|
@ -19,7 +19,7 @@ import net.corda.core.StubOutForDJVM
|
|||||||
fun <T: Any> createInstancesOfClassesImplementing(classloader: ClassLoader, clazz: Class<T>): Set<T> {
|
fun <T: Any> createInstancesOfClassesImplementing(classloader: ClassLoader, clazz: Class<T>): Set<T> {
|
||||||
return ClassGraph().addClassLoader(classloader)
|
return ClassGraph().addClassLoader(classloader)
|
||||||
.enableClassInfo()
|
.enableClassInfo()
|
||||||
.scan()
|
.pooledScan()
|
||||||
.use {
|
.use {
|
||||||
it.getClassesImplementing(clazz.name)
|
it.getClassesImplementing(clazz.name)
|
||||||
.filterNot { it.isAbstract }
|
.filterNot { it.isAbstract }
|
||||||
@ -38,4 +38,4 @@ fun <T: Any?> executeWithThreadContextClassLoader(classloader: ClassLoader, fn:
|
|||||||
Thread.currentThread().contextClassLoader = threadClassLoader
|
Thread.currentThread().contextClassLoader = threadClassLoader
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -2,7 +2,6 @@ package net.corda.core.serialization.internal
|
|||||||
|
|
||||||
import net.corda.core.CordaException
|
import net.corda.core.CordaException
|
||||||
import net.corda.core.KeepForDJVM
|
import net.corda.core.KeepForDJVM
|
||||||
import net.corda.core.internal.createInstancesOfClassesImplementing
|
|
||||||
import net.corda.core.contracts.Attachment
|
import net.corda.core.contracts.Attachment
|
||||||
import net.corda.core.contracts.ContractAttachment
|
import net.corda.core.contracts.ContractAttachment
|
||||||
import net.corda.core.contracts.TransactionVerificationException.OverlappingAttachmentsException
|
import net.corda.core.contracts.TransactionVerificationException.OverlappingAttachmentsException
|
||||||
@ -10,10 +9,6 @@ import net.corda.core.crypto.SecureHash
|
|||||||
import net.corda.core.crypto.sha256
|
import net.corda.core.crypto.sha256
|
||||||
import net.corda.core.internal.*
|
import net.corda.core.internal.*
|
||||||
import net.corda.core.internal.cordapp.targetPlatformVersion
|
import net.corda.core.internal.cordapp.targetPlatformVersion
|
||||||
import net.corda.core.serialization.CordaSerializable
|
|
||||||
import net.corda.core.serialization.SerializationCustomSerializer
|
|
||||||
import net.corda.core.serialization.SerializationFactory
|
|
||||||
import net.corda.core.serialization.SerializationWhitelist
|
|
||||||
import net.corda.core.serialization.*
|
import net.corda.core.serialization.*
|
||||||
import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory.toUrl
|
import net.corda.core.serialization.internal.AttachmentURLStreamHandlerFactory.toUrl
|
||||||
import net.corda.core.utilities.contextLogger
|
import net.corda.core.utilities.contextLogger
|
||||||
@ -187,7 +182,7 @@ internal object AttachmentsClassLoaderBuilder {
|
|||||||
private const val CACHE_SIZE = 1000
|
private const val CACHE_SIZE = 1000
|
||||||
|
|
||||||
// This runs in the DJVM so it can't use caffeine.
|
// This runs in the DJVM so it can't use caffeine.
|
||||||
private val cache: MutableMap<Set<SecureHash>, SerializationContext> = createSimpleCache(CACHE_SIZE)
|
private val cache: MutableMap<Set<SecureHash>, SerializationContext> = createSimpleCache<Set<SecureHash>, SerializationContext>(CACHE_SIZE).toSynchronised()
|
||||||
|
|
||||||
fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>, block: (ClassLoader) -> T): T {
|
fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>, block: (ClassLoader) -> T): T {
|
||||||
val attachmentIds = attachments.map { it.id }.toSet()
|
val attachmentIds = attachments.map { it.id }.toSet()
|
||||||
|
@ -6,6 +6,7 @@ import net.corda.behave.scenarios.api.StepsBlock
|
|||||||
import net.corda.behave.scenarios.api.StepsProvider
|
import net.corda.behave.scenarios.api.StepsProvider
|
||||||
import net.corda.behave.scenarios.steps.*
|
import net.corda.behave.scenarios.steps.*
|
||||||
import net.corda.core.internal.objectOrNewInstance
|
import net.corda.core.internal.objectOrNewInstance
|
||||||
|
import net.corda.core.internal.pooledScan
|
||||||
import net.corda.core.utilities.contextLogger
|
import net.corda.core.utilities.contextLogger
|
||||||
|
|
||||||
@Suppress("KDocMissingDocumentation")
|
@Suppress("KDocMissingDocumentation")
|
||||||
@ -18,7 +19,7 @@ class StepsContainer(val state: ScenarioState) : En {
|
|||||||
ClassGraph()
|
ClassGraph()
|
||||||
.addClassLoader(this::class.java.classLoader)
|
.addClassLoader(this::class.java.classLoader)
|
||||||
.enableAllInfo()
|
.enableAllInfo()
|
||||||
.scan()
|
.pooledScan()
|
||||||
.use { it.getClassesImplementing(StepsProvider::class.java.name).loadClasses(StepsProvider::class.java) }
|
.use { it.getClassesImplementing(StepsProvider::class.java.name).loadClasses(StepsProvider::class.java) }
|
||||||
.map { it.kotlin.objectOrNewInstance() }
|
.map { it.kotlin.objectOrNewInstance() }
|
||||||
}
|
}
|
||||||
|
@ -9,7 +9,6 @@ import net.corda.core.crypto.SecureHash
|
|||||||
import net.corda.core.internal.*
|
import net.corda.core.internal.*
|
||||||
import org.slf4j.LoggerFactory
|
import org.slf4j.LoggerFactory
|
||||||
import java.io.InputStream
|
import java.io.InputStream
|
||||||
import java.net.URLClassLoader
|
|
||||||
import java.nio.file.Files
|
import java.nio.file.Files
|
||||||
import java.nio.file.Path
|
import java.nio.file.Path
|
||||||
import java.nio.file.StandardCopyOption
|
import java.nio.file.StandardCopyOption
|
||||||
@ -28,7 +27,7 @@ class ContractsJarFile(private val file: Path) : ContractsJar {
|
|||||||
override val hash: SecureHash by lazy(LazyThreadSafetyMode.NONE, file::hash)
|
override val hash: SecureHash by lazy(LazyThreadSafetyMode.NONE, file::hash)
|
||||||
|
|
||||||
override fun scan(): List<ContractClassName> {
|
override fun scan(): List<ContractClassName> {
|
||||||
val scanResult = ClassGraph().overrideClasspath(singleton(file)).enableClassInfo().scan()
|
val scanResult = ClassGraph().overrideClasspath(singleton(file)).enableClassInfo().pooledScan()
|
||||||
|
|
||||||
return scanResult.use { result ->
|
return scanResult.use { result ->
|
||||||
coreContractClasses
|
coreContractClasses
|
||||||
|
@ -280,7 +280,7 @@ class JarScanningCordappLoader private constructor(private val cordappJarPaths:
|
|||||||
private fun scanCordapp(cordappJarPath: RestrictedURL): RestrictedScanResult {
|
private fun scanCordapp(cordappJarPath: RestrictedURL): RestrictedScanResult {
|
||||||
logger.info("Scanning CorDapp in ${cordappJarPath.url}")
|
logger.info("Scanning CorDapp in ${cordappJarPath.url}")
|
||||||
return cachedScanResult.computeIfAbsent(cordappJarPath) {
|
return cachedScanResult.computeIfAbsent(cordappJarPath) {
|
||||||
val scanResult = ClassGraph().addClassLoader(appClassLoader).overrideClasspath(cordappJarPath.url).enableAllInfo().scan()
|
val scanResult = ClassGraph().addClassLoader(appClassLoader).overrideClasspath(cordappJarPath.url).enableAllInfo().pooledScan()
|
||||||
RestrictedScanResult(scanResult, cordappJarPath.qualifiedNamePrefix)
|
RestrictedScanResult(scanResult, cordappJarPath.qualifiedNamePrefix)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -7,10 +7,7 @@ import net.corda.core.DeleteForDJVM
|
|||||||
import net.corda.core.KeepForDJVM
|
import net.corda.core.KeepForDJVM
|
||||||
import net.corda.core.StubOutForDJVM
|
import net.corda.core.StubOutForDJVM
|
||||||
import net.corda.core.cordapp.Cordapp
|
import net.corda.core.cordapp.Cordapp
|
||||||
import net.corda.core.internal.isAbstractClass
|
import net.corda.core.internal.*
|
||||||
import net.corda.core.internal.objectOrNewInstance
|
|
||||||
import net.corda.core.internal.toSynchronised
|
|
||||||
import net.corda.core.internal.uncheckedCast
|
|
||||||
import net.corda.core.serialization.*
|
import net.corda.core.serialization.*
|
||||||
import net.corda.core.utilities.ByteSequence
|
import net.corda.core.utilities.ByteSequence
|
||||||
import net.corda.core.utilities.contextLogger
|
import net.corda.core.utilities.contextLogger
|
||||||
@ -90,7 +87,7 @@ abstract class AbstractAMQPSerializationScheme(
|
|||||||
.whitelistPackages(scanSpec)
|
.whitelistPackages(scanSpec)
|
||||||
.addClassLoader(cl)
|
.addClassLoader(cl)
|
||||||
.enableAllInfo()
|
.enableAllInfo()
|
||||||
.scan()
|
.pooledScan()
|
||||||
.use {
|
.use {
|
||||||
val serializerClass = SerializationCustomSerializer::class.java
|
val serializerClass = SerializationCustomSerializer::class.java
|
||||||
it.getClassesImplementing(serializerClass.name).loadClasses(serializerClass)
|
it.getClassesImplementing(serializerClass.name).loadClasses(serializerClass)
|
||||||
|
@ -52,7 +52,7 @@ data class CustomCordapp(
|
|||||||
val scanResult = classGraph
|
val scanResult = classGraph
|
||||||
.whitelistPackages(*packages.toTypedArray())
|
.whitelistPackages(*packages.toTypedArray())
|
||||||
.whitelistClasses(*classes.map { it.name }.toTypedArray())
|
.whitelistClasses(*classes.map { it.name }.toTypedArray())
|
||||||
.scan()
|
.pooledScan()
|
||||||
|
|
||||||
scanResult.use {
|
scanResult.use {
|
||||||
JarOutputStream(file.outputStream()).use { jos ->
|
JarOutputStream(file.outputStream()).use { jos ->
|
||||||
|
@ -55,7 +55,7 @@ data class TestCordappImpl(val scanPackage: String, override val config: Map<Str
|
|||||||
return packageToRootPaths.computeIfAbsent(scanPackage) {
|
return packageToRootPaths.computeIfAbsent(scanPackage) {
|
||||||
ClassGraph()
|
ClassGraph()
|
||||||
.whitelistPackages(scanPackage)
|
.whitelistPackages(scanPackage)
|
||||||
.scan()
|
.pooledScan()
|
||||||
.use { it.allResources }
|
.use { it.allResources }
|
||||||
.asSequence()
|
.asSequence()
|
||||||
.map { it.classpathElementFile.toPath() }
|
.map { it.classpathElementFile.toPath() }
|
||||||
|
Loading…
Reference in New Issue
Block a user