CORDA-3644: Scan the CorDapp classloader directly for SerializationWhitelist. (#6014)

* CORDA-3644: Scan the CorDapp classloader directly for SerializationWhitelist.

* CORDA-3644: Filter CorDapps from out-of-process node classpaths by their manifest attributes. Also exclude directories and blatant test artifacts.

* Fix IRS Demo - its "tests" artifact had a non-standard classifier of "test".

docs/source/example-code/src/integration-test/java/net/corda/docs/java/tutorial/test/TutorialFlowAsyncOperationTest.java

@@ -1,6 +1,5 @@
 package net.corda.docs.java.tutorial.test;
 
-import kotlin.Unit;
 import net.corda.client.rpc.CordaRPCClient;
 import net.corda.core.messaging.CordaRPCOps;
 import net.corda.core.utilities.KotlinUtilsKt;
@@ -10,24 +9,24 @@ import net.corda.testing.driver.*;
 import net.corda.testing.node.User;
 import org.junit.Test;
 
-import java.util.Collections;
-import java.util.HashSet;
 import java.util.concurrent.Future;
 
+import static java.util.Collections.singleton;
+import static java.util.Collections.singletonList;
 import static net.corda.testing.core.TestConstants.ALICE_NAME;
+import static net.corda.testing.driver.Driver.driver;
+import static net.corda.testing.node.internal.InternalTestUtilsKt.cordappWithPackages;
 import static org.junit.Assert.assertEquals;
 
-public final class TutorialFlowAsyncOperationTest {
+public class TutorialFlowAsyncOperationTest {
     // DOCSTART summingWorks
     @Test
-    public final void summingWorks() {
-        Driver.driver(new DriverParameters(), (DriverDSL dsl) -> {
-            User aliceUser = new User("aliceUser", "testPassword1",
-                    new HashSet<>(Collections.singletonList(Permissions.all()))
-            );
+    public void summingWorks() {
+        driver(new DriverParameters(singletonList(cordappWithPackages("net.corda.docs.java.tutorial.flowstatemachines"))), (DriverDSL dsl) -> {
+            User aliceUser = new User("aliceUser", "testPassword1", singleton(Permissions.all()));
             Future<NodeHandle> aliceFuture = dsl.startNode(new NodeParameters()
                     .withProvidedName(ALICE_NAME)
-                    .withRpcUsers(Collections.singletonList(aliceUser))
+                    .withRpcUsers(singletonList(aliceUser))
             );
             NodeHandle alice = KotlinUtilsKt.getOrThrow(aliceFuture, null);
             CordaRPCClient aliceClient = new CordaRPCClient(alice.getRpcAddress());
@@ -35,7 +34,7 @@ public final class TutorialFlowAsyncOperationTest {
             Future<Integer> answerFuture = aliceProxy.startFlowDynamic(ExampleSummingFlow.class).getReturnValue();
             int answer = KotlinUtilsKt.getOrThrow(answerFuture, null);
             assertEquals(3, answer);
-            return Unit.INSTANCE;
+            return null;
         });
     }
     // DOCEND summingWorks

docs/source/example-code/src/integration-test/kotlin/net/corda/docs/kotlin/tutorial/test/TutorialFlowAsyncOperationTest.kt

@@ -10,7 +10,6 @@ import net.corda.testing.driver.DriverParameters
 import net.corda.testing.driver.driver
 import net.corda.testing.node.User
 import net.corda.testing.node.internal.cordappWithPackages
-import net.corda.testing.node.internal.findCordapp
 import org.junit.Test
 import kotlin.test.assertEquals
 

node/src/integration-test/kotlin/net/corda/node/logging/ErrorCodeLoggingTests.kt

@@ -1,6 +1,5 @@
 package net.corda.node.logging
 
-import net.corda.core.flows.FlowException
 import net.corda.core.flows.FlowLogic
 import net.corda.core.flows.InitiatingFlow
 import net.corda.core.flows.StartableByRPC
@@ -23,7 +22,13 @@ class ErrorCodeLoggingTests {
             node.rpc.startFlow(::MyFlow).waitForCompletion()
             val logFile = node.logFile()
 
-            val linesWithErrorCode = logFile.useLines { lines -> lines.filter { line -> line.contains("[errorCode=") }.filter { line -> line.contains("moreInformationAt=https://errors.corda.net/") }.toList() }
+            val linesWithErrorCode = logFile.useLines { lines ->
+                lines.filter { line ->
+                    line.contains("[errorCode=")
+                }.filter { line ->
+                    line.contains("moreInformationAt=https://errors.corda.net/")
+                }.toList()
+            }
 
             assertThat(linesWithErrorCode).isNotEmpty
         }
@@ -35,10 +40,11 @@ class ErrorCodeLoggingTests {
 	fun `When logging is set to error level, there are no other levels logged after node startup`() {
         driver(DriverParameters(notarySpecs = emptyList())) {
             val node = startNode(startInSameProcess = false, logLevelOverride = "ERROR").getOrThrow()
-            node.rpc.startFlow(::MyFlow).waitForCompletion()
             val logFile = node.logFile()
+            val lengthAfterStart = logFile.length()
+            node.rpc.startFlow(::MyFlow).waitForCompletion()
             // An exception thrown in a flow will log at the "INFO" level.
-            assertThat(logFile.length()).isEqualTo(0)
+            assertThat(logFile.length()).isEqualTo(lengthAfterStart)
         }
     }
 

node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt

@@ -22,7 +22,6 @@ import net.corda.node.VersionInfo
 import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.nodeapi.internal.coreContractClasses
 import net.corda.serialization.internal.DefaultWhitelist
-import org.apache.commons.collections4.map.LRUMap
 import java.lang.reflect.Modifier
 import java.math.BigInteger
 import java.net.URL
@@ -293,9 +292,7 @@ class JarScanningCordappLoader private constructor(private val cordappJarPaths:
     }
 
     private fun findWhitelists(cordappJarPath: RestrictedURL): List<SerializationWhitelist> {
-        val whitelists = URLClassLoader(arrayOf(cordappJarPath.url)).use {
-            ServiceLoader.load(SerializationWhitelist::class.java, it).toList()
-        }
+        val whitelists = ServiceLoader.load(SerializationWhitelist::class.java, appClassLoader).toList()
         return whitelists.filter {
             it.javaClass.location == cordappJarPath.url && it.javaClass.name.startsWith(cordappJarPath.qualifiedNamePrefix)
         } + DefaultWhitelist // Always add the DefaultWhitelist to the whitelist for an app.
@@ -309,19 +306,21 @@ class JarScanningCordappLoader private constructor(private val cordappJarPaths:
         return scanResult.getClassesWithSuperclass(MappedSchema::class).instances().toSet()
     }
 
-    private val cachedScanResult = LRUMap<RestrictedURL, RestrictedScanResult>(1000)
-
     private fun scanCordapp(cordappJarPath: RestrictedURL): RestrictedScanResult {
-        logger.info("Scanning CorDapp in ${cordappJarPath.url}")
-        return cachedScanResult.computeIfAbsent(cordappJarPath) {
-            val scanResult = ClassGraph().addClassLoader(appClassLoader).overrideClasspath(cordappJarPath.url).enableAllInfo().pooledScan()
-            RestrictedScanResult(scanResult, cordappJarPath.qualifiedNamePrefix)
-        }
+        val cordappElement = cordappJarPath.url.toString()
+        logger.info("Scanning CorDapp in $cordappElement")
+        val scanResult = ClassGraph()
+            .filterClasspathElements { elt -> elt == cordappElement }
+            .overrideClassLoaders(appClassLoader)
+            .ignoreParentClassLoaders()
+            .enableAllInfo()
+            .pooledScan()
+        return RestrictedScanResult(scanResult, cordappJarPath.qualifiedNamePrefix)
     }
 
     private fun <T : Any> loadClass(className: String, type: KClass<T>): Class<out T>? {
         return try {
-            appClassLoader.loadClass(className).asSubclass(type.java)
+            Class.forName(className, false, appClassLoader).asSubclass(type.java)
         } catch (e: ClassCastException) {
             logger.warn("As $className must be a sub-type of ${type.java.name}")
             null

samples/irs-demo/cordapp/workflows-irs/build.gradle

@@ -56,7 +56,7 @@ jar {
 }
 
 task testJar(type: Jar) {
-    classifier "test"
+    classifier "tests"
     from sourceSets.main.output
     from sourceSets.test.output
 }

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/DriverDSLImpl.kt

@@ -21,8 +21,20 @@ import net.corda.core.internal.concurrent.fork
 import net.corda.core.internal.concurrent.map
 import net.corda.core.internal.concurrent.openFuture
 import net.corda.core.internal.concurrent.transpose
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_CONTRACT_NAME
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_CONTRACT_LICENCE
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_CONTRACT_VENDOR
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_CONTRACT_VERSION
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_WORKFLOW_NAME
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_WORKFLOW_LICENCE
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_WORKFLOW_VENDOR
+import net.corda.core.internal.cordapp.CordappImpl.Companion.CORDAPP_WORKFLOW_VERSION
+import net.corda.core.internal.cordapp.CordappImpl.Companion.MIN_PLATFORM_VERSION
+import net.corda.core.internal.cordapp.CordappImpl.Companion.TARGET_PLATFORM_VERSION
+import net.corda.core.internal.cordapp.get
 import net.corda.core.internal.createDirectories
 import net.corda.core.internal.div
+import net.corda.core.internal.isRegularFile
 import net.corda.core.internal.list
 import net.corda.core.internal.packageName_
 import net.corda.core.internal.readObject
@@ -80,24 +92,26 @@ import okhttp3.OkHttpClient
 import okhttp3.Request
 import rx.Subscription
 import rx.schedulers.Schedulers
-import java.io.File
 import java.net.ConnectException
 import java.net.URL
 import java.net.URLClassLoader
+import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.Paths
 import java.security.cert.X509Certificate
 import java.time.Duration
 import java.time.Instant
 import java.time.ZoneOffset.UTC
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
-import java.util.Random
-import java.util.UUID
+import java.util.*
+import java.util.Collections.unmodifiableList
 import java.util.concurrent.Executors
 import java.util.concurrent.ScheduledExecutorService
 import java.util.concurrent.TimeUnit
 import java.util.concurrent.TimeoutException
 import java.util.concurrent.atomic.AtomicInteger
+import java.util.jar.JarInputStream
 import kotlin.collections.ArrayList
 import kotlin.collections.HashMap
 import kotlin.collections.HashSet
@@ -792,6 +806,17 @@ class DriverDSLImpl(
                 Permissions.invokeRpc(CordaRPCOps::killFlow)
         )
 
+        private val CORDAPP_MANIFEST_ATTRIBUTES: List<String> = unmodifiableList(listOf(
+            CORDAPP_CONTRACT_NAME,
+            CORDAPP_CONTRACT_LICENCE,
+            CORDAPP_CONTRACT_VENDOR,
+            CORDAPP_CONTRACT_VERSION,
+            CORDAPP_WORKFLOW_NAME,
+            CORDAPP_WORKFLOW_LICENCE,
+            CORDAPP_WORKFLOW_VENDOR,
+            CORDAPP_WORKFLOW_VERSION
+        ))
+
         /**
          * Add the DJVM's sources to the node's configuration file.
          * These will all be ignored unless devMode is also true.
@@ -923,12 +948,11 @@ class DriverDSLImpl(
 
             // The following dependencies are excluded from the classpath of the created JVM,
             // so that the environment resembles a real one as close as possible.
-            // These are either classes that will be added as attachments to the node (i.e. samples, finance, opengamma etc.)
-            // or irrelevant testing libraries (test, corda-mock etc.).
-            // TODO: There is pending work to fix this issue without custom blacklisting. See: https://r3-cev.atlassian.net/browse/CORDA-2164.
-            val exclude = listOf("samples", "finance", "integrationTest", "test", "corda-mock", "com.opengamma.strata")
-            val cp = ProcessUtilities.defaultClassPath.filterNot { cpEntry ->
-                exclude.any { token -> cpEntry.contains("${File.separatorChar}$token") } || cpEntry.endsWith("-tests.jar")
+           val cp = ProcessUtilities.defaultClassPath.filter { cpEntry ->
+                val cpPathEntry = Paths.get(cpEntry)
+                cpPathEntry.isRegularFile()
+                    && !isTestArtifact(cpPathEntry.fileName.toString())
+                    && !cpPathEntry.isCorDapp
             }
 
             return ProcessUtilities.startJavaProcess(
@@ -944,6 +968,27 @@ class DriverDSLImpl(
             )
         }
 
+        // Obvious test artifacts. This is NOT intended to be an exhaustive list!
+        // It is only intended to remove those FEW jars which BLATANTLY do not
+        // belong inside a Corda Node.
+        private fun isTestArtifact(name: String): Boolean {
+            return name.endsWith("-tests.jar")
+                    || name.endsWith("-test.jar")
+                    || name.startsWith("corda-mock")
+                    || name.startsWith("junit")
+                    || name.startsWith("testng")
+                    || name.startsWith("mockito")
+        }
+
+        // Identify CorDapp JARs by their attributes in MANIFEST.MF.
+        private val Path.isCorDapp: Boolean get() {
+            return JarInputStream(Files.newInputStream(this).buffered()).use { jar ->
+                val manifest = jar.manifest ?: return false
+                CORDAPP_MANIFEST_ATTRIBUTES.any { manifest[it] != null }
+                    || (manifest[TARGET_PLATFORM_VERSION] != null && manifest[MIN_PLATFORM_VERSION] != null)
+            }
+        }
+
         private fun startWebserver(handle: NodeHandleInternal, debugPort: Int?, maximumHeapSize: String): Process {
             val className = "net.corda.webserver.WebServer"
             writeConfig(handle.baseDirectory, "web-server.conf", handle.toWebServerConfig())