ENT-12366 ExternalVerifier no longer needs legacy contracts folder, and can derive everything it needs from attachments. (#7866)

* ENT-12366 ExternalVerifier no longer needs legacy contracts folder, and can derive everything it needs from attachments.

* ENT-12366 Fix compiler warnings

* Revert "ENT-12366 Fix compiler warnings"

This reverts commit 4e884a5519.

* ENT-12366 Attempt to appease warnings in both 1.2 and 1.9 compilers

core-tests/src/test/kotlin/net/corda/coretests/flows/ContractUpgradeFlowRPCTest.kt

@@ -13,21 +13,26 @@ import net.corda.core.internal.getRequiredTransaction
 import net.corda.core.messaging.CordaRPCOps
 import net.corda.core.transactions.ContractUpgradeLedgerTransaction
 import net.corda.core.transactions.SignedTransaction
+import net.corda.coretesting.internal.matchers.rpc.willReturn
+import net.corda.coretesting.internal.matchers.rpc.willThrow
 import net.corda.node.services.Permissions.Companion.startFlow
 import net.corda.testing.contracts.DummyContract
 import net.corda.testing.contracts.DummyContractV2
 import net.corda.testing.core.ALICE_NAME
 import net.corda.testing.core.BOB_NAME
 import net.corda.testing.core.singleIdentity
-import net.corda.coretesting.internal.matchers.rpc.willReturn
-import net.corda.coretesting.internal.matchers.rpc.willThrow
 import net.corda.testing.node.User
-import net.corda.testing.node.internal.*
+import net.corda.testing.node.internal.DUMMY_CONTRACTS_CORDAPP
+import net.corda.testing.node.internal.InternalMockNetwork
+import net.corda.testing.node.internal.RPCDriverDSL
+import net.corda.testing.node.internal.TestStartedNode
+import net.corda.testing.node.internal.enclosedCordapp
+import net.corda.testing.node.internal.rpcDriver
+import net.corda.testing.node.internal.rpcTestUser
+import net.corda.testing.node.internal.startRpcClient
 import org.junit.AfterClass
-import org.junit.Ignore
 import org.junit.Test
 
-@Ignore("Explicit contract upgrade not supported in 4.12")
 class ContractUpgradeFlowRPCTest : WithContracts, WithFinality {
     companion object {
         private val classMockNet = InternalMockNetwork(cordappsForAllNodes = listOf(DUMMY_CONTRACTS_CORDAPP, enclosedCordapp()))

core-tests/src/test/kotlin/net/corda/coretests/flows/ContractUpgradeFlowTest.kt

@@ -47,11 +47,9 @@ import net.corda.testing.node.internal.TestStartedNode
 import net.corda.testing.node.internal.enclosedCordapp
 import net.corda.testing.node.internal.startFlow
 import org.junit.AfterClass
-import org.junit.Ignore
 import org.junit.Test
 import java.util.Currency
 
-@Ignore("Explicit contract upgrade not supported in 4.12")
 class ContractUpgradeFlowTest : WithContracts, WithFinality {
 
     companion object {

core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt

@@ -33,6 +33,7 @@ import net.corda.core.node.ServicesForResolution
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.serialization.DeprecatedConstructorForDeserialization
 import net.corda.core.serialization.deserialize
+import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
 import net.corda.core.transactions.ContractUpgradeFilteredTransaction.FilteredComponent
 import net.corda.core.transactions.ContractUpgradeWireTransaction.Companion.calculateUpgradedState
 import net.corda.core.transactions.ContractUpgradeWireTransaction.Component.INPUTS
@@ -281,30 +282,38 @@ private constructor(
         fun resolve(verificationSupport: VerificationSupport,
                     wtx: ContractUpgradeWireTransaction,
                     sigs: List<TransactionSignature>): ContractUpgradeLedgerTransaction {
-            val inputs = wtx.inputs.map(verificationSupport::getStateAndRef)
+
             val (legacyContractAttachment, upgradedContractAttachment) = verificationSupport.getAttachments(listOf(
                     wtx.legacyContractAttachmentId,
                     wtx.upgradedContractAttachmentId
             ))
+            if (legacyContractAttachment == null) throw AttachmentResolutionException(wtx.legacyContractAttachmentId)
+            if (upgradedContractAttachment == null) throw AttachmentResolutionException(wtx.upgradedContractAttachmentId)
             val networkParameters = verificationSupport.getNetworkParameters(wtx.networkParametersHash)
                     ?: throw TransactionResolutionException(wtx.id)
-            val upgradedContract = loadUpgradedContract(wtx.upgradedContractClassName, wtx.id, verificationSupport.appClassLoader)
-            return ContractUpgradeLedgerTransaction(
-                    inputs,
-                    wtx.notary,
-                    legacyContractAttachment ?: throw AttachmentResolutionException(wtx.legacyContractAttachmentId),
-                    upgradedContractAttachment ?: throw AttachmentResolutionException(wtx.upgradedContractAttachmentId),
-                    wtx.id,
-                    wtx.privacySalt,
-                    sigs,
+
+            return AttachmentsClassLoaderBuilder.withAttachmentsClassLoaderContext(
+                    listOf(legacyContractAttachment, upgradedContractAttachment),
                     networkParameters,
-                    upgradedContract
-            )
+                    wtx.id,
+                    verificationSupport::isAttachmentTrusted,
+                    attachmentsClassLoaderCache = verificationSupport.attachmentsClassLoaderCache
+            ) { serializationContext ->
+                val inputs = wtx.inputs.map(verificationSupport::getStateAndRef)
+                val upgradedContract = loadUpgradedContract(wtx.upgradedContractClassName, wtx.id, serializationContext.deserializationClassLoader)
+                ContractUpgradeLedgerTransaction(
+                        inputs,
+                        wtx.notary,
+                        legacyContractAttachment,
+                        upgradedContractAttachment,
+                        wtx.id,
+                        wtx.privacySalt,
+                        sigs,
+                        networkParameters,
+                        upgradedContract)
+            }
         }
 
-        // TODO There is an inconsistency with the class loader used with this method. Transaction resolution uses the app class loader,
-        //  whilst TransactionStorageVerification.getContractUpdateOutput uses an attachments class loder comprised of the the legacy and
-        //  upgraded attachments
         @CordaInternal
         @JvmSynthetic
         internal fun loadUpgradedContract(className: ContractClassName, id: SecureHash, classLoader: ClassLoader): UpgradedContract<ContractState, *> {

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -30,7 +30,6 @@ import net.corda.core.internal.TransactionDeserialisationException
 import net.corda.core.internal.createComponentGroups
 import net.corda.core.internal.deserialiseComponentGroup
 import net.corda.core.internal.equivalent
-import net.corda.core.internal.flatMapToSet
 import net.corda.core.internal.getGroup
 import net.corda.core.internal.isUploaderTrusted
 import net.corda.core.internal.lazyMapped
@@ -190,19 +189,17 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
     @JvmSynthetic
     internal fun toLedgerTransactionInternal(verificationSupport: VerificationSupport): LedgerTransaction {
         // Look up public keys to authenticated identities.
-        val authenticatedCommands = if (verificationSupport.isInProcess) {
-            commands.lazyMapped { cmd, _ ->
+        if (!verificationSupport.isInProcess) {
+            val signersGroup: List<List<PublicKey>> = uncheckedCast(deserialiseComponentGroup(componentGroups, List::class, SIGNERS_GROUP))
+            if (signersGroup.isNotEmpty()) {
+                // Pre-fetch all signing keys if the signers component group is present (Corda 4+)
+                verificationSupport.getParties(signersGroup.flatten().toSet())
+            }
+        }
+        val authenticatedCommands = commands.lazyMapped { cmd, _ ->
                 val parties = verificationSupport.getParties(cmd.signers).filterNotNull()
                 CommandWithParties(cmd.signers, parties, cmd.value)
             }
-        } else {
-            val allSigners = commands.flatMapToSet { it.signers }
-            val allParties = verificationSupport.getParties(allSigners)
-            commands.map { cmd ->
-                val parties = cmd.signers.mapNotNull { allParties[allSigners.indexOf(it)] }
-                CommandWithParties(cmd.signers, parties, cmd.value)
-            }
-        }
 
         // Ensure that the lazy mappings will use the correct SerializationContext.
         val serializationFactory = SerializationFactory.defaultFactory

node/src/main/kotlin/net/corda/node/verification/ExternalVerifierHandleImpl.kt

@@ -58,7 +58,6 @@ import kotlin.io.path.div
 import kotlin.io.path.fileAttributesViewOrNull
 import kotlin.io.path.isExecutable
 import kotlin.io.path.isWritable
-import kotlin.io.path.notExists
 
 /**
  * Handle to the node's external verifier. The verifier process is started lazily on the first verification request.
@@ -117,12 +116,6 @@ class ExternalVerifierHandleImpl(
     }
 
     private fun startServer() {
-        val legacyContractsPath = (baseDirectory / "legacy-contracts")
-        if (legacyContractsPath.notExists()) {
-            log.error("Failed to start external verifier because $legacyContractsPath does not exist. Please create a legacy-contracts " +
-                    "directory under $baseDirectory and place your legacy contracts into this directory. See the documentation for details.")
-            throw IOException("Cannot start external verifier because $legacyContractsPath does not exist.")
-        }
         if (::socketFile.isInitialized) return
         // Try to create the UNIX domain file in /tmp to keep the full path under the 100 char limit. If we don't have access to it then
         // fallback to the temp dir specified by the JVM and hope it's short enough.

verifier/src/main/kotlin/net/corda/verifier/ExternalVerificationContext.kt

@@ -12,12 +12,13 @@ import net.corda.core.serialization.internal.AttachmentsClassLoaderCache
 import java.security.PublicKey
 
 class ExternalVerificationContext(
-        override val appClassLoader: ClassLoader,
         override val attachmentsClassLoaderCache: AttachmentsClassLoaderCache,
         private val externalVerifier: ExternalVerifier,
         private val transactionInputsAndReferences: Map<StateRef, SerializedTransactionState>,
         override val rotatedKeys: RotatedKeys
 ) : VerificationSupport {
+    override val appClassLoader: ClassLoader get() = throw NotImplementedError("Cannot call appClassLoader")
+
     override val isInProcess: Boolean get() = false
 
     override fun getParties(keys: Collection<PublicKey>): List<Party?> = externalVerifier.getParties(keys)

verifier/src/main/kotlin/net/corda/verifier/ExternalVerifier.kt

@@ -10,7 +10,6 @@ import net.corda.core.internal.mapToSet
 import net.corda.core.internal.objectOrNewInstance
 import net.corda.core.internal.toSimpleString
 import net.corda.core.internal.toSynchronised
-import net.corda.core.internal.toTypedArray
 import net.corda.core.internal.verification.AttachmentFixups
 import net.corda.core.node.NetworkParameters
 import net.corda.core.serialization.SerializationContext
@@ -45,19 +44,14 @@ import net.corda.serialization.internal.verifier.ExternalVerifierOutbound.Verifi
 import net.corda.serialization.internal.verifier.ExternalVerifierOutbound.VerifierRequest.GetNetworkParameters
 import net.corda.serialization.internal.verifier.ExternalVerifierOutbound.VerifierRequest.GetParties
 import net.corda.serialization.internal.verifier.ExternalVerifierOutbound.VerifierRequest.GetTrustedClassAttachments
-import net.corda.serialization.internal.verifier.loadCustomSerializationScheme
 import net.corda.serialization.internal.verifier.readCordaSerializable
 import net.corda.serialization.internal.verifier.writeCordaSerializable
-import java.net.URLClassLoader
 import java.nio.channels.SocketChannel
-import java.nio.file.Path
 import java.security.PublicKey
 import java.util.Optional
-import kotlin.io.path.div
-import kotlin.io.path.listDirectoryEntries
 
 @Suppress("MagicNumber")
-class ExternalVerifier(private val baseDirectory: Path, private val channel: SocketChannel) {
+class ExternalVerifier(private val channel: SocketChannel) {
     companion object {
         private val log = contextLogger()
     }
@@ -69,7 +63,6 @@ class ExternalVerifier(private val baseDirectory: Path, private val channel: Soc
     private val networkParametersMap: OptionalCache<SecureHash, NetworkParameters>
     private val trustedClassAttachments: Cache<String, List<SecureHash>>
 
-    private lateinit var appClassLoader: ClassLoader
     private lateinit var currentNetworkParameters: NetworkParameters
     private lateinit var rotatedKeys: RotatedKeys
 
@@ -102,39 +95,15 @@ class ExternalVerifier(private val baseDirectory: Path, private val channel: Soc
         val initialisation = channel.readCordaSerializable(Initialisation::class)
         log.info("Received $initialisation")
 
-        appClassLoader = createAppClassLoader()
-
-        // Then use the initialisation message to create the correct serialization context
-        _contextSerializationEnv.set(null)
-        _contextSerializationEnv.set(SerializationEnvironment.with(
-                verifierSerializationFactory(initialisation, appClassLoader).apply {
-                    initialisation.customSerializationSchemeClassName?.let {
-                        registerScheme(loadCustomSerializationScheme(it, appClassLoader))
-                    }
-                },
-                p2pContext = AMQP_P2P_CONTEXT.withClassLoader(appClassLoader)
-        ))
-
-        attachmentFixups.load(appClassLoader)
-
         currentNetworkParameters = initialisation.currentNetworkParameters
         networkParametersMap.put(initialisation.serializedCurrentNetworkParameters.hash, Optional.of(currentNetworkParameters))
         rotatedKeys = initialisation.rotatedKeys
         log.info("External verifier initialised")
     }
 
-    private fun createAppClassLoader(): ClassLoader {
-        val cordappJarUrls = (baseDirectory / "legacy-contracts").listDirectoryEntries("*.jar")
-                .stream()
-                .map { it.toUri().toURL() }
-                .toTypedArray()
-        log.debug { "CorDapps: ${cordappJarUrls?.joinToString()}" }
-        return URLClassLoader(cordappJarUrls, javaClass.classLoader)
-    }
-
     @Suppress("INVISIBLE_MEMBER")
     private fun verifyTransaction(request: VerificationRequest) {
-        val verificationContext = ExternalVerificationContext(appClassLoader, attachmentsClassLoaderCache, this,
+        val verificationContext = ExternalVerificationContext(attachmentsClassLoaderCache, this,
                 request.ctxInputsAndReferences, rotatedKeys)
         val result: Try<Unit> = try {
             val ctx = request.ctx

verifier/src/main/kotlin/net/corda/verifier/Main.kt

@@ -26,7 +26,7 @@ object Main {
             val channel = SocketChannel.open(StandardProtocolFamily.UNIX)
             channel.connect(UnixDomainSocketAddress.of(socketFile))
             log.info("Connected to node on UNIX domain file $socketFile")
-            ExternalVerifier(baseDirectory, channel).run()
+            ExternalVerifier(channel).run()
         } catch (t: Throwable) {
             log.error("Unexpected error which has terminated the verifier", t)
             exitProcess(1)