[CORDA-2636] Ensure states created with contract upgrades can be migrated (#4786)

* Ensure states created with contract upgrades can be migrated

* Remove line from api-current.txt representing an uncallable constructor

.ci/api-current.txt

@@ -7744,7 +7744,6 @@ public class net.corda.testing.node.MockServices extends java.lang.Object implem
   public <init>(net.corda.core.identity.CordaX500Name, net.corda.core.node.services.IdentityService)
   public <init>(net.corda.core.identity.CordaX500Name, net.corda.core.node.services.IdentityService, int, kotlin.jvm.internal.DefaultConstructorMarker)
   public <init>(net.corda.core.identity.CordaX500Name, net.corda.core.node.services.IdentityService, java.security.KeyPair, java.security.KeyPair...)
-  public <init>(net.corda.node.cordapp.CordappLoader, net.corda.core.node.services.IdentityService, net.corda.core.node.NetworkParameters, net.corda.testing.core.TestIdentity, java.security.KeyPair[], net.corda.core.node.services.KeyManagementService, kotlin.jvm.internal.DefaultConstructorMarker)
   public <init>(net.corda.testing.core.TestIdentity, net.corda.core.node.NetworkParameters, net.corda.testing.core.TestIdentity...)
   public <init>(net.corda.testing.core.TestIdentity, net.corda.testing.core.TestIdentity...)
   public final void addMockCordapp(String)

core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt

@@ -307,12 +307,12 @@ object AttachmentsClassLoaderBuilder {
      *
      * @param txId The transaction ID that triggered this request; it's unused except for error messages and exceptions that can occur during setup.
      */
-    fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>, params: NetworkParameters, txId: SecureHash, block: (ClassLoader) -> T): T {
+    fun <T> withAttachmentsClassloaderContext(attachments: List<Attachment>, params: NetworkParameters, txId: SecureHash, parent: ClassLoader = ClassLoader.getSystemClassLoader(), block: (ClassLoader) -> T): T {
         val attachmentIds = attachments.map { it.id }.toSet()
 
         val serializationContext = cache.computeIfAbsent(Key(attachmentIds, params)) {
             // Create classloader and load serializers, whitelisted classes
-            val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId)
+            val transactionClassLoader = AttachmentsClassLoader(attachments, params, txId, parent)
             val serializers = createInstancesOfClassesImplementing(transactionClassLoader, SerializationCustomSerializer::class.java)
             val whitelistedClasses = ServiceLoader.load(SerializationWhitelist::class.java, transactionClassLoader)
                     .flatMap { it.whitelist }

node-api/src/main/kotlin/net/corda/nodeapi/internal/cordapp/CordappLoader.kt

@@ -1,4 +1,4 @@
-package net.corda.node.cordapp
+package net.corda.nodeapi.internal.cordapp
 
 import net.corda.core.cordapp.Cordapp
 import net.corda.core.flows.FlowLogic

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt

@@ -12,6 +12,7 @@ import net.corda.core.identity.CordaX500Name
 import net.corda.nodeapi.internal.MigrationHelpers.getMigrationResource
 import net.corda.core.schemas.MappedSchema
 import net.corda.core.utilities.contextLogger
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import sun.security.x509.X500Name
 import java.io.ByteArrayInputStream
 import java.io.InputStream
@@ -28,7 +29,7 @@ class SchemaMigration(
         val schemas: Set<MappedSchema>,
         val dataSource: DataSource,
         private val databaseConfig: DatabaseConfig,
-        private val classLoader: ClassLoader = Thread.currentThread().contextClassLoader,
+        cordappLoader: CordappLoader? = null,
         private val currentDirectory: Path?,
         private val ourName: CordaX500Name) {
 
@@ -36,8 +37,15 @@ class SchemaMigration(
         private val logger = contextLogger()
         const val NODE_BASE_DIR_KEY = "liquibase.nodeDaseDir"
         const val NODE_X500_NAME = "liquibase.nodeName"
+        val loader = ThreadLocal<CordappLoader>()
     }
 
+    init {
+        loader.set(cordappLoader)
+    }
+
+    private val classLoader = cordappLoader?.appClassLoader ?: Thread.currentThread().contextClassLoader
+
     /**
      * Main entry point to the schema migration.
      * Called during node startup.

node-api/src/test/kotlin/net/corda/nodeapi/internal/AttachmentsClassLoaderStaticContractTests.kt

@@ -16,7 +16,7 @@ import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.transactions.TransactionBuilder
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.internal.cordapp.CordappProviderImpl
 import net.corda.node.internal.cordapp.JarScanningCordappLoader
 import net.corda.nodeapi.internal.AttachmentsClassLoaderStaticContractTests.AttachmentDummyContract.Companion.ATTACHMENT_PROGRAM_ID

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -34,7 +34,7 @@ import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.minutes
 import net.corda.node.CordaClock
 import net.corda.node.VersionInfo
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.internal.classloading.requireAnnotation
 import net.corda.node.internal.cordapp.*
 import net.corda.node.internal.rpc.proxies.AuthenticatedRpcOpsProxy
@@ -762,7 +762,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
     protected open fun startDatabase() {
         val props = configuration.dataSourceProperties
         if (props.isEmpty) throw DatabaseConfigurationException("There must be a database configured.")
-        database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry, this.cordappLoader.appClassLoader, configuration.baseDirectory, configuration.myLegalName)
+        database.startHikariPool(props, configuration.database, schemaService.internalSchemas(), metricRegistry, this.cordappLoader, configuration.baseDirectory, configuration.myLegalName)
         // Now log the vendor string as this will also cause a connection to be tested eagerly.
         logVendorString(database, log)
     }
@@ -1074,10 +1074,10 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig,
     return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, cacheFactory, attributeConverters, customClassLoader)
 }
 
-fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set<MappedSchema>, metricRegistry: MetricRegistry? = null, classloader: ClassLoader = Thread.currentThread().contextClassLoader, currentDir: Path? = null, ourName: CordaX500Name) {
+fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set<MappedSchema>, metricRegistry: MetricRegistry? = null, cordappLoader: CordappLoader? = null, currentDir: Path? = null, ourName: CordaX500Name) {
     try {
         val dataSource = DataSourceFactory.createDataSource(hikariProperties, metricRegistry = metricRegistry)
-        val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig, classloader, currentDir, ourName)
+        val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig, cordappLoader, currentDir, ourName)
         schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L })
         start(dataSource)
     } catch (ex: Exception) {

node/src/main/kotlin/net/corda/node/internal/cordapp/CordappProviderImpl.kt

@@ -12,7 +12,7 @@ import net.corda.core.node.services.AttachmentId
 import net.corda.core.node.services.AttachmentStorage
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.utilities.contextLogger
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.services.persistence.AttachmentStorageInternal
 import java.net.URL
 import java.util.concurrent.ConcurrentHashMap

node/src/main/kotlin/net/corda/node/internal/cordapp/JarScanningCordappLoader.kt

@@ -20,7 +20,7 @@ import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.utilities.contextLogger
 import net.corda.node.VersionInfo
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.nodeapi.internal.coreContractClasses
 import net.corda.serialization.internal.DefaultWhitelist
 import org.apache.commons.collections4.map.LRUMap

node/src/main/kotlin/net/corda/node/migration/MigrationServicesForResolution.kt

@@ -1,6 +1,7 @@
 package net.corda.node.migration
 
 import net.corda.core.contracts.*
+import net.corda.core.cordapp.CordappContext
 import net.corda.core.cordapp.CordappProvider
 import net.corda.core.crypto.SecureHash
 import net.corda.core.internal.deserialiseComponentGroup
@@ -8,10 +9,7 @@ import net.corda.core.internal.div
 import net.corda.core.internal.readObject
 import net.corda.core.node.NetworkParameters
 import net.corda.core.node.ServicesForResolution
-import net.corda.core.node.services.AttachmentStorage
+import net.corda.core.node.services.*
-import net.corda.core.node.services.IdentityService
-import net.corda.core.node.services.NetworkParametersService
-import net.corda.core.node.services.TransactionStorage
 import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.internal.AttachmentsClassLoaderBuilder
 import net.corda.core.transactions.ContractUpgradeLedgerTransaction
@@ -23,6 +21,7 @@ import net.corda.nodeapi.internal.network.NETWORK_PARAMS_FILE_NAME
 import net.corda.nodeapi.internal.network.SignedNetworkParameters
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.SchemaMigration
+import sun.reflect.generics.reflectiveObjects.NotImplementedException
 import java.nio.file.Paths
 import java.time.Clock
 import java.time.Duration
@@ -39,7 +38,19 @@ class MigrationServicesForResolution(
         val logger = contextLogger()
     }
     override val cordappProvider: CordappProvider
-        get() = throw NotImplementedError()
+        get() = object : CordappProvider {
+
+            val cordappLoader = SchemaMigration.loader.get()
+
+            override fun getAppContext(): CordappContext {
+                throw NotImplementedException()
+            }
+
+            override fun getContractAttachmentID(contractClassName: ContractClassName): AttachmentId? {
+                throw NotImplementedException()
+            }
+        }
+    private val cordappLoader = SchemaMigration.loader.get()
 
     private fun defaultNetworkParameters(): NetworkParameters {
         logger.warn("Using a dummy set of network parameters for migration.")
@@ -96,7 +107,7 @@ class MigrationServicesForResolution(
     private fun extractStateFromTx(tx: WireTransaction, stateIndices: Collection<Int>): List<TransactionState<ContractState>> {
         return try {
             val attachments = tx.attachments.mapNotNull { attachments.openAttachment(it)}
-            val states = AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(attachments, networkParameters, tx.id) {
+            val states = AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(attachments, networkParameters, tx.id, cordappLoader.appClassLoader) {
                 deserialiseComponentGroup(tx.componentGroups, TransactionState::class, ComponentGroupEnum.OUTPUTS_GROUP, forceDeserialize = true)
             }
             states.filterIndexed {index, _ -> stateIndices.contains(index)}.toList()

node/src/main/kotlin/net/corda/node/utilities/NotaryLoader.kt

@@ -6,7 +6,7 @@ import net.corda.core.internal.notary.NotaryService
 import net.corda.core.utilities.contextLogger
 import net.corda.node.SerialFilter
 import net.corda.node.VersionInfo
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.internal.cordapp.VirtualCordapp
 import net.corda.node.services.api.ServiceHubInternal
 import net.corda.node.services.config.NotaryConfig

node/src/test/kotlin/net/corda/node/services/vault/VaultSoftLockManagerTest.kt

@@ -19,7 +19,7 @@ import net.corda.core.utilities.NonEmptySet
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.getOrThrow
 import net.corda.core.utilities.unwrap
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.services.api.VaultServiceInternal
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.testing.core.singleIdentity

testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt

@@ -21,7 +21,7 @@ import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.utilities.NetworkHostAndPort
 import net.corda.node.VersionInfo
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.internal.ServicesForResolutionImpl
 import net.corda.node.internal.cordapp.JarScanningCordappLoader
 import net.corda.node.services.api.*

testing/test-utils/src/main/kotlin/net/corda/testing/internal/MockCordappProvider.kt

@@ -6,7 +6,7 @@ import net.corda.core.internal.DEPLOYED_CORDAPP_UPLOADER
 import net.corda.core.internal.cordapp.CordappImpl
 import net.corda.core.node.services.AttachmentId
 import net.corda.core.node.services.AttachmentStorage
-import net.corda.node.cordapp.CordappLoader
+import net.corda.nodeapi.internal.cordapp.CordappLoader
 import net.corda.node.internal.cordapp.CordappProviderImpl
 import net.corda.testing.services.MockAttachmentStorage
 import java.security.PublicKey