CORDA-2435 - Print an error and exit if the vault contains V3 states (#4619)

* Exit and raise an error if there are V3 states in the vault

* Add a test to check that old vault states are correctly detected

* Fix test to correctly close database transaction

* Update the documentation to add a warning about V3 states

* Address review comments and revert unnecessary refactor

* Address further review comments

* Add configuration option to disable exiting on V3 states

* Fix merge damage to reference.conf

* Fix issue with mock network

docs/source/app-upgrade-notes.rst

@@ -17,6 +17,10 @@ However, there are usually new features and other opt-in changes that may improv
 application that are worth considering for any actively maintained software. This guide shows you how to upgrade your app to benefit
 from the new features in the latest release.
 
+.. note:: A number of new features will only work with states created in V4 nodes at present. As a result, if a node is upgraded to use
+          Corda 4, but states created with Corda 3 are present in the node, the node will report an error and exit. Apps upgraded to use
+          Corda 4 should therefore start from a clean node. This will be fixed in Corda 4.1.
+
 .. contents::
    :depth: 3
 
@@ -342,7 +346,7 @@ states in the vault, to maintain backwards compatibility. However, it may make s
 to the node in question to query for only relevant states. See :doc:`api-vault-query.rst` for more details on how to do this. Not doing this
 may result in queries returning more states than expected if the node is using Observer node functionality (see ":doc:`tutorial-observer-nodes.rst`").
 
-Step 10. Explore other new features that may be useful
+Step 11. Explore other new features that may be useful
 ------------------------------------------------------
 
 Corda 4 adds several new APIs that help you build applications. Why not explore:

docs/source/release-notes.rst

@@ -15,6 +15,10 @@ comes with those same guarantees. States and apps valid in Corda 3 are transpare
    along with how you can adjust your app to opt-in to new features making your app more secure and
    easier to upgrade.
 
+.. note:: Currently, states created with a node running Corda 3 are incompatible with some Corda 4 features.
+   If a node running Corda 4 detects that these states are present, it will exit to prevent errors from
+   occurring while using those states. This is a temporary condition that will be fixed for Corda 4.1.
+
 Additionally, be aware that the data model upgrades are changes to the Corda consensus rules. To use
 apps that benefit from them, *all* nodes in a compatibility zone must be upgraded and the zone must be
 enforcing that upgrade. This may take time in large zones like the testnet. Please take this into

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -380,6 +380,10 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             installCordaServices()
             contractUpgradeService.start()
             vaultService.start()
+            if (!configuration.allowPreV4States && vaultService.oldStatesPresent()) {
+                stop()
+                throw OldStatesException()
+            }
             ScheduledActivityObserver.install(vaultService, schedulerService, flowLogicRefFactory)
 
             val frozenTokenizableServices = tokenizableServices!!
@@ -1073,6 +1077,9 @@ class FlowStarterImpl(private val smm: StateMachineManager, private val flowLogi
 
 class ConfigurationException(message: String) : CordaException(message)
 
+class OldStatesException : Exception("Detected states created using Corda 3 in the vault. Currently certain Corda 4 features cannot work with Corda 3" +
+        " states. See the release notes for more details. Exiting.")
+
 fun createCordaPersistence(databaseConfig: DatabaseConfig,
                            wellKnownPartyFromX500Name: (CordaX500Name) -> Party?,
                            wellKnownPartyFromAnonymous: (AbstractParty) -> Party?,

node/src/main/kotlin/net/corda/node/services/api/VaultServiceInternal.kt

@@ -9,6 +9,12 @@ import net.corda.core.transactions.WireTransaction
 interface VaultServiceInternal : VaultService {
     fun start()
 
+    /**
+     * Check that there are no states in the vault that were created using an old version of Corda. These may not be usable with new
+     * features, so prevent the node from starting up in this case.
+     */
+    fun oldStatesPresent(): Boolean
+
     /**
      * Splits the provided [txns] into batches of [WireTransaction] and [NotaryChangeWireTransaction].
      * This is required because the batches get aggregated into single updates, and we want to be able to

node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

@@ -84,6 +84,7 @@ interface NodeConfiguration {
     val cordappSignerKeyFingerprintBlacklist: List<String>
 
     val networkParameterAcceptanceSettings: NetworkParameterAcceptanceSettings
+    val allowPreV4States: Boolean
 
     companion object {
         // default to at least 8MB and a bit extra for larger heap sizes

node/src/main/kotlin/net/corda/node/services/config/NodeConfigurationImpl.kt

@@ -75,7 +75,8 @@ data class NodeConfigurationImpl(
         override val jmxReporterType: JmxReporterType? = Defaults.jmxReporterType,
         override val flowOverrides: FlowOverrideConfig?,
         override val cordappSignerKeyFingerprintBlacklist: List<String> = Defaults.cordappSignerKeyFingerprintBlacklist,
-        override val networkParameterAcceptanceSettings: NetworkParameterAcceptanceSettings = Defaults.networkParameterAcceptanceSettings
+        override val networkParameterAcceptanceSettings: NetworkParameterAcceptanceSettings = Defaults.networkParameterAcceptanceSettings,
+        override val allowPreV4States: Boolean = Defaults.allowPreV4States
 ) : NodeConfiguration {
     internal object Defaults {
         val jmxMonitoringHttpPort: Int? = null
@@ -108,6 +109,7 @@ data class NodeConfigurationImpl(
         val jmxReporterType: JmxReporterType = NodeConfiguration.defaultJmxReporterType
         val cordappSignerKeyFingerprintBlacklist: List<String> = DEV_PUB_KEY_HASHES.map { it.toString() }
         val networkParameterAcceptanceSettings: NetworkParameterAcceptanceSettings = NetworkParameterAcceptanceSettings()
+        const val allowPreV4States: Boolean = false
 
         fun cordappsDirectories(baseDirectory: Path) = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT)
 

node/src/main/kotlin/net/corda/node/services/config/schema/v1/V1NodeConfigurationSpec.kt

@@ -73,6 +73,7 @@ internal object V1NodeConfigurationSpec : Configuration.Specification<NodeConfig
     private val jarDirs by string().list().optional().withDefaultValue(Defaults.jarDirs)
     private val cordappDirectories by string().mapValid(::toPath).list().optional()
     private val cordappSignerKeyFingerprintBlacklist by string().list().optional().withDefaultValue(Defaults.cordappSignerKeyFingerprintBlacklist)
+    private val allowPreV4States by boolean().optional().withDefaultValue(Defaults.allowPreV4States)
     @Suppress("unused")
     private val custom by nestedObject().optional()
 
@@ -128,7 +129,8 @@ internal object V1NodeConfigurationSpec : Configuration.Specification<NodeConfig
                     h2port = configuration[h2port],
                     jarDirs = configuration[jarDirs],
                     cordappDirectories = cordappDirectories,
-                    cordappSignerKeyFingerprintBlacklist = configuration[cordappSignerKeyFingerprintBlacklist]
+                    cordappSignerKeyFingerprintBlacklist = configuration[cordappSignerKeyFingerprintBlacklist],
+                    allowPreV4States = configuration[allowPreV4States]
             ))
         } catch (e: Exception) {
             return when (e) {

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -458,6 +458,45 @@ class NodeVaultService(
         return claimedStates
     }
 
+    private fun getPersistentStateCount(session: Session): Long {
+        val criteriaQuery = criteriaBuilder.createQuery(Long::class.java)
+        val queryRootStates = criteriaQuery.from(VaultSchemaV1.VaultStates::class.java)
+        criteriaQuery.select(criteriaBuilder.countDistinct(queryRootStates.get<PersistentStateRef>("stateRef").get<String>("txId")))
+        val query = session.createQuery(criteriaQuery)
+        val result = query.singleResult
+
+        log.debug("Found $result vault states")
+        return result
+    }
+
+    private fun getPersistentPartyCount(session: Session): Long {
+        val criteriaQuery = criteriaBuilder.createQuery(Long::class.java)
+        val queryRootPersistentStates = criteriaQuery.from(VaultSchemaV1.PersistentParty::class.java)
+        criteriaQuery.select(criteriaBuilder.countDistinct(queryRootPersistentStates
+                .get<VaultSchemaV1.PersistentStateRefAndKey>("compositeKey")
+                .get<PersistentStateRef>("stateRef")
+                .get<String>("txId")))
+        val query = session.createQuery(criteriaQuery)
+        val result = query.singleResult
+
+        log.debug("Found $result persistent party entries")
+        return result
+    }
+
+    override fun oldStatesPresent(): Boolean {
+        log.info("Checking for states in vault from a previous version")
+        val oldStatesPresent = database.transaction {
+            val session = getSession()
+            val persistentStates = getPersistentStateCount(session)
+            val stateParties = getPersistentPartyCount(session)
+
+            // There are no V3 states if all the states in the vault are also in the state_party table
+            stateParties != persistentStates
+        }
+        log.info("Finished checking for old states. Old states present: $oldStatesPresent")
+        return oldStatesPresent
+    }
+
     @VisibleForTesting
     internal fun isRelevant(state: ContractState, myKeys: Set<PublicKey>): Boolean {
         val keysToCheck = when (state) {
@@ -491,7 +530,6 @@ class NodeVaultService(
 
             val criteriaQuery = criteriaBuilder.createQuery(Tuple::class.java)
             val queryRootVaultStates = criteriaQuery.from(VaultSchemaV1.VaultStates::class.java)
-
             // TODO: revisit (use single instance of parser for all queries)
             val criteriaParser = HibernateQueryCriteriaParser(contractStateType, contractStateTypeMappings, criteriaBuilder, criteriaQuery, queryRootVaultStates)
 

node/src/main/resources/reference.conf

@@ -27,4 +27,4 @@ flowTimeout {
     backoffBase = 1.8
 }
 jmxReporterType = JOLOKIA
-
+allowPreV4States = false

node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt

@@ -15,6 +15,7 @@ import net.corda.core.node.services.*
 import net.corda.core.node.services.vault.PageSpecification
 import net.corda.core.node.services.vault.QueryCriteria
 import net.corda.core.node.services.vault.QueryCriteria.*
+import net.corda.core.schemas.PersistentStateRef
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.NonEmptySet
@@ -28,6 +29,7 @@ import net.corda.finance.utils.sumCash
 import net.corda.node.services.api.IdentityServiceInternal
 import net.corda.node.services.api.WritableTransactionStorage
 import net.corda.nodeapi.internal.persistence.CordaPersistence
+import net.corda.nodeapi.internal.persistence.currentDBSession
 import net.corda.testing.common.internal.testNetworkParameters
 import net.corda.testing.contracts.DummyContract
 import net.corda.testing.contracts.DummyState
@@ -41,6 +43,7 @@ import org.assertj.core.api.Assertions.assertThatExceptionOfType
 import org.junit.*
 import rx.observers.TestSubscriber
 import java.math.BigDecimal
+import java.time.Instant
 import java.util.*
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.Executors
@@ -885,6 +888,45 @@ class NodeVaultServiceTest {
         // We should never see 2 or 7.
     }
 
+    @Test
+    fun `Checking for old vault states works correctly`() {
+        fun createTx(number: Int, vararg participants: Party): SignedTransaction {
+            return services.signInitialTransaction(TransactionBuilder(DUMMY_NOTARY).apply {
+                addOutputState(DummyState(number, participants.toList()), DummyContract.PROGRAM_ID)
+                addCommand(DummyCommandData, listOf(megaCorp.publicKey))
+            })
+        }
+
+        services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(1, megaCorp.party)))
+        services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(2, miniCorp.party)))
+        services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(3, miniCorp.party, megaCorp.party)))
+        services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(4, miniCorp.party)))
+        services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(5, bankOfCorda.party)))
+        services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(6, megaCorp.party, bankOfCorda.party)))
+        services.recordTransactions(StatesToRecord.NONE, listOf(createTx(7, bankOfCorda.party)))
+
+        database.transaction {
+            assertEquals(false, vaultService.oldStatesPresent())
+        }
+
+        database.transaction {
+            val session = currentDBSession()
+            val stateToAdd = VaultSchemaV1.VaultStates(
+                notary = DUMMY_NOTARY,
+                contractStateClassName = DummyState::class.java.toString(),
+                stateStatus = Vault.StateStatus.UNCONSUMED,
+                recordedTime = Instant.now(),
+                relevancyStatus = Vault.RelevancyStatus.RELEVANT,
+                constraintType = Vault.ConstraintInfo.Type.ALWAYS_ACCEPT
+            )
+            val persistentStateRef = PersistentStateRef("C517D22982867E517E3E933A99C8F53658C8708A7B84FE37C36D0D8AF1EA167C", 23)
+            stateToAdd.stateRef = persistentStateRef
+            session.save(stateToAdd)
+
+            assertEquals(true, vaultService.oldStatesPresent())
+        }
+    }
+
     @Test
     @Ignore
     fun `trackByCriteria filters updates and snapshots`() {

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt

@@ -615,6 +615,7 @@ private fun mockNodeConfiguration(certificatesDirectory: Path): NodeConfiguratio
         doReturn(5.seconds.toMillis()).whenever(it).additionalNodeInfoPollingFrequencyMsec
         doReturn(null).whenever(it).devModeOptions
         doReturn(NetworkParameterAcceptanceSettings()).whenever(it).networkParameterAcceptanceSettings
+        doReturn(true).whenever(it).allowPreV4States
     }
 }