CORDA-1939 Regression: Liquibase fails to migrate existing pre-liquibase database (#3855)

* Bugfix for corda.jar manually deployed without coping the matching corda-finance-VERSION.jar.
If the older finance cordapp version is detected (which doesn't have Liquibase migration scripts) fail node at startup and print message: "Could not create the DataSource: Detected incompatible corda-finance cordapp without database migration scripts, replace the existing corda-finance-VERSION.jar with the latest one."
Since coda-finance is an optional cordapp,the presence of Liquibase scripts is only checked if corda-finance-VERSION.jar is present in cordapps folder.
*  Allow to start using Liquibase from any point of 4.0-SNAPSHOT before Liquibase was introduced (not only from 3.0/3.X versions) - 2 database changes introduced after 3.2 but before Liquibase are now conditional database changes.

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt

@@ -106,10 +106,16 @@ class SchemaMigration(
 
     /** For existing database created before verions 4.0 add Liquibase support - creates DATABASECHANGELOG and DATABASECHANGELOGLOCK tables and mark changesets are executed. */
     private fun migrateOlderDatabaseToUseLiquibase(existingCheckpoints: Boolean): Boolean {
+        //workaround to detect that if Corda finance module is in use then the most recent version with Liquibase migration scripts was deployed
+        if (schemas.any { schema ->
+                    (schema::class.qualifiedName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.qualifiedName == "net.corda.finance.schemas.CommercialPaperSchemaV1")
+                            && schema.migrationResource == null
+                })
+            throw DatabaseMigrationException("Detected incompatible corda-finance cordapp without database migration scripts, replace the existing corda-finance-VERSION.jar with the latest one.")
+
         val isExistingDBWithoutLiquibase = dataSource.connection.use {
-            it.metaData.getTables(null, null, "NODE%", null).next() &&
+            (it.metaData.getTables(null, null, "NODE%", null).next() &&
-                    !it.metaData.getTables(null, null, "DATABASECHANGELOG", null).next() &&
+                    !it.metaData.getTables(null, null, "DATABASECHANGELOG%", null).next())
-                    !it.metaData.getTables(null, null, "DATABASECHANGELOGLOCK", null).next()
         }
         when {
             isExistingDBWithoutLiquibase && existingCheckpoints -> throw CheckpointsException()
@@ -119,8 +125,7 @@ class SchemaMigration(
 
                 dataSource.connection.use { connection ->
                     // Schema migrations pre release 4.0
-                    val preV4Baseline =
+                    val preV4Baseline = mutableListOf("migration/common.changelog-init.xml",
-                            listOf("migration/common.changelog-init.xml",
                             "migration/node-info.changelog-init.xml",
                             "migration/node-info.changelog-v1.xml",
                             "migration/node-info.changelog-v2.xml",
@@ -132,16 +137,19 @@ class SchemaMigration(
                             "migration/vault-schema.changelog-init.xml",
                             "migration/vault-schema.changelog-v3.xml",
                             "migration/vault-schema.changelog-v4.xml",
-                                    "migration/vault-schema.changelog-pkey.xml",
-                                    "migration/cash.changelog-init.xml",
-                                    "migration/cash.changelog-v1.xml",
-                                    "migration/commercial-paper.changelog-init.xml",
-                                    "migration/commercial-paper.changelog-v1.xml") +
-                                    if (schemas.any { schema -> schema.migrationResource == "node-notary.changelog-master" })
-                                        listOf("migration/node-notary.changelog-init.xml",
-                                                "migration/node-notary.changelog-v1.xml",
                             "migration/vault-schema.changelog-pkey.xml")
-                                    else emptyList()
+
+                    if (schemas.any { schema -> schema.migrationResource == "cash.changelog-master" })
+                        preV4Baseline.addAll(listOf("migration/cash.changelog-init.xml",
+                                "migration/cash.changelog-v1.xml"))
+
+                    if (schemas.any { schema -> schema.migrationResource == "commercial-paper.changelog-master" })
+                        preV4Baseline.addAll(listOf("migration/commercial-paper.changelog-init.xml",
+                                "migration/commercial-paper.changelog-v1.xml"))
+
+                    if (schemas.any { schema -> schema.migrationResource == "node-notary.changelog-master" })
+                        preV4Baseline.addAll(listOf("migration/node-notary.changelog-init.xml",
+                                "migration/node-notary.changelog-v1.xml"))
 
                     val customResourceAccessor = CustomResourceAccessor(dynamicInclude, preV4Baseline, classLoader)
                     val liquibase = Liquibase(dynamicInclude, customResourceAccessor, getLiquibaseDatabase(JdbcConnection(connection)))

node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt

@@ -69,7 +69,7 @@ class NodeSchemaService(private val extraSchemas: Set<MappedSchema> = emptySet()
      if (includeNotarySchemas) mapOf(Pair(NodeNotaryV1, SchemaOptions())) else emptyMap()
 
     fun internalSchemas() = requiredSchemas.keys + extraSchemas.filter { schema -> // when mapped schemas from the finance module are present, they are considered as internal ones
-        schema::class.simpleName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.simpleName == "net.corda.finance.schemas.CommercialPaperSchemaV1" }
+        schema::class.qualifiedName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.qualifiedName == "net.corda.finance.schemas.CommercialPaperSchemaV1" }
 
     override val schemaOptions: Map<MappedSchema, SchemaService.SchemaOptions> = requiredSchemas + extraSchemas.associateBy({ it }, { SchemaOptions() })
 

node/src/main/resources/migration/node-core.changelog-tx-mapping.xml

@@ -3,6 +3,7 @@
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
                    xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
     <changeSet author="R3.Corda" id="add_tx_mapping_column">
+        <preConditions onFail="MARK_RAN"><not><columnExists tableName="node_transactions" columnName="state_machine_run_id"/></not></preConditions>
         <addColumn tableName="node_transactions">
             <column name="state_machine_run_id" type="NVARCHAR(36)">
                 <constraints nullable="true"/>

node/src/main/resources/migration/vault-schema.changelog-v5.xml

@@ -1,9 +1,9 @@
 <?xml version="1.1" encoding="UTF-8" standalone="no"?>
 <databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
-                   xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
                    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
     <changeSet author="R3.Corda" id="add_is_relevant_column">
+        <preConditions onFail="MARK_RAN"><not><columnExists tableName="vault_states" columnName="is_relevant"/></not></preConditions>
         <addColumn tableName="vault_states">
             <column name="is_relevant" type="INT"/>
 	    </addColumn>