From 487cad7d06bc343696bef6214fdc4ed9a26c1d6a Mon Sep 17 00:00:00 2001 From: szymonsztuka Date: Thu, 23 Aug 2018 16:30:02 +0100 Subject: [PATCH 1/4] CORDA-1471 Database schema setup for internal tables via Liquibase (#3815) Internal tables (the tables from node and finance modules) are now tracked /created by Liquibase script. Tables backing MappedSchemma in Cordapps are created by Hibernate (as before). The PR scope added Liquibase library, setup code SchemaMigration and XML scripts and from Enterprise. For existing database installation - the node will auto-upgrade to use Liquibase. Method migrateOlderDatabaseToUseLiquibase checks for any 3.X existing Corda database to upgrade database to use Liquibase. When the existing database without Liquibase integral tables is detected, the node (at startup) will create Liquibase tracking tables and fill them with all migration scripts (marked as done), this ensure the database will look as it would use Liquibase from the beginning. The database changes gradually introduced by the subsequent 3.X releases (3.1, 3.2) are conditionally run by Liquibase. --- build.gradle | 1 + .../net/corda/core/schemas/CommonSchema.kt | 2 + .../net/corda/core/schemas/PersistentTypes.kt | 6 + .../net/corda/finance/schemas/CashSchemaV1.kt | 3 + .../schemas/CommercialPaperSchemaV1.kt | 3 + .../migration/cash.changelog-init.xml | 37 ++++ .../migration/cash.changelog-master.xml | 7 + .../resources/migration/cash.changelog-v1.xml | 27 +++ .../commercial-paper.changelog-init.xml | 45 +++++ .../commercial-paper.changelog-master.xml | 7 + .../commercial-paper.changelog-v1.xml | 27 +++ node-api/build.gradle | 5 + .../nodeapi/internal/MigrationHelpers.kt | 32 ++++ .../internal/persistence/CordaPersistence.kt | 32 ---- .../internal/persistence/SchemaMigration.kt | 181 ++++++++++++++++++ ...gratedAttachmentContractsTableNameTests.kt | 70 ------- .../net/corda/node/internal/AbstractNode.kt | 22 ++- .../node/internal/schemas/NodeInfoSchema.kt | 2 + .../node/services/api/CheckpointStorage.kt | 11 +- .../persistence/DBCheckpointStorage.kt | 16 ++ .../node/services/schema/NodeSchemaService.kt | 19 +- .../corda/node/services/vault/VaultSchema.kt | 3 + .../migration/common.changelog-init.xml | 16 ++ .../migration/common.changelog-master.xml | 9 + .../migration/node-core.changelog-init.xml | 158 +++++++++++++++ .../migration/node-core.changelog-master.xml | 16 ++ .../migration/node-core.changelog-pkey.xml | 41 ++++ .../node-core.changelog-postgres-blob.xml | 25 +++ .../node-core.changelog-tx-mapping.xml | 17 ++ .../migration/node-core.changelog-v3.xml | 30 +++ .../migration/node-core.changelog-v4.xml | 37 ++++ .../migration/node-core.changelog-v5.xml | 10 + .../migration/node-core.changelog-v8.xml | 17 ++ .../migration/node-info.changelog-init.xml | 84 ++++++++ .../migration/node-info.changelog-master.xml | 11 ++ .../migration/node-info.changelog-v1.xml | 14 ++ .../migration/node-info.changelog-v2.xml | 17 ++ .../migration/node-notary.changelog-init.xml | 77 ++++++++ .../node-notary.changelog-master.xml | 12 ++ .../migration/node-notary.changelog-pkey.xml | 21 ++ .../migration/node-notary.changelog-v1.xml | 16 ++ .../migration/vault-schema.changelog-init.xml | 138 +++++++++++++ .../vault-schema.changelog-master.xml | 12 ++ .../migration/vault-schema.changelog-pkey.xml | 35 ++++ .../migration/vault-schema.changelog-v3.xml | 16 ++ .../migration/vault-schema.changelog-v4.xml | 33 ++++ .../migration/vault-schema.changelog-v5.xml | 15 ++ .../services/vault/VaultQueryJavaTests.java | 2 - .../vault/VaultQueryExceptionsTests.kt | 4 +- .../net/corda/testing/node/MockServices.kt | 2 +- 50 files changed, 1318 insertions(+), 125 deletions(-) create mode 100644 finance/src/main/resources/migration/cash.changelog-init.xml create mode 100644 finance/src/main/resources/migration/cash.changelog-master.xml create mode 100644 finance/src/main/resources/migration/cash.changelog-v1.xml create mode 100644 finance/src/main/resources/migration/commercial-paper.changelog-init.xml create mode 100644 finance/src/main/resources/migration/commercial-paper.changelog-master.xml create mode 100644 finance/src/main/resources/migration/commercial-paper.changelog-v1.xml create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt create mode 100644 node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt delete mode 100644 node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt create mode 100644 node/src/main/resources/migration/common.changelog-init.xml create mode 100644 node/src/main/resources/migration/common.changelog-master.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-init.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-master.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-pkey.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-postgres-blob.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-tx-mapping.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v3.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v4.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v5.xml create mode 100644 node/src/main/resources/migration/node-core.changelog-v8.xml create mode 100644 node/src/main/resources/migration/node-info.changelog-init.xml create mode 100644 node/src/main/resources/migration/node-info.changelog-master.xml create mode 100644 node/src/main/resources/migration/node-info.changelog-v1.xml create mode 100644 node/src/main/resources/migration/node-info.changelog-v2.xml create mode 100644 node/src/main/resources/migration/node-notary.changelog-init.xml create mode 100644 node/src/main/resources/migration/node-notary.changelog-master.xml create mode 100644 node/src/main/resources/migration/node-notary.changelog-pkey.xml create mode 100644 node/src/main/resources/migration/node-notary.changelog-v1.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-init.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-master.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-pkey.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-v3.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-v4.xml create mode 100644 node/src/main/resources/migration/vault-schema.changelog-v5.xml diff --git a/build.gradle b/build.gradle index 1710ae0987..eda5774878 100644 --- a/build.gradle +++ b/build.gradle @@ -57,6 +57,7 @@ buildscript { ext.shiro_version = '1.4.0' ext.shadow_version = '2.0.4' ext.artifactory_plugin_version = constants.getProperty('artifactoryPluginVersion') + ext.liquibase_version = '3.6.2' ext.artifactory_contextUrl = 'https://ci-artifactory.corda.r3cev.com/artifactory' ext.snake_yaml_version = constants.getProperty('snakeYamlVersion') ext.docker_compose_rule_version = '0.33.0' diff --git a/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt b/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt index 035f248e10..3a41f0ead8 100644 --- a/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt +++ b/core/src/main/kotlin/net/corda/core/schemas/CommonSchema.kt @@ -18,6 +18,8 @@ object CommonSchema */ object CommonSchemaV1 : MappedSchema(schemaFamily = CommonSchema.javaClass, version = 1, mappedTypes = emptyList()) { + override val migrationResource = "common.changelog-master" + @MappedSuperclass class LinearState( /** [ContractState] attributes */ diff --git a/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt b/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt index 838e1bfe0e..07f88f3f43 100644 --- a/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt +++ b/core/src/main/kotlin/net/corda/core/schemas/PersistentTypes.kt @@ -45,6 +45,12 @@ open class MappedSchema(schemaFamily: Class<*>, val version: Int, val mappedTypes: Iterable>) { val name: String = schemaFamily.name + + /** + * Optional classpath resource containing the database changes for the [mappedTypes] + */ + open val migrationResource: String? = null + override fun toString(): String = "${this.javaClass.simpleName}(name=$name, version=$version)" override fun equals(other: Any?): Boolean { diff --git a/finance/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt b/finance/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt index 3dfa84d354..166cfc764b 100644 --- a/finance/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt +++ b/finance/src/main/kotlin/net/corda/finance/schemas/CashSchemaV1.kt @@ -20,6 +20,9 @@ object CashSchema */ @CordaSerializable object CashSchemaV1 : MappedSchema(schemaFamily = CashSchema.javaClass, version = 1, mappedTypes = listOf(PersistentCashState::class.java)) { + + override val migrationResource = "cash.changelog-master" + @Entity @Table(name = "contract_cash_states", indexes = [Index(name = "ccy_code_idx", columnList = "ccy_code"), Index(name = "pennies_idx", columnList = "pennies")]) class PersistentCashState( diff --git a/finance/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt b/finance/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt index 73f725573f..87c31cfc4a 100644 --- a/finance/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt +++ b/finance/src/main/kotlin/net/corda/finance/schemas/CommercialPaperSchemaV1.kt @@ -23,6 +23,9 @@ object CommercialPaperSchema */ @CordaSerializable object CommercialPaperSchemaV1 : MappedSchema(schemaFamily = CommercialPaperSchema.javaClass, version = 1, mappedTypes = listOf(PersistentCommercialPaperState::class.java)) { + + override val migrationResource = "commercial-paper.changelog-master" + @Entity @Table(name = "cp_states", indexes = [Index(name = "ccy_code_index", columnList = "ccy_code"), Index(name = "maturity_index", columnList = "maturity_instant"), Index(name = "face_value_index", columnList = "face_value")]) class PersistentCommercialPaperState( diff --git a/finance/src/main/resources/migration/cash.changelog-init.xml b/finance/src/main/resources/migration/cash.changelog-init.xml new file mode 100644 index 0000000000..3e83f5226e --- /dev/null +++ b/finance/src/main/resources/migration/cash.changelog-init.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/finance/src/main/resources/migration/cash.changelog-master.xml b/finance/src/main/resources/migration/cash.changelog-master.xml new file mode 100644 index 0000000000..c01eeff1e4 --- /dev/null +++ b/finance/src/main/resources/migration/cash.changelog-master.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/finance/src/main/resources/migration/cash.changelog-v1.xml b/finance/src/main/resources/migration/cash.changelog-v1.xml new file mode 100644 index 0000000000..a657a9f075 --- /dev/null +++ b/finance/src/main/resources/migration/cash.changelog-v1.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/finance/src/main/resources/migration/commercial-paper.changelog-init.xml b/finance/src/main/resources/migration/commercial-paper.changelog-init.xml new file mode 100644 index 0000000000..80975378be --- /dev/null +++ b/finance/src/main/resources/migration/commercial-paper.changelog-init.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/finance/src/main/resources/migration/commercial-paper.changelog-master.xml b/finance/src/main/resources/migration/commercial-paper.changelog-master.xml new file mode 100644 index 0000000000..a91edb5f2d --- /dev/null +++ b/finance/src/main/resources/migration/commercial-paper.changelog-master.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/finance/src/main/resources/migration/commercial-paper.changelog-v1.xml b/finance/src/main/resources/migration/commercial-paper.changelog-v1.xml new file mode 100644 index 0000000000..12dea2be06 --- /dev/null +++ b/finance/src/main/resources/migration/commercial-paper.changelog-v1.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node-api/build.gradle b/node-api/build.gradle index c13bafd99f..9da77eaf2e 100644 --- a/node-api/build.gradle +++ b/node-api/build.gradle @@ -35,6 +35,11 @@ dependencies { // For caches rather than guava compile "com.github.ben-manes.caffeine:caffeine:$caffeine_version" + // For db migration + compile "org.liquibase:liquibase-core:$liquibase_version" + compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version" + runtime 'com.mattbertolini:liquibase-slf4j:2.0.0' + // Unit testing helpers. testCompile "junit:junit:$junit_version" testCompile "org.assertj:assertj-core:$assertj_version" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt new file mode 100644 index 0000000000..52d20360c1 --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt @@ -0,0 +1,32 @@ +package net.corda.nodeapi.internal + +import com.google.common.base.CaseFormat +import net.corda.core.schemas.MappedSchema + +object MigrationHelpers { + private const val MIGRATION_PREFIX = "migration" + private const val DEFAULT_MIGRATION_EXTENSION = "xml" + private const val CHANGELOG_NAME = "changelog-master" + private val possibleMigrationExtensions = listOf(".xml", ".sql", ".yml", ".json") + + fun getMigrationResource(schema: MappedSchema, classLoader: ClassLoader): String? { + val declaredMigration = schema.migrationResource + + if (declaredMigration == null) { + // try to apply the naming convention and find the migration file in the classpath + val resource = migrationResourceNameForSchema(schema) + return possibleMigrationExtensions.map { "$resource$it" }.firstOrNull { + classLoader.getResource(it) != null + } + } + + return "$MIGRATION_PREFIX/$declaredMigration.$DEFAULT_MIGRATION_EXTENSION" + } + + // SchemaName will be transformed from camel case to lower_hyphen then add ".changelog-master" + private fun migrationResourceNameForSchema(schema: MappedSchema): String { + val name: String = schema::class.simpleName!! + val fileName = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, name) + return "$MIGRATION_PREFIX/$fileName.$CHANGELOG_NAME" + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index 10347e8de1..9bdbeccfcd 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -83,8 +83,6 @@ class CordaPersistence( // Check not in read-only mode. transaction { check(!connection.metaData.isReadOnly) { "Database should not be readonly." } - checkCorrectAttachmentsContractsTableName(connection) - checkCorrectCheckpointTypeOnPostgres(connection) } } @@ -272,33 +270,3 @@ private fun Throwable.hasSQLExceptionCause(): Boolean = } class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception() - -class DatabaseIncompatibleException(override val message: String?, override val cause: Throwable? = null) : Exception() - -private fun checkCorrectAttachmentsContractsTableName(connection: Connection) { - val correctName = "NODE_ATTACHMENTS_CONTRACTS" - val incorrectV30Name = "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME" - val incorrectV31Name = "NODE_ATTCHMENTS_CONTRACTS" - - fun warning(incorrectName: String, version: String) = "The database contains the older table name $incorrectName instead of $correctName, see upgrade notes to migrate from Corda database version $version https://docs.corda.net/head/upgrade-notes.html." - - if (!connection.metaData.getTables(null, null, correctName, null).next()) { - if (connection.metaData.getTables(null, null, incorrectV30Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV30Name, "3.0")) } - if (connection.metaData.getTables(null, null, incorrectV31Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV31Name, "3.1")) } - } -} - -private fun checkCorrectCheckpointTypeOnPostgres(connection: Connection) { - val metaData = connection.metaData - if (metaData.getDatabaseProductName() != "PostgreSQL") { - return - } - - val result = metaData.getColumns(null, null, "node_checkpoints", "checkpoint_value") - if (result.next()) { - val type = result.getString("TYPE_NAME") - if (type != "bytea") { - throw DatabaseIncompatibleException("The type of the 'checkpoint_value' table must be 'bytea', but 'oid' was found. See upgrade notes to migrate from Corda database version 3.1 https://docs.corda.net/head/upgrade-notes.html.") - } - } -} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt new file mode 100644 index 0000000000..4cafba895c --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt @@ -0,0 +1,181 @@ +package net.corda.nodeapi.internal.persistence + +import com.fasterxml.jackson.databind.ObjectMapper +import liquibase.Contexts +import liquibase.LabelExpression +import liquibase.Liquibase +import liquibase.database.Database +import liquibase.database.DatabaseFactory +import liquibase.database.jvm.JdbcConnection +import liquibase.resource.ClassLoaderResourceAccessor +import net.corda.nodeapi.internal.MigrationHelpers.getMigrationResource +import net.corda.core.schemas.MappedSchema +import net.corda.core.utilities.contextLogger +import java.io.ByteArrayInputStream +import java.io.InputStream +import java.io.Writer +import javax.sql.DataSource + +class SchemaMigration( + val schemas: Set, + val dataSource: DataSource, + private val databaseConfig: DatabaseConfig, + private val classLoader: ClassLoader = Thread.currentThread().contextClassLoader) { + + companion object { + private val logger = contextLogger() + } + + /** + * Main entry point to the schema migration. + * Called during node startup. + */ + fun nodeStartup(existingCheckpoints: Boolean) { + when { + databaseConfig.initialiseSchema -> { + migrateOlderDatabaseToUseLiquibase(existingCheckpoints) + runMigration(existingCheckpoints) + } + else -> checkState() + } + } + + /** + * Will run the Liquibase migration on the actual database. + */ + private fun runMigration(existingCheckpoints: Boolean) = doRunMigration(run = true, check = false, existingCheckpoints = existingCheckpoints) + + /** + * Ensures that the database is up to date with the latest migration changes. + */ + private fun checkState() = doRunMigration(run = false, check = true) + + /** Create a resourse accessor that aggregates the changelogs included in the schemas into one dynamic stream. */ + private class CustomResourceAccessor(val dynamicInclude: String, val changelogList: List, classLoader: ClassLoader) : ClassLoaderResourceAccessor(classLoader) { + override fun getResourcesAsStream(path: String): Set { + if (path == dynamicInclude) { + // Create a map in Liquibase format including all migration files. + val includeAllFiles = mapOf("databaseChangeLog" to changelogList.filter { it != null }.map { file -> mapOf("include" to mapOf("file" to file)) }) + + // Transform it to json. + val includeAllFilesJson = ObjectMapper().writeValueAsBytes(includeAllFiles) + + // Return the json as a stream. + return setOf(ByteArrayInputStream(includeAllFilesJson)) + } + return super.getResourcesAsStream(path)?.take(1)?.toSet() ?: emptySet() + } + } + + private fun doRunMigration(run: Boolean, check: Boolean, existingCheckpoints: Boolean? = null) { + + // Virtual file name of the changelog that includes all schemas. + val dynamicInclude = "master.changelog.json" + + dataSource.connection.use { connection -> + + // Collect all changelog file referenced in the included schemas. + // For backward compatibility reasons, when failOnMigrationMissing=false, we don't manage CorDapps via Liquibase but use the hibernate hbm2ddl=update. + val changelogList = schemas.map { mappedSchema -> + val resource = getMigrationResource(mappedSchema, classLoader) + when { + resource != null -> resource + else -> throw MissingMigrationException(mappedSchema) + } + } + + val customResourceAccessor = CustomResourceAccessor(dynamicInclude, changelogList, classLoader) + + val liquibase = Liquibase(dynamicInclude, customResourceAccessor, getLiquibaseDatabase(JdbcConnection(connection))) + + val unRunChanges = liquibase.listUnrunChangeSets(Contexts(), LabelExpression()) + + when { + (run && !check) && (unRunChanges.isNotEmpty() && existingCheckpoints!!) -> throw CheckpointsException() // Do not allow database migration when there are checkpoints + run && !check -> liquibase.update(Contexts()) + check && !run && unRunChanges.isNotEmpty() -> throw OutstandingDatabaseChangesException(unRunChanges.size) + check && !run -> {} // Do nothing will be interpreted as "check succeeded" + else -> throw IllegalStateException("Invalid usage.") + } + } + } + + private fun getLiquibaseDatabase(conn: JdbcConnection): Database { + return DatabaseFactory.getInstance().findCorrectDatabaseImplementation(conn) + } + + /** For existing database created before verions 4.0 add Liquibase support - creates DATABASECHANGELOG and DATABASECHANGELOGLOCK tables and mark changesets are executed. */ + private fun migrateOlderDatabaseToUseLiquibase(existingCheckpoints: Boolean): Boolean { + val isExistingDBWithoutLiquibase = dataSource.connection.use { + it.metaData.getTables(null, null, "NODE%", null).next() && + !it.metaData.getTables(null, null, "DATABASECHANGELOG", null).next() && + !it.metaData.getTables(null, null, "DATABASECHANGELOGLOCK", null).next() + } + when { + isExistingDBWithoutLiquibase && existingCheckpoints -> throw CheckpointsException() + isExistingDBWithoutLiquibase -> { + // Virtual file name of the changelog that includes all schemas. + val dynamicInclude = "master.changelog.json" + + dataSource.connection.use { connection -> + // Schema migrations pre release 4.0 + val preV4Baseline = + listOf("migration/common.changelog-init.xml", + "migration/node-info.changelog-init.xml", + "migration/node-info.changelog-v1.xml", + "migration/node-info.changelog-v2.xml", + "migration/node-core.changelog-init.xml", + "migration/node-core.changelog-v3.xml", + "migration/node-core.changelog-v4.xml", + "migration/node-core.changelog-v5.xml", + "migration/node-core.changelog-pkey.xml", + "migration/vault-schema.changelog-init.xml", + "migration/vault-schema.changelog-v3.xml", + "migration/vault-schema.changelog-v4.xml", + "migration/vault-schema.changelog-pkey.xml", + "migration/cash.changelog-init.xml", + "migration/cash.changelog-v1.xml", + "migration/commercial-paper.changelog-init.xml", + "migration/commercial-paper.changelog-v1.xml") + + if (schemas.any { schema -> schema.migrationResource == "node-notary.changelog-master" }) + listOf("migration/node-notary.changelog-init.xml", + "migration/node-notary.changelog-v1.xml", + "migration/vault-schema.changelog-pkey.xml") + else emptyList() + + val customResourceAccessor = CustomResourceAccessor(dynamicInclude, preV4Baseline, classLoader) + val liquibase = Liquibase(dynamicInclude, customResourceAccessor, getLiquibaseDatabase(JdbcConnection(connection))) + liquibase.changeLogSync(Contexts(), LabelExpression()) + } + } + } + return isExistingDBWithoutLiquibase + } +} + +open class DatabaseMigrationException(message: String) : IllegalArgumentException(message) { + override val message: String = super.message!! +} + +class MissingMigrationException(@Suppress("MemberVisibilityCanBePrivate") val mappedSchema: MappedSchema) : DatabaseMigrationException(errorMessageFor(mappedSchema)) { + internal companion object { + fun errorMessageFor(mappedSchema: MappedSchema): String = "No migration defined for schema: ${mappedSchema.name} v${mappedSchema.version}" + } +} + +class OutstandingDatabaseChangesException(@Suppress("MemberVisibilityCanBePrivate") private val count: Int) : DatabaseMigrationException(errorMessageFor(count)) { + internal companion object { + fun errorMessageFor(count: Int): String = "There are $count outstanding database changes that need to be run." + } +} + +class CheckpointsException : DatabaseMigrationException("Attempting to update the database while there are flows in flight. " + + "This is dangerous because the node might not be able to restore the flows correctly and could consequently fail. " + + "Updating the database would make reverting to the previous version more difficult. " + + "Please drain your node first. See: https://docs.corda.net/upgrading-cordapps.html#flow-drains") + +class DatabaseIncompatibleException(@Suppress("MemberVisibilityCanBePrivate") private val reason: String) : DatabaseMigrationException(errorMessageFor(reason)) { + internal companion object { + fun errorMessageFor(reason: String): String = "Incompatible database schema version detected, please run the node with configuration option database.initialiseSchema=true. Reason: $reason" + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt b/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt deleted file mode 100644 index aabc23e0e2..0000000000 --- a/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt +++ /dev/null @@ -1,70 +0,0 @@ -package net.corda.node.persistence - -import net.corda.client.rpc.CordaRPCClient -import net.corda.core.internal.packageName -import net.corda.core.messaging.startFlow -import net.corda.core.utilities.getOrThrow -import net.corda.node.services.Permissions -import net.corda.testMessage.Message -import net.corda.testMessage.MessageState -import net.corda.testing.core.singleIdentity -import net.corda.testing.driver.DriverParameters -import net.corda.testing.driver.driver -import net.corda.testing.node.User -import org.junit.Test -import java.nio.file.Path -import java.sql.DriverManager -import kotlin.test.assertFailsWith -import kotlin.test.assertFalse -import kotlin.test.assertTrue - -class FailNodeOnNotMigratedAttachmentContractsTableNameTests { - @Test - fun `node fails when detecting table name not migrated from version 3 dot 0`() { - `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME") - } - - @Test - fun `node fails when detecting table name not migrated from version 3 dot 1`() { - `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTCHMENTS_CONTRACTS") - } - - private fun `node fails when not detecting compatible table name`(tableNameFromMapping: String, tableNameInDB: String) { - val user = User("mark", "dadada", setOf(Permissions.startFlow(), Permissions.invokeRpc("vaultQuery"))) - val message = Message("Hello world!") - val baseDir: Path = driver(DriverParameters( - inMemoryDB = false, - startNodesInProcess = isQuasarAgentSpecified(), - extraCordappPackagesToScan = listOf(MessageState::class.packageName) - )) { - val (nodeName, baseDir) = { - val nodeHandle = startNode(rpcUsers = listOf(user)).getOrThrow() - val nodeName = nodeHandle.nodeInfo.singleIdentity().name - CordaRPCClient(nodeHandle.rpcAddress).start(user.username, user.password).use { - it.proxy.startFlow(::SendMessageFlow, message, defaultNotaryIdentity).returnValue.getOrThrow() - } - nodeHandle.stop() - Pair(nodeName, nodeHandle.baseDirectory) - }() - - // replace the correct table name with one from the former release - DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use { - it.createStatement().execute("ALTER TABLE $tableNameFromMapping RENAME TO $tableNameInDB") - it.commit() - } - assertFailsWith(net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException::class) { - val nodeHandle = startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow() - nodeHandle.stop() - } - baseDir - } - - // check that the node didn't recreated the correct table matching it's entity mapping - val (hasTableFromMapping, hasTableFromDB) = DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use { - Pair(it.metaData.getTables(null, null, tableNameFromMapping, null).next(), - it.metaData.getTables(null, null, tableNameInDB, null).next()) - } - assertFalse(hasTableFromMapping) - assertTrue(hasTableFromDB) - } -} \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 27fc2e842e..64e5fb7cfc 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -25,6 +25,7 @@ import net.corda.core.internal.uncheckedCast import net.corda.core.messaging.* import net.corda.core.node.* import net.corda.core.node.services.* +import net.corda.core.schemas.MappedSchema import net.corda.core.serialization.SerializationWhitelist import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SingletonSerializeAsToken @@ -66,10 +67,7 @@ import net.corda.nodeapi.internal.DevIdentityGenerator import net.corda.nodeapi.internal.NodeInfoAndSigned import net.corda.nodeapi.internal.SignedNodeInfo import net.corda.nodeapi.internal.crypto.X509Utilities -import net.corda.nodeapi.internal.persistence.CordaPersistence -import net.corda.nodeapi.internal.persistence.CouldNotCreateDataSourceException -import net.corda.nodeapi.internal.persistence.DatabaseConfig -import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException +import net.corda.nodeapi.internal.persistence.* import net.corda.nodeapi.internal.storeLegalIdentity import net.corda.tools.shell.InteractiveShell import org.apache.activemq.artemis.utils.ReusableLatch @@ -732,7 +730,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, protected open fun startDatabase() { val props = configuration.dataSourceProperties if (props.isEmpty) throw DatabaseConfigurationException("There must be a database configured.") - database.startHikariPool(props) + database.startHikariPool(props, configuration.database, schemaService.internalSchemas()) // Now log the vendor string as this will also cause a connection to be tested eagerly. logVendorString(database, log) } @@ -993,9 +991,10 @@ fun configureDatabase(hikariProperties: Properties, databaseConfig: DatabaseConfig, wellKnownPartyFromX500Name: (CordaX500Name) -> Party?, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, - schemaService: SchemaService = NodeSchemaService()): CordaPersistence { + schemaService: SchemaService = NodeSchemaService(), + internalSchemas: Set = NodeSchemaService().internalSchemas()): CordaPersistence { val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService, hikariProperties) - persistence.startHikariPool(hikariProperties) + persistence.startHikariPool(hikariProperties, databaseConfig, internalSchemas) return persistence } @@ -1014,14 +1013,17 @@ fun createCordaPersistence(databaseConfig: DatabaseConfig, return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, attributeConverters) } -fun CordaPersistence.startHikariPool(hikariProperties: Properties) { +fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set) { try { - start(DataSourceFactory.createDataSource(hikariProperties)) + val dataSource = DataSourceFactory.createDataSource(hikariProperties) + val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig) + schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + start(dataSource) } catch (ex: Exception) { when { ex is HikariPool.PoolInitializationException -> throw CouldNotCreateDataSourceException("Could not connect to the database. Please check your JDBC connection URL, or the connectivity to the database.", ex) ex.cause is ClassNotFoundException -> throw CouldNotCreateDataSourceException("Could not find the database driver class. Please add it to the 'drivers' folder. See: https://docs.corda.net/corda-configuration-file.html") - ex is DatabaseIncompatibleException -> throw ex + ex is OutstandingDatabaseChangesException -> throw (DatabaseIncompatibleException(ex.message)) else -> throw CouldNotCreateDataSourceException("Could not create the DataSource: ${ex.message}", ex) } } diff --git a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt index 69332d0e69..13c57cf8da 100644 --- a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt +++ b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt @@ -19,6 +19,8 @@ object NodeInfoSchemaV1 : MappedSchema( version = 1, mappedTypes = listOf(PersistentNodeInfo::class.java, DBPartyAndCertificate::class.java, DBHostAndPort::class.java, NodePropertiesPersistentStore.DBNodeProperty::class.java) ) { + override val migrationResource = "node-info.changelog-master" + @Entity @Table(name = "node_infos") class PersistentNodeInfo( diff --git a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt index dadaced6e5..b463372909 100644 --- a/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/api/CheckpointStorage.kt @@ -1,9 +1,9 @@ package net.corda.node.services.api -import net.corda.core.cordapp.Cordapp import net.corda.core.flows.StateMachineRunId import net.corda.core.serialization.SerializedBytes import net.corda.node.services.statemachine.Checkpoint +import java.sql.Connection import java.util.stream.Stream /** @@ -20,7 +20,6 @@ interface CheckpointStorage { */ fun updateCheckpoint(id: StateMachineRunId, checkpoint: SerializedBytes) - /** * Remove existing checkpoint from the store. * @return whether the id matched a checkpoint that was removed. @@ -38,4 +37,12 @@ interface CheckpointStorage { * underlying database connection is closed, so any processing should happen before it is closed. */ fun getAllCheckpoints(): Stream>> + + /** + * This needs to run before Hibernate is initialised. + * + * @param connection The SQL Connection. + * @return the number of checkpoints stored in the database. + */ + fun getCheckpointCount(connection: Connection): Long } \ No newline at end of file diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index ccf00e0459..85a2a5bc87 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -16,6 +16,8 @@ import javax.persistence.Column import javax.persistence.Entity import javax.persistence.Id import org.hibernate.annotations.Type +import java.sql.Connection +import java.sql.SQLException /** * Simple checkpoint key value storage in DB. @@ -75,4 +77,18 @@ class DBCheckpointStorage : CheckpointStorage { StateMachineRunId(UUID.fromString(it.checkpointId)) to SerializedBytes(it.checkpoint) } } + + override fun getCheckpointCount(connection: Connection): Long = + try { + connection.prepareStatement("select count(*) from node_checkpoints").use { ps -> + ps.executeQuery().use { rs -> + rs.next() + rs.getLong(1) + } + } + } catch (e: SQLException) { + // Happens when the table was not created yet. + 0L + } + } diff --git a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt index 80fca08e14..8116afc681 100644 --- a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt +++ b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt @@ -29,7 +29,7 @@ import net.corda.node.services.vault.VaultSchemaV1 * TODO: support plugins for schema version upgrading or custom mapping not supported by original [QueryableState]. * TODO: create whitelisted tables when a CorDapp is first installed */ -class NodeSchemaService(extraSchemas: Set = emptySet(), includeNotarySchemas: Boolean = false) : SchemaService, SingletonSerializeAsToken() { +class NodeSchemaService(private val extraSchemas: Set = emptySet(), includeNotarySchemas: Boolean = false) : SchemaService, SingletonSerializeAsToken() { // Core Entities used by a Node object NodeCore @@ -43,7 +43,9 @@ class NodeSchemaService(extraSchemas: Set = emptySet(), includeNot PersistentIdentityService.PersistentIdentity::class.java, PersistentIdentityService.PersistentIdentityNames::class.java, ContractUpgradeServiceImpl.DBContractUpgrade::class.java - )) + )) { + override val migrationResource = "node-core.changelog-master" + } // Entities used by a Notary object NodeNotary @@ -54,17 +56,22 @@ class NodeSchemaService(extraSchemas: Set = emptySet(), includeNot PersistentUniquenessProvider.CommittedState::class.java, RaftUniquenessProvider.CommittedState::class.java, BFTNonValidatingNotaryService.CommittedState::class.java - )) + )) { + override val migrationResource = "node-notary.changelog-master" + } // Required schemas are those used by internal Corda services private val requiredSchemas: Map = mapOf(Pair(CommonSchemaV1, SchemaOptions()), Pair(VaultSchemaV1, SchemaOptions()), Pair(NodeInfoSchemaV1, SchemaOptions()), - Pair(NodeCoreV1, SchemaOptions())) - private val notarySchemas = if (includeNotarySchemas) mapOf(Pair(NodeNotaryV1, SchemaOptions())) else emptyMap() + Pair(NodeCoreV1, SchemaOptions())) + + if (includeNotarySchemas) mapOf(Pair(NodeNotaryV1, SchemaOptions())) else emptyMap() - override val schemaOptions: Map = requiredSchemas + notarySchemas + extraSchemas.associateBy({ it }, { SchemaOptions() }) + fun internalSchemas() = requiredSchemas.keys + extraSchemas.filter { schema -> // when mapped schemas from the finance module are present, they are considered as internal ones + schema::class.simpleName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.simpleName == "net.corda.finance.schemas.CommercialPaperSchemaV1" } + + override val schemaOptions: Map = requiredSchemas + extraSchemas.associateBy({ it }, { SchemaOptions() }) // Currently returns all schemas supported by the state, with no filtering or enrichment. override fun selectSchemas(state: ContractState): Iterable { diff --git a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt index 56897b3087..5ade98461d 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt @@ -26,6 +26,9 @@ object VaultSchema @CordaSerializable object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, version = 1, mappedTypes = listOf(VaultStates::class.java, VaultLinearStates::class.java, VaultFungibleStates::class.java, VaultTxnNote::class.java)) { + + override val migrationResource = "vault-schema.changelog-master" + @Entity @Table(name = "vault_states", indexes = [Index(name = "state_status_idx", columnList = "state_status"), Index(name = "lock_id_idx", columnList = "lock_id, state_status")]) class VaultStates( diff --git a/node/src/main/resources/migration/common.changelog-init.xml b/node/src/main/resources/migration/common.changelog-init.xml new file mode 100644 index 0000000000..d8abdbfab3 --- /dev/null +++ b/node/src/main/resources/migration/common.changelog-init.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/common.changelog-master.xml b/node/src/main/resources/migration/common.changelog-master.xml new file mode 100644 index 0000000000..4f4293dba6 --- /dev/null +++ b/node/src/main/resources/migration/common.changelog-master.xml @@ -0,0 +1,9 @@ + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-init.xml b/node/src/main/resources/migration/node-core.changelog-init.xml new file mode 100644 index 0000000000..dcf6e5b0c6 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-init.xml @@ -0,0 +1,158 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml new file mode 100644 index 0000000000..aec9787da9 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-pkey.xml b/node/src/main/resources/migration/node-core.changelog-pkey.xml new file mode 100644 index 0000000000..1e3f76ba94 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-pkey.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml b/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml new file mode 100644 index 0000000000..211c2b71ee --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml @@ -0,0 +1,25 @@ + + + + + + + + + SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 'node_checkpoints' AND COLUMN_NAME = 'checkpoint_value' + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-tx-mapping.xml b/node/src/main/resources/migration/node-core.changelog-tx-mapping.xml new file mode 100644 index 0000000000..65ff155ac6 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-tx-mapping.xml @@ -0,0 +1,17 @@ + + + + + + + + + + update node_transactions set state_machine_run_id=(select state_machine_run_id from + node_transaction_mappings where node_transactions.tx_id = node_transaction_mappings.tx_id) + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-v3.xml b/node/src/main/resources/migration/node-core.changelog-v3.xml new file mode 100644 index 0000000000..44e90a497d --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v3.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v4.xml b/node/src/main/resources/migration/node-core.changelog-v4.xml new file mode 100644 index 0000000000..cbdff14802 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v4.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v5.xml b/node/src/main/resources/migration/node-core.changelog-v5.xml new file mode 100644 index 0000000000..7ba903fdcb --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v5.xml @@ -0,0 +1,10 @@ + + + + + + + diff --git a/node/src/main/resources/migration/node-core.changelog-v8.xml b/node/src/main/resources/migration/node-core.changelog-v8.xml new file mode 100644 index 0000000000..380faf518d --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-v8.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-info.changelog-init.xml b/node/src/main/resources/migration/node-info.changelog-init.xml new file mode 100644 index 0000000000..87f56548ef --- /dev/null +++ b/node/src/main/resources/migration/node-info.changelog-init.xml @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/node-info.changelog-master.xml b/node/src/main/resources/migration/node-info.changelog-master.xml new file mode 100644 index 0000000000..4c1a49441c --- /dev/null +++ b/node/src/main/resources/migration/node-info.changelog-master.xml @@ -0,0 +1,11 @@ + + + + + + + + diff --git a/node/src/main/resources/migration/node-info.changelog-v1.xml b/node/src/main/resources/migration/node-info.changelog-v1.xml new file mode 100644 index 0000000000..b8173a2c4c --- /dev/null +++ b/node/src/main/resources/migration/node-info.changelog-v1.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-info.changelog-v2.xml b/node/src/main/resources/migration/node-info.changelog-v2.xml new file mode 100644 index 0000000000..c710807729 --- /dev/null +++ b/node/src/main/resources/migration/node-info.changelog-v2.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-notary.changelog-init.xml b/node/src/main/resources/migration/node-notary.changelog-init.xml new file mode 100644 index 0000000000..fed5c691b8 --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-init.xml @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-notary.changelog-master.xml b/node/src/main/resources/migration/node-notary.changelog-master.xml new file mode 100644 index 0000000000..9f169670fc --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-master.xml @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/node/src/main/resources/migration/node-notary.changelog-pkey.xml b/node/src/main/resources/migration/node-notary.changelog-pkey.xml new file mode 100644 index 0000000000..64a99cc978 --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-pkey.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/node-notary.changelog-v1.xml b/node/src/main/resources/migration/node-notary.changelog-v1.xml new file mode 100644 index 0000000000..0856d543b4 --- /dev/null +++ b/node/src/main/resources/migration/node-notary.changelog-v1.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/vault-schema.changelog-init.xml b/node/src/main/resources/migration/vault-schema.changelog-init.xml new file mode 100644 index 0000000000..388e5b030d --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-init.xml @@ -0,0 +1,138 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/vault-schema.changelog-master.xml b/node/src/main/resources/migration/vault-schema.changelog-master.xml new file mode 100644 index 0000000000..a9fbc181c5 --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-master.xml @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/node/src/main/resources/migration/vault-schema.changelog-pkey.xml b/node/src/main/resources/migration/vault-schema.changelog-pkey.xml new file mode 100644 index 0000000000..b8a76bc392 --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-pkey.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/vault-schema.changelog-v3.xml b/node/src/main/resources/migration/vault-schema.changelog-v3.xml new file mode 100644 index 0000000000..65a2707423 --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-v3.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + diff --git a/node/src/main/resources/migration/vault-schema.changelog-v4.xml b/node/src/main/resources/migration/vault-schema.changelog-v4.xml new file mode 100644 index 0000000000..b3f239f0cf --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-v4.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/node/src/main/resources/migration/vault-schema.changelog-v5.xml b/node/src/main/resources/migration/vault-schema.changelog-v5.xml new file mode 100644 index 0000000000..86a31530c8 --- /dev/null +++ b/node/src/main/resources/migration/vault-schema.changelog-v5.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java index 296412bf53..6ccb168d58 100644 --- a/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java +++ b/node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java @@ -11,7 +11,6 @@ import net.corda.core.identity.Party; import net.corda.core.messaging.DataFeed; import net.corda.core.node.services.IdentityService; import net.corda.core.node.services.Vault; -import net.corda.core.node.services.VaultQueryException; import net.corda.core.node.services.VaultService; import net.corda.core.node.services.vault.*; import net.corda.core.node.services.vault.QueryCriteria.LinearStateQueryCriteria; @@ -33,7 +32,6 @@ import org.junit.After; import org.junit.Before; import org.junit.Rule; import org.junit.Test; -import rx.Observable; import java.util.*; import java.util.stream.Collectors; diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt index a1018d5526..9d3cb8a5f8 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt @@ -6,8 +6,8 @@ import net.corda.core.node.services.vault.* import net.corda.core.node.services.vault.QueryCriteria.* import net.corda.finance.* import net.corda.finance.contracts.asset.Cash -import net.corda.finance.schemas.CashSchemaV1 import net.corda.finance.schemas.SampleCashSchemaV3 +import net.corda.finance.schemas.CashSchemaV1 import net.corda.testing.core.* import net.corda.testing.internal.vault.DummyLinearStateSchemaV1 import org.assertj.core.api.Assertions.assertThatThrownBy @@ -51,4 +51,4 @@ class VaultQueryExceptionsTests : VaultQueryParties by rule { }.isInstanceOf(VaultQueryException::class.java).hasMessageContaining("Please register the entity") } } -} \ No newline at end of file +} diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt index cfd6eeee6a..6e085b6c70 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt @@ -111,7 +111,7 @@ open class MockServices private constructor( val cordappLoader = cordappLoaderForPackages(cordappPackages) val dataSourceProps = makeTestDataSourceProperties() val schemaService = NodeSchemaService(cordappLoader.cordappSchemas) - val database = configureDatabase(dataSourceProps, DatabaseConfig(), identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, schemaService) + val database = configureDatabase(dataSourceProps, DatabaseConfig(), identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, schemaService, schemaService.internalSchemas()) val mockService = database.transaction { object : MockServices(cordappLoader, identityService, networkParameters, initialIdentity, moreKeys) { override val vaultService: VaultService = makeVaultService(schemaService, database) From c4f33ef533fc5f91d62de8644b00e3bb1b661de5 Mon Sep 17 00:00:00 2001 From: szymonsztuka Date: Thu, 23 Aug 2018 11:20:24 +0100 Subject: [PATCH 2/4] Rename column HOST to HOST_NAME in table NODE_INFO_HOSTS - required by ENT-2253 --- .../net/corda/node/internal/schemas/NodeInfoSchema.kt | 1 + .../resources/migration/node-info.changelog-master.xml | 1 + .../main/resources/migration/node-info.changelog-v3.xml | 9 +++++++++ 3 files changed, 11 insertions(+) create mode 100644 node/src/main/resources/migration/node-info.changelog-v3.xml diff --git a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt index 13c57cf8da..024fcf74ca 100644 --- a/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt +++ b/node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt @@ -72,6 +72,7 @@ object NodeInfoSchemaV1 : MappedSchema( @GeneratedValue @Column(name = "hosts_id", nullable = false) var id: Int, + @Column(name = "host_name") val host: String? = null, val port: Int? = null ) { diff --git a/node/src/main/resources/migration/node-info.changelog-master.xml b/node/src/main/resources/migration/node-info.changelog-master.xml index 4c1a49441c..b5f43abc1c 100644 --- a/node/src/main/resources/migration/node-info.changelog-master.xml +++ b/node/src/main/resources/migration/node-info.changelog-master.xml @@ -7,5 +7,6 @@ + diff --git a/node/src/main/resources/migration/node-info.changelog-v3.xml b/node/src/main/resources/migration/node-info.changelog-v3.xml new file mode 100644 index 0000000000..20f24bd33c --- /dev/null +++ b/node/src/main/resources/migration/node-info.changelog-v3.xml @@ -0,0 +1,9 @@ + + + + + + + \ No newline at end of file From c7f666102f3fbdd804af2029aed648b20a78fdd9 Mon Sep 17 00:00:00 2001 From: szymonsztuka Date: Fri, 24 Aug 2018 12:27:47 +0100 Subject: [PATCH 3/4] Fixes after merge remote-tracking branch 'remotes/open/master' into szymonsztuka/os-merge-20180824 --- .idea/compiler.xml | 8 +++++--- .../net/corda/core/internal/InternalUtils.kt | 5 ----- .../corda/flowworker/FlowWorkerServiceHub.kt | 6 +++++- node-api/build.gradle | 5 ----- .../corda/nodeapi/internal/MigrationHelpers.kt | 2 +- .../internal/persistence/SchemaMigration.kt | 9 ++++----- .../net/corda/node/internal/AbstractNode.kt | 18 +++++++----------- .../node/services/schema/NodeSchemaService.kt | 11 +++++++---- .../persistence/SchemaMigrationTest.kt | 2 +- .../net/corda/testing/node/MockServices.kt | 1 - .../com/r3/corda/dbmigration/Launcher.kt | 4 ++-- 11 files changed, 32 insertions(+), 39 deletions(-) diff --git a/.idea/compiler.xml b/.idea/compiler.xml index ef69357c69..e0cf3b031c 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -43,14 +43,16 @@ + + + + - - @@ -326,4 +328,4 @@ - + \ No newline at end of file diff --git a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt index 4c35473fe7..fee301b3f7 100644 --- a/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt +++ b/core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt @@ -515,8 +515,3 @@ fun SerializedBytes.checkPayloadIs(type: Class): Untrustworthy return type.castIfPossible(payloadData)?.let { UntrustworthyData(it) } ?: throw IllegalArgumentException("We were expecting a ${type.name} but we instead got a ${payloadData.javaClass.name} ($payloadData)") } - -/** - * Extension method to make this method visible to nodeapi module. - */ -fun MappedSchema.getMigrationResource(): String? = this.internalGetMigrationResource() \ No newline at end of file diff --git a/experimental/flow-worker/src/main/kotlin/net/corda/flowworker/FlowWorkerServiceHub.kt b/experimental/flow-worker/src/main/kotlin/net/corda/flowworker/FlowWorkerServiceHub.kt index fc6be347ec..e10ed86091 100644 --- a/experimental/flow-worker/src/main/kotlin/net/corda/flowworker/FlowWorkerServiceHub.kt +++ b/experimental/flow-worker/src/main/kotlin/net/corda/flowworker/FlowWorkerServiceHub.kt @@ -54,6 +54,7 @@ import net.corda.node.utilities.AffinityExecutor import net.corda.nodeapi.internal.DEV_ROOT_CA import net.corda.nodeapi.internal.crypto.X509Utilities import net.corda.nodeapi.internal.persistence.CordaPersistence +import net.corda.nodeapi.internal.persistence.isH2Database import net.corda.serialization.internal.* import org.apache.activemq.artemis.utils.ReusableLatch import rx.schedulers.Schedulers @@ -242,7 +243,10 @@ class FlowWorkerServiceHub(override val configuration: NodeConfiguration, overri servicesForResolution.start(networkParameters) persistentNetworkMapCache.start(networkParameters.notaries) - database.startHikariPool(configuration.dataSourceProperties, configuration.database, schemaService) + val isH2Database = isH2Database(configuration.dataSourceProperties.getProperty("dataSource.url", "")) + val schemas = if (isH2Database) schemaService.internalSchemas() else schemaService.schemaOptions.keys + + database.startHikariPool(configuration.dataSourceProperties, configuration.database, schemas) identityService.start(trustRoot, listOf(myInfo.legalIdentitiesAndCerts.first().certificate, nodeCa)) database.transaction { diff --git a/node-api/build.gradle b/node-api/build.gradle index 01452cb19c..2aa8f97de4 100644 --- a/node-api/build.gradle +++ b/node-api/build.gradle @@ -63,11 +63,6 @@ dependencies { // For caches rather than guava compile "com.github.ben-manes.caffeine:caffeine:$caffeine_version" - // For db migration - compile "org.liquibase:liquibase-core:$liquibase_version" - compile "com.fasterxml.jackson.core:jackson-databind:$jackson_version" - runtime 'com.mattbertolini:liquibase-slf4j:2.0.0' - // Unit testing helpers. testCompile "junit:junit:$junit_version" testCompile "org.assertj:assertj-core:$assertj_version" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt index b098574556..96686008fd 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/MigrationHelpers.kt @@ -34,7 +34,7 @@ object MigrationHelpers { } // SchemaName will be transformed from camel case to lower_hyphen then add ".changelog-master" - private fun migrationResourceNameForSchema(schema: MappedSchema): String { + fun migrationResourceNameForSchema(schema: MappedSchema): String { val name: String = schema::class.simpleName!! val fileName = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_HYPHEN, name) return "$MIGRATION_PREFIX/$fileName.$CHANGELOG_NAME" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt index 0d15b28268..3364c7a468 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/SchemaMigration.kt @@ -42,13 +42,13 @@ class SchemaMigration( * Main entry point to the schema migration. * Called during node startup. */ - fun nodeStartup(existingCheckpoints: Boolean) { + fun nodeStartup(existingCheckpoints: Boolean, isH2Database: Boolean) { when { - databaseConfig.initialiseSchema -> { - //TODO if it's h2 only + databaseConfig.initialiseSchema && isH2Database -> { migrateOlderDatabaseToUseLiquibase(existingCheckpoints) runMigration(existingCheckpoints) } + databaseConfig.initialiseSchema -> runMigration(existingCheckpoints) else -> checkState() } } @@ -66,7 +66,7 @@ class SchemaMigration( /** * Ensures that the database is up to date with the latest migration changes. */ - private fun checkState() = doRunMigration(run = false, outputWriter = null, check = true) + fun checkState() = doRunMigration(run = false, outputWriter = null, check = true) /** * Can be used from an external tool to release the lock in case something went terribly wrong. @@ -138,7 +138,6 @@ class SchemaMigration( check && !run && unRunChanges.isNotEmpty() -> throw OutstandingDatabaseChangesException(unRunChanges.size) check && !run -> {} // Do nothing will be interpreted as "check succeeded" (outputWriter != null) && !check && !run -> liquibase.update(Contexts(), outputWriter) - (outputWriter != null) && !check && !run -> liquibase.update(Contexts(), outputWriter) else -> throw IllegalStateException("Invalid usage.") } } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index f0de64bf0d..a7fe619f9a 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -149,8 +149,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration, configuration.database, identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, - schemaService, - configuration.dataSourceProperties + schemaService ) init { // TODO Break cyclic dependency @@ -1049,11 +1048,10 @@ fun configureDatabase(hikariProperties: Properties, databaseConfig: DatabaseConfig, wellKnownPartyFromX500Name: (CordaX500Name) -> Party?, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, - schemaService: NodeSchemaService = NodeSchemaService()): CordaPersistence { - + schemaService: SchemaService = NodeSchemaService()): CordaPersistence { val isH2Database = isH2Database(hikariProperties.getProperty("dataSource.url", "")) - val schemas = if (isH2Database) schemaService.internalSchemas() else schemaService.schemaOptions.keys - createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService) + val schemas = if (isH2Database) NodeSchemaService().internalSchemas() else NodeSchemaService().schemaOptions.keys + return createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService) .apply { startHikariPool(hikariProperties, databaseConfig, schemas) } } @@ -1061,16 +1059,14 @@ fun configureDatabase(hikariProperties: Properties, fun createCordaPersistence(databaseConfig: DatabaseConfig, wellKnownPartyFromX500Name: (CordaX500Name) -> Party?, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, - schemaService: SchemaService, - hikariProperties: Properties): CordaPersistence { + schemaService: SchemaService): CordaPersistence { // Register the AbstractPartyDescriptor so Hibernate doesn't warn when encountering AbstractParty. Unfortunately // Hibernate warns about not being able to find a descriptor if we don't provide one, but won't use it by default // so we end up providing both descriptor and converter. We should re-examine this in later versions to see if // either Hibernate can be convinced to stop warning, use the descriptor by default, or something else. JavaTypeDescriptorRegistry.INSTANCE.addDescriptor(AbstractPartyDescriptor(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) val attributeConverters = listOf(AbstractPartyToX500NameAsStringConverter(wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous)) - val jdbcUrl = hikariProperties.getProperty("dataSource.url", "") - return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, jdbcUrl, attributeConverters) + return CordaPersistence(databaseConfig, schemaService.schemaOptions.keys, attributeConverters) } fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfig: DatabaseConfig, schemas: Set) { @@ -1078,7 +1074,7 @@ fun CordaPersistence.startHikariPool(hikariProperties: Properties, databaseConfi val dataSource = DataSourceFactory.createDataSource(hikariProperties) val jdbcUrl = hikariProperties.getProperty("dataSource.url", "") val schemaMigration = SchemaMigration(schemas, dataSource, databaseConfig) - schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) + schemaMigration.nodeStartup(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }, isH2Database(jdbcUrl)) start(dataSource, jdbcUrl) } catch (ex: Exception) { when { diff --git a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt index 2b60955184..5ba3223ea8 100644 --- a/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt +++ b/node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt @@ -75,10 +75,13 @@ class NodeSchemaService(private val extraSchemas: Set = emptySet() // Required schemas are those used by internal Corda services private val requiredSchemas: Map = mapOf(Pair(CommonSchemaV1, SchemaOptions()), - Pair(VaultSchemaV1, SchemaOptions()), - Pair(NodeInfoSchemaV1, SchemaOptions()), - Pair(NodeCoreV1, SchemaOptions())) + - if (includeNotarySchemas) mapOf(Pair(NodeNotaryV1, SchemaOptions())) else emptyMap() + Pair(VaultSchemaV1, SchemaOptions()), + Pair(NodeInfoSchemaV1, SchemaOptions()), + Pair(NodeCoreV1, SchemaOptions())) + + if (includeNotarySchemas) mapOf(Pair(NodeNotaryV1, SchemaOptions())) else emptyMap() + + fun internalSchemas() = requiredSchemas.keys + extraSchemas.filter { schema -> // when mapped schemas from the finance module are present, they are considered as internal ones + schema::class.simpleName == "net.corda.finance.schemas.CashSchemaV1" || schema::class.simpleName == "net.corda.finance.schemas.CommercialPaperSchemaV1" } override val schemaOptions: Map = requiredSchemas + extraSchemas.associateBy({ it }, { SchemaOptions() }) diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt b/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt index 35ecb94c26..ec0289ef68 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt @@ -60,7 +60,7 @@ class SchemaMigrationTest { val dataSourceProps = MockServices.makeTestDataSourceProperties() //run the migration on the database - val migration = SchemaMigration(schemaService.schemaOptions.keys, HikariDataSource(HikariConfig(dataSourceProps)), true, DatabaseConfig()) + val migration = SchemaMigration(schemaService.schemaOptions.keys, HikariDataSource(HikariConfig(dataSourceProps)), DatabaseConfig()) migration.runMigration(false) //start the node with "runMigration = false" and check that it started correctly diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt index 59aaf19db4..8c1f5dd7e5 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt @@ -114,7 +114,6 @@ open class MockServices private constructor( val cordappLoader = cordappLoaderForPackages(cordappPackages) val dataSourceProps = makeInternalTestDataSourceProperties(initialIdentity.name.organisation, SecureHash.randomSHA256().toString()) val schemaService = NodeSchemaService(cordappLoader.cordappSchemas) - //TODO different schemas based on h2 or not val database = configureDatabase(dataSourceProps, makeTestDatabaseProperties(initialIdentity.name.organisation), identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, schemaService) val mockService = database.transaction { object : MockServices(cordappLoader, identityService, networkParameters, initialIdentity, moreKeys) { diff --git a/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt b/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt index c282dc289b..4cd857126d 100644 --- a/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt +++ b/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt @@ -169,12 +169,12 @@ private fun handleCommand(options: OptionSet, baseDirectory: Path, configFile: P val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle) fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource -> - withMigration(SchemaMigration(schemas, dataSource, true, config.database, classLoader), dataSource) + withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource) } when { options.has(RELEASE_LOCK) -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve().parseAs(Configuration::class), baseDirectory, classLoader) { - SchemaMigration(emptySet(), it, true, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock() + SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock() } options.has(DRY_RUN) -> { val writer = getMigrationOutput(baseDirectory, options) From 33e8e05a8dd5411aa46a55a47e7aebc1f5aa5e27 Mon Sep 17 00:00:00 2001 From: szymonsztuka Date: Fri, 24 Aug 2018 13:24:43 +0100 Subject: [PATCH 4/4] Fixes after merge remote-tracking branch 'remotes/open/master' into szymonsztuka/os-merge-20180824 --- .../net/corda/node/internal/AbstractNode.kt | 2 +- .../persistence/SchemaMigrationTest.kt | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index a7fe619f9a..90f8907912 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -1050,7 +1050,7 @@ fun configureDatabase(hikariProperties: Properties, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, schemaService: SchemaService = NodeSchemaService()): CordaPersistence { val isH2Database = isH2Database(hikariProperties.getProperty("dataSource.url", "")) - val schemas = if (isH2Database) NodeSchemaService().internalSchemas() else NodeSchemaService().schemaOptions.keys + val schemas = if (isH2Database) NodeSchemaService().internalSchemas() else schemaService.schemaOptions.keys return createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService) .apply { startHikariPool(hikariProperties, databaseConfig, schemas) } diff --git a/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt b/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt index ec0289ef68..e03a9e01b8 100644 --- a/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt +++ b/node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt @@ -16,7 +16,8 @@ import net.corda.core.contracts.UniqueIdentifier import net.corda.core.identity.AbstractParty import net.corda.core.schemas.CommonSchemaV1 import net.corda.core.schemas.MappedSchema -import net.corda.node.internal.configureDatabase +import net.corda.node.internal.createCordaPersistence +import net.corda.node.internal.startHikariPool import net.corda.node.services.schema.NodeSchemaService import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig @@ -32,9 +33,16 @@ import javax.persistence.* import java.net.URLClassLoader import java.nio.file.Files import java.nio.file.Path +import java.util.* class SchemaMigrationTest { + private fun configureDatabase(hikariProperties: Properties, + databaseConfig: DatabaseConfig, + schemaService: NodeSchemaService = NodeSchemaService()): CordaPersistence = + createCordaPersistence(databaseConfig, { null }, { null }, schemaService) + .apply { startHikariPool(hikariProperties, databaseConfig, schemaService.schemaOptions.keys) } + @Test fun `Ensure that runMigration is disabled by default`() { assertThat(DatabaseConfig().runMigration).isFalse() @@ -43,14 +51,14 @@ class SchemaMigrationTest { @Test fun `Migration is run when runMigration is disabled, and database is H2`() { val dataSourceProps = MockServices.makeTestDataSourceProperties() - val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = false), { null }, { null }) + val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = false)) checkMigrationRun(db) } @Test fun `Migration is run when runMigration is enabled`() { val dataSourceProps = MockServices.makeTestDataSourceProperties() - val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = true), { null }, { null }) + val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = true)) checkMigrationRun(db) } @@ -64,7 +72,7 @@ class SchemaMigrationTest { migration.runMigration(false) //start the node with "runMigration = false" and check that it started correctly - val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = false), { null }, { null }, schemaService) + val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = false), schemaService) checkMigrationRun(db) } @@ -78,7 +86,7 @@ class SchemaMigrationTest { addToClassPath(tmpFolder) // run the migrations for DummyTestSchemaV1, which should pick up the migration file - val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = true), { null }, { null }, NodeSchemaService(extraSchemas = setOf(DummyTestSchemaV1))) + val db = configureDatabase(dataSourceProps, DatabaseConfig(runMigration = true), NodeSchemaService(extraSchemas = setOf(DummyTestSchemaV1))) // check that the file was picked up val nrOfChangesOnDiscoveredFile = db.dataSource.connection.use {