ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2025-03-15 16:46:12 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ENT-2219 add option to automatically package migration scripts into a migratio… (#1508)

Browse Source 
* add option to automatically package migration scripts into a migration cordapp

* fix compile error

* add explict "OK" from user to create jar
add warning about possible data corruption
refactor check for --jar to be easier to read.
This commit is contained in:
Stefano Franz 2018-10-26 15:30:16 +01:00 committed by GitHub
parent b1a787e649
commit 21afd256d5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 104 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
node/src/main/kotlin/net/corda/node/services/persistence/MigrationExporter.kt
View File

@ -34,13 +34,13 @@ class MigrationExporter(val parent: Path, val datasourceProperties: Properties,
const val CORDA_USER = "R3.Corda.Generated"
}
fun generateMigrationForCorDapp(schemaName: String): Path {
fun generateMigrationForCorDapp(schemaName: String): Pair<Class<*>, Path> {
val schemaClass = cordappClassLoader.loadClass(schemaName)
val schemaObject = schemaClass.kotlin.objectOrNewInstance() as MappedSchema
return generateMigrationForCorDapp(schemaObject)
}
fun generateMigrationForCorDapp(mappedSchema: MappedSchema): Path {
fun generateMigrationForCorDapp(mappedSchema: MappedSchema): Pair<Class<*>, Path> {
//create hibernate metadata for MappedSchema
val metadata = createHibernateMetadataForSchema(mappedSchema)
@ -64,7 +64,7 @@ class MigrationExporter(val parent: Path, val datasourceProperties: Properties,
setOutputFile(outputFile.absolutePath)
execute(EnumSet.of(TargetType.SCRIPT), SchemaExport.Action.CREATE, metadata)
}
return outputFile.toPath()
return mappedSchema::class.java to outputFile.toPath()
}
private fun createHibernateMetadataForSchema(mappedSchema: MappedSchema): Metadata {

2
node/src/test/kotlin/net/corda/node/services/persistence/SchemaMigrationTest.kt
View File

@ -73,7 +73,7 @@ class SchemaMigrationTest {
// create a migration file for the DummyTestSchemaV1 and add it to the classpath
val tmpFolder = Files.createTempDirectory("test")
val fileName = MigrationExporter(tmpFolder, dataSourceProps, Thread.currentThread().contextClassLoader, HikariDataSource(HikariConfig(dataSourceProps))).generateMigrationForCorDapp(DummyTestSchemaV1).fileName
val fileName = MigrationExporter(tmpFolder, dataSourceProps, Thread.currentThread().contextClassLoader, HikariDataSource(HikariConfig(dataSourceProps))).generateMigrationForCorDapp(DummyTestSchemaV1).second.fileName
addToClassPath(tmpFolder)
// run the migrations for DummyTestSchemaV1, which should pick up the migration file

121
tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt
View File

@ -7,6 +7,7 @@ import com.typesafe.config.ConfigParseOptions
import net.corda.cliutils.CordaCliWrapper
import net.corda.cliutils.ExitCodes
import net.corda.cliutils.start
import net.corda.core.internal.copyTo
import net.corda.core.internal.div
import net.corda.core.internal.exists
import net.corda.core.schemas.MappedSchema
@ -15,7 +16,6 @@ import net.corda.node.internal.cordapp.JarScanningCordappLoader
import net.corda.node.services.config.ConfigHelper
import net.corda.node.services.config.NodeConfigurationImpl
import net.corda.node.services.config.configOf
import net.corda.node.services.config.parseAsNodeConfiguration
import net.corda.node.services.persistence.DBCheckpointStorage
import net.corda.node.services.persistence.MigrationExporter
import net.corda.node.services.schema.NodeSchemaService
@ -28,15 +28,14 @@ import net.corda.nodeapi.internal.persistence.SchemaMigration
import org.slf4j.LoggerFactory
import picocli.CommandLine.Mixin
import picocli.CommandLine.Option
import java.io.File
import java.io.FileWriter
import java.io.PrintWriter
import java.io.Writer
import java.io.*
import java.net.URLClassLoader
import java.nio.file.Path
import java.nio.file.Paths
import java.text.SimpleDateFormat
import java.util.*
import java.util.jar.JarOutputStream
import java.util.zip.ZipEntry
import javax.sql.DataSource
//command line arguments
@ -44,13 +43,14 @@ const val DOORMAN_JAR_PATH = "doorman-jar-path"
const val EXECUTE_MIGRATION = "execute-migration"
const val DRY_RUN = "dry-run"
const val CREATE_MIGRATION_CORDAPP = "create-migration-sql-for-cordapp"
const val JAR_OUTPUT = "jar"
const val RELEASE_LOCK = "release-lock"
// output type
const val CONSOLE = "CONSOLE"
// initialise loggers lazily as some configuration is changed on startup and if loggers are already initialised it will be ignored
private val migrationLogger by lazy {LoggerFactory.getLogger("migration.tool") }
private val migrationLogger by lazy { LoggerFactory.getLogger("migration.tool") }
private val errorLogger by lazy { LoggerFactory.getLogger("errors") }
private enum class Mode {
@ -106,7 +106,14 @@ private class DbManagementToolOptions {
)
var createMigrationSqlForCordappPath: String? = null
val createMigrationSqlForCordapp : Boolean get() = createMigrationSqlForCordappPath != null
@Option(
names = ["--$JAR_OUTPUT"],
arity = "0..0",
description = ["Place generated migration scripts into a jar"]
)
var jarOutput: Boolean = false
val createMigrationSqlForCordapp: Boolean get() = createMigrationSqlForCordappPath != null
@Option(
names = ["--$RELEASE_LOCK"],
@ -131,13 +138,18 @@ private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda
if (cmdLineOptions.executeMigration) selectedOptions.add(EXECUTE_MIGRATION)
if (cmdLineOptions.createMigrationSqlForCordapp) selectedOptions.add(CREATE_MIGRATION_CORDAPP)
if (cmdLineOptions.releaseLock) selectedOptions.add(RELEASE_LOCK)
require(selectedOptions.count() != 0) {"You must call database-manager with a command option. See --help for further info."}
require(selectedOptions.count() == 1) {"You cannot call more than one of: ${selectedOptions.joinToString(", ")}. See --help for further info."}
require(selectedOptions.count() != 0) { "You must call database-manager with a command option. See --help for further info." }
require(selectedOptions.count() == 1) { "You cannot call more than one of: ${selectedOptions.joinToString(", ")}. See --help for further info." }
if (cmdLineOptions.jarOutput){
require(cmdLineOptions.createMigrationSqlForCordapp) { "You must specify --$CREATE_MIGRATION_CORDAPP to use the --$JAR_OUTPUT option" }
}
}
override fun runProgram(): Int {
checkOnlyOneCommandSelected()
require(cmdLineOptions.baseDirectory != null) {"You must specify a base directory"}
require(cmdLineOptions.baseDirectory != null) { "You must specify a base directory" }
fun baseDirectory() = cmdLineOptions.baseDirectory?.toAbsolutePath()?.normalize()!!
fun configFile(defaultCfgName: String) = baseDirectory() / (cmdLineOptions.configFile ?: defaultCfgName)
when {
@ -150,12 +162,12 @@ private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda
if (!config.exists()) {
error("Not a valid node folder. Could not find the config file: '$config'.")
}
val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAs<NodeConfigurationImpl>(UnknownConfigKeysPolicy.IGNORE::handle)
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps"))
val cordappsFolder = baseDirectory / "cordapps"
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, cordappsFolder))
val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas)
handleCommand(baseDirectory, config, cmdLineOptions.mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys)
handleCommand(baseDirectory, config, cmdLineOptions.mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys, cmdLineOptions.jarOutput)
}
cmdLineOptions.mode == Mode.DOORMAN -> {
if (cmdLineOptions.doormanJarPath != null) {
@ -168,14 +180,14 @@ private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda
val doormanClassloader = classLoaderFromJar(fatJarPath)
val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1"
val schema = loadMappedSchema(doormanSchema, doormanClassloader)
handleCommand(baseDirectory(), configFile("network-management.conf"), cmdLineOptions.mode, doormanClassloader, setOf(schema))
handleCommand(baseDirectory(), configFile("network-management.conf"), cmdLineOptions.mode, doormanClassloader, setOf(schema), false)
}
}
migrationLogger.info("Done")
return ExitCodes.SUCCESS
}
private fun handleCommand(baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>) {
private fun handleCommand(baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>, jarOutput: Boolean) {
val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let {
if (mode == Mode.NODE) {
it.withFallback(configOf("baseDirectory" to baseDirectory.toString()))
@ -207,29 +219,96 @@ private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda
runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) }
}
cmdLineOptions.createMigrationSqlForCordapp && (mode == Mode.NODE) -> {
fun generateMigrationFileForSchema(schemaClass: String) {
// do we really want to creating captured functions here.
fun generateMigrationFileForSchema(schemaClass: String): Pair<Class<*>, Path>? {
migrationLogger.info("Creating database migration files for schema: $schemaClass into ${(baseDirectory / "migration").toString().trim()}")
var liquiBaseOutput: Pair<Class<*>, Path>? = null
try {
runWithDataSource(config, baseDirectory, classLoader) {
MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
liquiBaseOutput = MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
}
} catch (e: Exception) {
wrappedError("Could not generate migration for $schemaClass: ${e.message}", e)
}
return liquiBaseOutput
}
val outputSchemaMigrations = mutableListOf<Pair<Class<*>, Path>>()
if (cmdLineOptions.createMigrationSqlForCordappPath != "") {
generateMigrationFileForSchema(cmdLineOptions.createMigrationSqlForCordappPath!!)
generateMigrationFileForSchema(cmdLineOptions.createMigrationSqlForCordappPath!!)?.let { outputSchemaMigrations.add(it) }
} else {
schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach {
outputSchemaMigrations.addAll(schemas.filter { true || MigrationHelpers.getMigrationResource(it, classLoader) == null }.mapNotNull {
generateMigrationFileForSchema(it.javaClass.name)
})
}
if (jarOutput) {
val groupedBySourceJar = outputSchemaMigrations.map { (klazz, path) ->
//get the source jar for this particular schema
klazz.protectionDomain.codeSource to path
}.filter {
//filter all entries without a code source
it.first != null
}.map {
//convert codesource to a File
File(it.first.location.toURI()) to it.second
}.groupBy {
//group by codesource File
it.first
}.map {
// convert File into a filename for the codesource
// if run from within an IDE, there is possibility of some schemas to be
// loaded from a build folder
// so we must handle this case
val fileName = if (it.key.name.endsWith("jar")) {
it.key.path
} else {
"unknown-cordapp.jar"
}
fileName to it.value.map { it.second }
}.toMap()
groupedBySourceJar.entries.forEach { (_, migrationFiles) ->
migrationFiles.forEach {
//use System.out directly instead of logger as we MUST output to screen due to user input requirement
System.out.println("#####${it.fileName}#####")
it.copyTo(System.out)
System.out.println("Is this output as expected? [y/N]")
validateUserResponse()
}
}
System.out.println("There is potential for data corruption.")
System.out.println("Are you sure that the migration scripts are acceptable? [y/N]")
validateUserResponse()
groupedBySourceJar.entries.forEach { (jar, migrationFiles) ->
val sourceJarFile = File(jar)
val migrationJarFile = File(sourceJarFile.parent, "migration-${sourceJarFile.name}")
JarOutputStream(FileOutputStream(migrationJarFile)).use { jos ->
migrationLogger.info("Creating migration cordapp at: ${migrationJarFile.absolutePath}")
migrationFiles.map { ZipEntry("migration/${it.fileName}") }.forEach {
jos.putNextEntry(it)
}
}
}
}
}
else -> error("Please specify a correct command")
}
}
private fun validateUserResponse() {
val userInput = Scanner(System.`in`).nextLine().toLowerCase()
if (userInput != "y" && userInput != "yes") {
migrationLogger.warn("quitting due to user input")
System.exit(22)
}
}
private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray())
private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema
@ -281,5 +360,5 @@ private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda
}
}
class ConfigurationException(message: String): Exception(message)
class WrappedConfigurationException(message: String, val innerException: Exception): Exception(message)
class ConfigurationException(message: String) : Exception(message)
class WrappedConfigurationException(message: String, val innerException: Exception) : Exception(message)