Make database manager use picocli base class withstandardised options

This commit is contained in:
Anthony Keenan 2018-09-25 16:14:09 +01:00
parent 02ae92fc87
commit b3307eaecb
3 changed files with 216 additions and 171 deletions

View File

@ -14,8 +14,7 @@ apply plugin: 'com.jfrog.artifactory'
dependencies { dependencies {
compile project(':node') compile project(':node')
// JOpt: for command line flags. compile project(':tools:cliutils')
compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
} }
import de.sebastianboegl.gradle.plugins.shadow.transformers.Log4j2PluginsFileTransformer import de.sebastianboegl.gradle.plugins.shadow.transformers.Log4j2PluginsFileTransformer

View File

@ -4,11 +4,9 @@ package com.r3.corda.dbmigration
import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigParseOptions import com.typesafe.config.ConfigParseOptions
import joptsimple.OptionException import net.corda.cliutils.CordaCliWrapper
import joptsimple.OptionParser import net.corda.cliutils.ExitCodes
import joptsimple.OptionSet import net.corda.cliutils.start
import joptsimple.util.EnumConverter
import net.corda.nodeapi.internal.MigrationHelpers
import net.corda.core.internal.div import net.corda.core.internal.div
import net.corda.core.internal.exists import net.corda.core.internal.exists
import net.corda.core.schemas.MappedSchema import net.corda.core.schemas.MappedSchema
@ -20,12 +18,15 @@ import net.corda.node.services.config.parseAsNodeConfiguration
import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.DBCheckpointStorage
import net.corda.node.services.persistence.MigrationExporter import net.corda.node.services.persistence.MigrationExporter
import net.corda.node.services.schema.NodeSchemaService import net.corda.node.services.schema.NodeSchemaService
import net.corda.nodeapi.internal.MigrationHelpers
import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
import net.corda.nodeapi.internal.config.parseAs import net.corda.nodeapi.internal.config.parseAs
import net.corda.nodeapi.internal.persistence.CheckpointsException import net.corda.nodeapi.internal.persistence.CheckpointsException
import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.DatabaseConfig
import net.corda.nodeapi.internal.persistence.SchemaMigration import net.corda.nodeapi.internal.persistence.SchemaMigration
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
import picocli.CommandLine.Mixin
import picocli.CommandLine.Option
import java.io.File import java.io.File
import java.io.FileWriter import java.io.FileWriter
import java.io.PrintWriter import java.io.PrintWriter
@ -38,12 +39,8 @@ import java.util.*
import javax.sql.DataSource import javax.sql.DataSource
//command line arguments //command line arguments
const val HELP = "help"
const val MODE = "mode"
const val BASE_DIRECTORY = "base-directory"
const val CONFIG = "config-file"
const val DOORMAN_JAR_PATH = "doorman-jar-path" const val DOORMAN_JAR_PATH = "doorman-jar-path"
const val RUN_MIGRATION = "execute-migration" const val EXECUTE_MIGRATION = "execute-migration"
const val DRY_RUN = "dry-run" const val DRY_RUN = "dry-run"
const val CREATE_MIGRATION_CORDAPP = "create-migration-sql-for-cordapp" const val CREATE_MIGRATION_CORDAPP = "create-migration-sql-for-cordapp"
const val RELEASE_LOCK = "release-lock" const val RELEASE_LOCK = "release-lock"
@ -51,195 +48,237 @@ const val RELEASE_LOCK = "release-lock"
// output type // output type
const val CONSOLE = "CONSOLE" const val CONSOLE = "CONSOLE"
private val migrationLogger = LoggerFactory.getLogger("migration.tool") // initialise loggers lazily as some configuration is changed on startup and if loggers are already initialised it will be ignored
private val errorLogger = LoggerFactory.getLogger("errors") private val migrationLogger by lazy {LoggerFactory.getLogger("migration.tool") }
private val errorLogger by lazy { LoggerFactory.getLogger("errors") }
private enum class Mode { private enum class Mode {
NODE, DOORMAN NODE, DOORMAN
} }
private fun initOptionParser(): OptionParser = OptionParser().apply { private class DbManagementToolOptions {
accepts(MODE, "Either 'NODE' or 'DOORMAN'. By default 'NODE'") @Option(
.withOptionalArg() names = ["--mode"],
.withValuesConvertedBy(object : EnumConverter<Mode>(Mode::class.java) {}) description = ["The operating mode."]
.defaultsTo(Mode.NODE) )
var mode: Mode = Mode.NODE
accepts(BASE_DIRECTORY, "The node or doorman directory") @Option(
.withRequiredArg().required() names = ["-b", "--base-directory"],
description = ["The node or doorman directory."]
)
var baseDirectory: Path? = null
accepts(CONFIG, "The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman.") @Option(
.withOptionalArg() names = ["-f", "--config-file"],
description = ["The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman."]
)
var configFile: String? = null
accepts(DOORMAN_JAR_PATH, "The path to the doorman JAR") @Option(
.withOptionalArg() names = ["--$DOORMAN_JAR_PATH"],
description = ["The path to the doorman JAR."]
)
var doormanJarPath: Path? = null
val runMig = accepts(RUN_MIGRATION, @Option(
"This option will run the db migration on the configured database. This is the only command that will actually write to the database.") names = ["--$EXECUTE_MIGRATION"],
description = ["This option will run the db migration on the configured database. This is the only command that will actually write to the database."]
)
var executeMigration: Boolean = false
val dryRun = accepts(DRY_RUN, """Output the database migration to the specified output file. @Option(
|The output directory is the base-directory. names = ["--$DRY_RUN"],
|You can specify a file name or 'CONSOLE' if you want to send the output to the console.""".trimMargin()) arity = "0..1",
description = ["Output the database migration to the specified output file.",
"The output directory is the base-directory.",
"You can specify a file name or 'CONSOLE' if you want to send the output to the console."]
)
var dryRun: String? = null
dryRun.withOptionalArg() @Option(
dryRun.availableUnless(runMig) names = ["--$CREATE_MIGRATION_CORDAPP"],
arity = "0..1",
description = ["Create migration files for a CorDapp.",
"You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations.",
"The output directory is the base-directory, where a `migration` folder is created."]
)
var createMigrationSqlForCordappPath: String? = null
accepts(CREATE_MIGRATION_CORDAPP, """Create migration files for a CorDapp. val createMigrationSqlForCordapp : Boolean get() = createMigrationSqlForCordappPath != null
|You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations.
|The output directory is the base-directory, where a `migration` folder is created.""".trimMargin())
.withOptionalArg()
accepts(RELEASE_LOCK, "Releases whatever locks are on the database change log table, in case shutdown failed.") @Option(
names = ["--$RELEASE_LOCK"],
accepts(HELP).forHelp() description = ["Releases whatever locks are on the database change log table, in case shutdown failed."]
)
var releaseLock: Boolean = false
} }
fun main(args: Array<String>) { fun main(args: Array<String>) {
val parser = initOptionParser() DbManagementTool().start(args)
try {
val options = parser.parse(*args)
runCommand(options, parser)
} catch (e: OptionException) {
errorAndExit(e.message)
}
} }
data class Configuration(val dataSourceProperties: Properties, val database: DatabaseConfig, val jarDirs: List<String> = emptyList()) data class Configuration(val dataSourceProperties: Properties, val database: DatabaseConfig, val jarDirs: List<String> = emptyList())
private fun runCommand(options: OptionSet, parser: OptionParser) { private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda database management tool.") {
@Mixin
var cmdLineOptions = DbManagementToolOptions()
fun baseDirectory() = Paths.get(options.valueOf(BASE_DIRECTORY) as String).toAbsolutePath().normalize() private fun checkOnlyOneCommandSelected() {
val mode = options.valueOf(MODE) as Mode val selectedOptions = mutableListOf<String>()
fun configFile(defaultCfgName: String) = baseDirectory() / ((options.valueOf(CONFIG) as String?) ?: defaultCfgName) if (cmdLineOptions.dryRun != null) selectedOptions.add(DRY_RUN)
if (cmdLineOptions.executeMigration) selectedOptions.add(EXECUTE_MIGRATION)
when { if (cmdLineOptions.createMigrationSqlForCordapp) selectedOptions.add(CREATE_MIGRATION_CORDAPP)
options.has(HELP) -> parser.printHelpOn(System.out) if (cmdLineOptions.releaseLock) selectedOptions.add(RELEASE_LOCK)
mode == Mode.NODE -> { require(selectedOptions.count() != 0) {"You must call database-manager with a command option. See --help for further info."}
val baseDirectory = baseDirectory() require(selectedOptions.count() == 1) {"You cannot call more than one of: ${selectedOptions.joinToString(", ")}. See --help for further info."}
if (!baseDirectory.exists()) {
errorAndExit("Could not find base-directory: '$baseDirectory'.")
}
val config = configFile("node.conf")
if (!config.exists()) {
errorAndExit("Not a valid node folder. Could not find the config file: '$config'.")
}
val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration()
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps"))
val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null)
handleCommand(options, baseDirectory, config, mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys)
}
mode == Mode.DOORMAN -> {
if (!options.has(DOORMAN_JAR_PATH)) {
errorAndExit("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.")
}
val fatJarPath = Paths.get(options.valueOf(DOORMAN_JAR_PATH) as String)
if (!fatJarPath.exists()) {
errorAndExit("Could not find the doorman jar in location: '$fatJarPath'.")
}
val doormanClassloader = classLoaderFromJar(fatJarPath)
val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1"
val schema = loadMappedSchema(doormanSchema, doormanClassloader)
handleCommand(options, baseDirectory(), configFile("network-management.conf"), mode, doormanClassloader, setOf(schema))
}
}
migrationLogger.info("Done")
}
private fun handleCommand(options: OptionSet, baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>) {
val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let {
if (mode == Mode.NODE) {
it.withFallback(configOf("baseDirectory" to baseDirectory.toString()))
.withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true)))
.resolve()
} else {
it
}
}
val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle)
fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource ->
withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource)
} }
when { override fun runProgram(): Int {
options.has(RELEASE_LOCK) -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve().parseAs(Configuration::class), baseDirectory, classLoader) { checkOnlyOneCommandSelected()
SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock() require(cmdLineOptions.baseDirectory != null) {"You must specify a base directory"}
} fun baseDirectory() = cmdLineOptions.baseDirectory?.toAbsolutePath()?.normalize()!!
options.has(DRY_RUN) -> { fun configFile(defaultCfgName: String) = baseDirectory() / (cmdLineOptions.configFile ?: defaultCfgName)
val writer = getMigrationOutput(baseDirectory, options) when {
migrationLogger.info("Exporting the current db migrations ...") cmdLineOptions.mode == Mode.NODE -> {
runMigrationCommand { migration, _ -> val baseDirectory = baseDirectory()
migration.generateMigrationScript(writer) if (!baseDirectory.exists()) {
} error("Could not find base-directory: '$baseDirectory'.")
}
options.has(RUN_MIGRATION) -> {
migrationLogger.info("Running the database migration on $baseDirectory")
runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) }
}
options.has(CREATE_MIGRATION_CORDAPP) && (mode == Mode.NODE) -> {
fun generateMigrationFileForSchema(schemaClass: String) {
migrationLogger.info("Creating database migration files for schema: $schemaClass into ${baseDirectory / "migration"}")
try {
runWithDataSource(config, baseDirectory, classLoader) {
MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
}
} catch (e: Exception) {
e.printStackTrace()
errorAndExit("Could not generate migration for $schemaClass: ${e.message}")
} }
} val config = configFile("node.conf")
if (!config.exists()) {
error("Not a valid node folder. Could not find the config file: '$config'.")
}
val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration()
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps"))
if (options.hasArgument(CREATE_MIGRATION_CORDAPP)) { val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null)
val schemaClass = options.valueOf(CREATE_MIGRATION_CORDAPP) as String
generateMigrationFileForSchema(schemaClass) handleCommand(baseDirectory, config, cmdLineOptions.mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys)
}
cmdLineOptions.mode == Mode.DOORMAN -> {
if (cmdLineOptions.doormanJarPath != null) {
error("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.")
}
val fatJarPath = cmdLineOptions.doormanJarPath!!
if (!fatJarPath.exists()) {
error("Could not find the doorman jar in location: '$fatJarPath'.")
}
val doormanClassloader = classLoaderFromJar(fatJarPath)
val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1"
val schema = loadMappedSchema(doormanSchema, doormanClassloader)
handleCommand(baseDirectory(), configFile("network-management.conf"), cmdLineOptions.mode, doormanClassloader, setOf(schema))
}
}
migrationLogger.info("Done")
return ExitCodes.SUCCESS
}
private fun handleCommand(baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>) {
val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let {
if (mode == Mode.NODE) {
it.withFallback(configOf("baseDirectory" to baseDirectory.toString()))
.withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true)))
.resolve()
} else { } else {
schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach { it
generateMigrationFileForSchema(it.javaClass.name)
}
} }
} }
else -> errorAndExit("Please specify a correct command") val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle)
}
}
private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray()) fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource ->
withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource)
}
private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema when {
cmdLineOptions.releaseLock -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve( ).parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle), baseDirectory, classLoader) {
SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock()
}
cmdLineOptions.dryRun != null -> {
val writer = getMigrationOutput(baseDirectory)
migrationLogger.info("Exporting the current db migrations ...")
runMigrationCommand { migration, _ ->
migration.generateMigrationScript(writer)
}
}
cmdLineOptions.executeMigration -> {
migrationLogger.info("Running the database migration on $baseDirectory")
runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) }
}
cmdLineOptions.createMigrationSqlForCordapp && (mode == Mode.NODE) -> {
fun generateMigrationFileForSchema(schemaClass: String) {
migrationLogger.info("Creating database migration files for schema: $schemaClass into ${(baseDirectory / "migration").toString().trim()}")
try {
runWithDataSource(config, baseDirectory, classLoader) {
MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
}
} catch (e: Exception) {
wrappedError("Could not generate migration for $schemaClass: ${e.message}", e)
}
}
private fun getMigrationOutput(baseDirectory: Path, options: OptionSet): Writer { if (cmdLineOptions.createMigrationSqlForCordappPath != "") {
val option = options.valueOf(DRY_RUN) as String? generateMigrationFileForSchema(cmdLineOptions.createMigrationSqlForCordappPath!!)
return when (option) { } else {
null -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql")) schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach {
CONSOLE -> PrintWriter(System.out) generateMigrationFileForSchema(it.javaClass.name)
else -> FileWriter(File(baseDirectory.toFile(), option)) }
} }
} }
else -> error("Please specify a correct command")
private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) {
val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() }
val jarDirs = config.jarDirs.map { Paths.get(it) }
for (jarDir in jarDirs) {
if (!jarDir.exists()) {
errorAndExit("Could not find the configured JDBC driver directory: '$jarDir'.")
} }
} }
return try { private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray())
withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs))
} catch (e: CheckpointsException) { private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema
errorAndExit(e.message)
} catch (e: Exception) { private fun getMigrationOutput(baseDirectory: Path): Writer {
errorAndExit("""Failed to create datasource. return when (cmdLineOptions.dryRun) {
|Please check that the correct JDBC driver is installed in one of the following folders: "" -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql"))
|${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")} CONSOLE -> PrintWriter(System.out)
|Caused By $e""".trimMargin(), e) else -> FileWriter(File(baseDirectory.toFile(), cmdLineOptions.dryRun))
}
}
private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) {
val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() }
val jarDirs = config.jarDirs.map { Paths.get(it) }
for (jarDir in jarDirs) {
if (!jarDir.exists()) {
error("Could not find the configured JDBC driver directory: '$jarDir'.")
}
}
return try {
withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs))
} catch (e: CheckpointsException) {
error(e)
} catch (e: ClassNotFoundException) {
wrappedError("Class not found in CorDapp", e)
} catch (e: Exception) {
wrappedError("""Failed to create datasource.
|Please check that the correct JDBC driver is installed in one of the following folders:
|${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")}
|Caused By $e""".trimMargin(), e)
}
}
private fun wrappedError(message: String, innerException: Exception) {
errorLogger.error(message, innerException)
throw WrappedConfigurationException(message, innerException)
}
private fun error(exception: Exception) {
errorLogger.error(exception.message, exception)
throw exception
}
private fun error(message: String) {
errorLogger.error(message)
throw ConfigurationException(message)
} }
} }
private fun errorAndExit(message: String?, exception: Exception? = null) { class ConfigurationException(message: String): Exception(message)
errorLogger.error(message, exception) class WrappedConfigurationException(message: String, val innerException: Exception): Exception(message)
System.err.println(message)
System.exit(1)
}

View File

@ -2,15 +2,21 @@
<Configuration status="info"> <Configuration status="info">
<Properties> <Properties>
<Property name="consoleLogLevel">info</Property> <Property name="consoleLogLevel">${sys:consoleLogLevel:-error}</Property>
<Property name="defaultLogLevel">debug</Property> <Property name="defaultLogLevel">${sys:defaultLogLevel:-info}</Property>
</Properties> </Properties>
<ThresholdFilter level="trace"/> <ThresholdFilter level="trace"/>
<Appenders> <Appenders>
<Console name="Console-Appender" target="SYSTEM_OUT"> <Console name="Console-Appender" target="SYSTEM_OUT">
<PatternLayout pattern="-- %date{ISO8601}{UTC}Z %c{2}.%method - %msg %n"/> <PatternLayout pattern="%highlight{[%level{length=5}] %date{HH:mm:ssZ} [%t] %c{2}.%method - %msg%n%throwable{0}}{INFO=yellow,WARN=red,FATAL=bright red}"/>
</Console>
<Console name="Simple-Console-Appender" target="SYSTEM_OUT">
<PatternLayout pattern="%msg%n%throwable{0}" />
<ThresholdFilter level="info"/>
</Console> </Console>
<File name="File-Appender" fileName="logs/migration.log"> <File name="File-Appender" fileName="logs/migration.log">
@ -30,7 +36,8 @@
<AppenderRef ref="Errors-File-Appender" /> <AppenderRef ref="Errors-File-Appender" />
</Logger> </Logger>
<Logger name="migration.tool" > <Logger name="migration.tool" >
<AppenderRef ref="Console-Appender"/> <AppenderRef ref="Simple-Console-Appender" level="INFO"/>
<AppenderRef ref="Console-Appender" level="${consoleLogLevel}"/>
<AppenderRef ref="File-Appender" /> <AppenderRef ref="File-Appender" />
</Logger> </Logger>
</Loggers> </Loggers>