Make database manager use picocli base class withstandardised options

This commit is contained in:
Anthony Keenan 2018-09-25 16:14:09 +01:00
parent 02ae92fc87
commit b3307eaecb
3 changed files with 216 additions and 171 deletions

View File

@ -14,8 +14,7 @@ apply plugin: 'com.jfrog.artifactory'
dependencies {
compile project(':node')
// JOpt: for command line flags.
compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
compile project(':tools:cliutils')
}
import de.sebastianboegl.gradle.plugins.shadow.transformers.Log4j2PluginsFileTransformer

View File

@ -4,11 +4,9 @@ package com.r3.corda.dbmigration
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigParseOptions
import joptsimple.OptionException
import joptsimple.OptionParser
import joptsimple.OptionSet
import joptsimple.util.EnumConverter
import net.corda.nodeapi.internal.MigrationHelpers
import net.corda.cliutils.CordaCliWrapper
import net.corda.cliutils.ExitCodes
import net.corda.cliutils.start
import net.corda.core.internal.div
import net.corda.core.internal.exists
import net.corda.core.schemas.MappedSchema
@ -20,12 +18,15 @@ import net.corda.node.services.config.parseAsNodeConfiguration
import net.corda.node.services.persistence.DBCheckpointStorage
import net.corda.node.services.persistence.MigrationExporter
import net.corda.node.services.schema.NodeSchemaService
import net.corda.nodeapi.internal.MigrationHelpers
import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy
import net.corda.nodeapi.internal.config.parseAs
import net.corda.nodeapi.internal.persistence.CheckpointsException
import net.corda.nodeapi.internal.persistence.DatabaseConfig
import net.corda.nodeapi.internal.persistence.SchemaMigration
import org.slf4j.LoggerFactory
import picocli.CommandLine.Mixin
import picocli.CommandLine.Option
import java.io.File
import java.io.FileWriter
import java.io.PrintWriter
@ -38,12 +39,8 @@ import java.util.*
import javax.sql.DataSource
//command line arguments
const val HELP = "help"
const val MODE = "mode"
const val BASE_DIRECTORY = "base-directory"
const val CONFIG = "config-file"
const val DOORMAN_JAR_PATH = "doorman-jar-path"
const val RUN_MIGRATION = "execute-migration"
const val EXECUTE_MIGRATION = "execute-migration"
const val DRY_RUN = "dry-run"
const val CREATE_MIGRATION_CORDAPP = "create-migration-sql-for-cordapp"
const val RELEASE_LOCK = "release-lock"
@ -51,195 +48,237 @@ const val RELEASE_LOCK = "release-lock"
// output type
const val CONSOLE = "CONSOLE"
private val migrationLogger = LoggerFactory.getLogger("migration.tool")
private val errorLogger = LoggerFactory.getLogger("errors")
// initialise loggers lazily as some configuration is changed on startup and if loggers are already initialised it will be ignored
private val migrationLogger by lazy {LoggerFactory.getLogger("migration.tool") }
private val errorLogger by lazy { LoggerFactory.getLogger("errors") }
private enum class Mode {
NODE, DOORMAN
}
private fun initOptionParser(): OptionParser = OptionParser().apply {
accepts(MODE, "Either 'NODE' or 'DOORMAN'. By default 'NODE'")
.withOptionalArg()
.withValuesConvertedBy(object : EnumConverter<Mode>(Mode::class.java) {})
.defaultsTo(Mode.NODE)
private class DbManagementToolOptions {
@Option(
names = ["--mode"],
description = ["The operating mode."]
)
var mode: Mode = Mode.NODE
accepts(BASE_DIRECTORY, "The node or doorman directory")
.withRequiredArg().required()
@Option(
names = ["-b", "--base-directory"],
description = ["The node or doorman directory."]
)
var baseDirectory: Path? = null
accepts(CONFIG, "The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman.")
.withOptionalArg()
@Option(
names = ["-f", "--config-file"],
description = ["The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman."]
)
var configFile: String? = null
accepts(DOORMAN_JAR_PATH, "The path to the doorman JAR")
.withOptionalArg()
@Option(
names = ["--$DOORMAN_JAR_PATH"],
description = ["The path to the doorman JAR."]
)
var doormanJarPath: Path? = null
val runMig = accepts(RUN_MIGRATION,
"This option will run the db migration on the configured database. This is the only command that will actually write to the database.")
@Option(
names = ["--$EXECUTE_MIGRATION"],
description = ["This option will run the db migration on the configured database. This is the only command that will actually write to the database."]
)
var executeMigration: Boolean = false
val dryRun = accepts(DRY_RUN, """Output the database migration to the specified output file.
|The output directory is the base-directory.
|You can specify a file name or 'CONSOLE' if you want to send the output to the console.""".trimMargin())
@Option(
names = ["--$DRY_RUN"],
arity = "0..1",
description = ["Output the database migration to the specified output file.",
"The output directory is the base-directory.",
"You can specify a file name or 'CONSOLE' if you want to send the output to the console."]
)
var dryRun: String? = null
dryRun.withOptionalArg()
dryRun.availableUnless(runMig)
@Option(
names = ["--$CREATE_MIGRATION_CORDAPP"],
arity = "0..1",
description = ["Create migration files for a CorDapp.",
"You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations.",
"The output directory is the base-directory, where a `migration` folder is created."]
)
var createMigrationSqlForCordappPath: String? = null
accepts(CREATE_MIGRATION_CORDAPP, """Create migration files for a CorDapp.
|You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations.
|The output directory is the base-directory, where a `migration` folder is created.""".trimMargin())
.withOptionalArg()
val createMigrationSqlForCordapp : Boolean get() = createMigrationSqlForCordappPath != null
accepts(RELEASE_LOCK, "Releases whatever locks are on the database change log table, in case shutdown failed.")
accepts(HELP).forHelp()
@Option(
names = ["--$RELEASE_LOCK"],
description = ["Releases whatever locks are on the database change log table, in case shutdown failed."]
)
var releaseLock: Boolean = false
}
fun main(args: Array<String>) {
val parser = initOptionParser()
try {
val options = parser.parse(*args)
runCommand(options, parser)
} catch (e: OptionException) {
errorAndExit(e.message)
}
DbManagementTool().start(args)
}
data class Configuration(val dataSourceProperties: Properties, val database: DatabaseConfig, val jarDirs: List<String> = emptyList())
private fun runCommand(options: OptionSet, parser: OptionParser) {
private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda database management tool.") {
@Mixin
var cmdLineOptions = DbManagementToolOptions()
fun baseDirectory() = Paths.get(options.valueOf(BASE_DIRECTORY) as String).toAbsolutePath().normalize()
val mode = options.valueOf(MODE) as Mode
fun configFile(defaultCfgName: String) = baseDirectory() / ((options.valueOf(CONFIG) as String?) ?: defaultCfgName)
when {
options.has(HELP) -> parser.printHelpOn(System.out)
mode == Mode.NODE -> {
val baseDirectory = baseDirectory()
if (!baseDirectory.exists()) {
errorAndExit("Could not find base-directory: '$baseDirectory'.")
}
val config = configFile("node.conf")
if (!config.exists()) {
errorAndExit("Not a valid node folder. Could not find the config file: '$config'.")
}
val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration()
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps"))
val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null)
handleCommand(options, baseDirectory, config, mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys)
}
mode == Mode.DOORMAN -> {
if (!options.has(DOORMAN_JAR_PATH)) {
errorAndExit("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.")
}
val fatJarPath = Paths.get(options.valueOf(DOORMAN_JAR_PATH) as String)
if (!fatJarPath.exists()) {
errorAndExit("Could not find the doorman jar in location: '$fatJarPath'.")
}
val doormanClassloader = classLoaderFromJar(fatJarPath)
val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1"
val schema = loadMappedSchema(doormanSchema, doormanClassloader)
handleCommand(options, baseDirectory(), configFile("network-management.conf"), mode, doormanClassloader, setOf(schema))
}
}
migrationLogger.info("Done")
}
private fun handleCommand(options: OptionSet, baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>) {
val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let {
if (mode == Mode.NODE) {
it.withFallback(configOf("baseDirectory" to baseDirectory.toString()))
.withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true)))
.resolve()
} else {
it
}
}
val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle)
fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource ->
withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource)
private fun checkOnlyOneCommandSelected() {
val selectedOptions = mutableListOf<String>()
if (cmdLineOptions.dryRun != null) selectedOptions.add(DRY_RUN)
if (cmdLineOptions.executeMigration) selectedOptions.add(EXECUTE_MIGRATION)
if (cmdLineOptions.createMigrationSqlForCordapp) selectedOptions.add(CREATE_MIGRATION_CORDAPP)
if (cmdLineOptions.releaseLock) selectedOptions.add(RELEASE_LOCK)
require(selectedOptions.count() != 0) {"You must call database-manager with a command option. See --help for further info."}
require(selectedOptions.count() == 1) {"You cannot call more than one of: ${selectedOptions.joinToString(", ")}. See --help for further info."}
}
when {
options.has(RELEASE_LOCK) -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve().parseAs(Configuration::class), baseDirectory, classLoader) {
SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock()
}
options.has(DRY_RUN) -> {
val writer = getMigrationOutput(baseDirectory, options)
migrationLogger.info("Exporting the current db migrations ...")
runMigrationCommand { migration, _ ->
migration.generateMigrationScript(writer)
}
}
options.has(RUN_MIGRATION) -> {
migrationLogger.info("Running the database migration on $baseDirectory")
runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) }
}
options.has(CREATE_MIGRATION_CORDAPP) && (mode == Mode.NODE) -> {
fun generateMigrationFileForSchema(schemaClass: String) {
migrationLogger.info("Creating database migration files for schema: $schemaClass into ${baseDirectory / "migration"}")
try {
runWithDataSource(config, baseDirectory, classLoader) {
MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
}
} catch (e: Exception) {
e.printStackTrace()
errorAndExit("Could not generate migration for $schemaClass: ${e.message}")
override fun runProgram(): Int {
checkOnlyOneCommandSelected()
require(cmdLineOptions.baseDirectory != null) {"You must specify a base directory"}
fun baseDirectory() = cmdLineOptions.baseDirectory?.toAbsolutePath()?.normalize()!!
fun configFile(defaultCfgName: String) = baseDirectory() / (cmdLineOptions.configFile ?: defaultCfgName)
when {
cmdLineOptions.mode == Mode.NODE -> {
val baseDirectory = baseDirectory()
if (!baseDirectory.exists()) {
error("Could not find base-directory: '$baseDirectory'.")
}
}
val config = configFile("node.conf")
if (!config.exists()) {
error("Not a valid node folder. Could not find the config file: '$config'.")
}
val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration()
val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps"))
if (options.hasArgument(CREATE_MIGRATION_CORDAPP)) {
val schemaClass = options.valueOf(CREATE_MIGRATION_CORDAPP) as String
generateMigrationFileForSchema(schemaClass)
val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null)
handleCommand(baseDirectory, config, cmdLineOptions.mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys)
}
cmdLineOptions.mode == Mode.DOORMAN -> {
if (cmdLineOptions.doormanJarPath != null) {
error("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.")
}
val fatJarPath = cmdLineOptions.doormanJarPath!!
if (!fatJarPath.exists()) {
error("Could not find the doorman jar in location: '$fatJarPath'.")
}
val doormanClassloader = classLoaderFromJar(fatJarPath)
val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1"
val schema = loadMappedSchema(doormanSchema, doormanClassloader)
handleCommand(baseDirectory(), configFile("network-management.conf"), cmdLineOptions.mode, doormanClassloader, setOf(schema))
}
}
migrationLogger.info("Done")
return ExitCodes.SUCCESS
}
private fun handleCommand(baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set<MappedSchema>) {
val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let {
if (mode == Mode.NODE) {
it.withFallback(configOf("baseDirectory" to baseDirectory.toString()))
.withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true)))
.resolve()
} else {
schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach {
generateMigrationFileForSchema(it.javaClass.name)
}
it
}
}
else -> errorAndExit("Please specify a correct command")
}
}
val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle)
private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray())
fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource ->
withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource)
}
private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema
when {
cmdLineOptions.releaseLock -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve( ).parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle), baseDirectory, classLoader) {
SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock()
}
cmdLineOptions.dryRun != null -> {
val writer = getMigrationOutput(baseDirectory)
migrationLogger.info("Exporting the current db migrations ...")
runMigrationCommand { migration, _ ->
migration.generateMigrationScript(writer)
}
}
cmdLineOptions.executeMigration -> {
migrationLogger.info("Running the database migration on $baseDirectory")
runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) }
}
cmdLineOptions.createMigrationSqlForCordapp && (mode == Mode.NODE) -> {
fun generateMigrationFileForSchema(schemaClass: String) {
migrationLogger.info("Creating database migration files for schema: $schemaClass into ${(baseDirectory / "migration").toString().trim()}")
try {
runWithDataSource(config, baseDirectory, classLoader) {
MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass)
}
} catch (e: Exception) {
wrappedError("Could not generate migration for $schemaClass: ${e.message}", e)
}
}
private fun getMigrationOutput(baseDirectory: Path, options: OptionSet): Writer {
val option = options.valueOf(DRY_RUN) as String?
return when (option) {
null -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql"))
CONSOLE -> PrintWriter(System.out)
else -> FileWriter(File(baseDirectory.toFile(), option))
}
}
private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) {
val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() }
val jarDirs = config.jarDirs.map { Paths.get(it) }
for (jarDir in jarDirs) {
if (!jarDir.exists()) {
errorAndExit("Could not find the configured JDBC driver directory: '$jarDir'.")
if (cmdLineOptions.createMigrationSqlForCordappPath != "") {
generateMigrationFileForSchema(cmdLineOptions.createMigrationSqlForCordappPath!!)
} else {
schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach {
generateMigrationFileForSchema(it.javaClass.name)
}
}
}
else -> error("Please specify a correct command")
}
}
return try {
withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs))
} catch (e: CheckpointsException) {
errorAndExit(e.message)
} catch (e: Exception) {
errorAndExit("""Failed to create datasource.
|Please check that the correct JDBC driver is installed in one of the following folders:
|${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")}
|Caused By $e""".trimMargin(), e)
private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray())
private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema
private fun getMigrationOutput(baseDirectory: Path): Writer {
return when (cmdLineOptions.dryRun) {
"" -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql"))
CONSOLE -> PrintWriter(System.out)
else -> FileWriter(File(baseDirectory.toFile(), cmdLineOptions.dryRun))
}
}
private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) {
val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() }
val jarDirs = config.jarDirs.map { Paths.get(it) }
for (jarDir in jarDirs) {
if (!jarDir.exists()) {
error("Could not find the configured JDBC driver directory: '$jarDir'.")
}
}
return try {
withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs))
} catch (e: CheckpointsException) {
error(e)
} catch (e: ClassNotFoundException) {
wrappedError("Class not found in CorDapp", e)
} catch (e: Exception) {
wrappedError("""Failed to create datasource.
|Please check that the correct JDBC driver is installed in one of the following folders:
|${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")}
|Caused By $e""".trimMargin(), e)
}
}
private fun wrappedError(message: String, innerException: Exception) {
errorLogger.error(message, innerException)
throw WrappedConfigurationException(message, innerException)
}
private fun error(exception: Exception) {
errorLogger.error(exception.message, exception)
throw exception
}
private fun error(message: String) {
errorLogger.error(message)
throw ConfigurationException(message)
}
}
private fun errorAndExit(message: String?, exception: Exception? = null) {
errorLogger.error(message, exception)
System.err.println(message)
System.exit(1)
}
class ConfigurationException(message: String): Exception(message)
class WrappedConfigurationException(message: String, val innerException: Exception): Exception(message)

View File

@ -2,15 +2,21 @@
<Configuration status="info">
<Properties>
<Property name="consoleLogLevel">info</Property>
<Property name="defaultLogLevel">debug</Property>
<Property name="consoleLogLevel">${sys:consoleLogLevel:-error}</Property>
<Property name="defaultLogLevel">${sys:defaultLogLevel:-info}</Property>
</Properties>
<ThresholdFilter level="trace"/>
<Appenders>
<Console name="Console-Appender" target="SYSTEM_OUT">
<PatternLayout pattern="-- %date{ISO8601}{UTC}Z %c{2}.%method - %msg %n"/>
<PatternLayout pattern="%highlight{[%level{length=5}] %date{HH:mm:ssZ} [%t] %c{2}.%method - %msg%n%throwable{0}}{INFO=yellow,WARN=red,FATAL=bright red}"/>
</Console>
<Console name="Simple-Console-Appender" target="SYSTEM_OUT">
<PatternLayout pattern="%msg%n%throwable{0}" />
<ThresholdFilter level="info"/>
</Console>
<File name="File-Appender" fileName="logs/migration.log">
@ -30,7 +36,8 @@
<AppenderRef ref="Errors-File-Appender" />
</Logger>
<Logger name="migration.tool" >
<AppenderRef ref="Console-Appender"/>
<AppenderRef ref="Simple-Console-Appender" level="INFO"/>
<AppenderRef ref="Console-Appender" level="${consoleLogLevel}"/>
<AppenderRef ref="File-Appender" />
</Logger>
</Loggers>