diff --git a/tools/dbmigration/build.gradle b/tools/dbmigration/build.gradle index c548d19474..4a4d131c38 100644 --- a/tools/dbmigration/build.gradle +++ b/tools/dbmigration/build.gradle @@ -14,8 +14,7 @@ apply plugin: 'com.jfrog.artifactory' dependencies { compile project(':node') - // JOpt: for command line flags. - compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version" + compile project(':tools:cliutils') } import de.sebastianboegl.gradle.plugins.shadow.transformers.Log4j2PluginsFileTransformer diff --git a/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt b/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt index 92f5c49e55..cd0517f9af 100644 --- a/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt +++ b/tools/dbmigration/src/main/kotlin/com/r3/corda/dbmigration/Launcher.kt @@ -4,11 +4,9 @@ package com.r3.corda.dbmigration import com.typesafe.config.ConfigFactory import com.typesafe.config.ConfigParseOptions -import joptsimple.OptionException -import joptsimple.OptionParser -import joptsimple.OptionSet -import joptsimple.util.EnumConverter -import net.corda.nodeapi.internal.MigrationHelpers +import net.corda.cliutils.CordaCliWrapper +import net.corda.cliutils.ExitCodes +import net.corda.cliutils.start import net.corda.core.internal.div import net.corda.core.internal.exists import net.corda.core.schemas.MappedSchema @@ -20,12 +18,15 @@ import net.corda.node.services.config.parseAsNodeConfiguration import net.corda.node.services.persistence.DBCheckpointStorage import net.corda.node.services.persistence.MigrationExporter import net.corda.node.services.schema.NodeSchemaService +import net.corda.nodeapi.internal.MigrationHelpers import net.corda.nodeapi.internal.config.UnknownConfigKeysPolicy import net.corda.nodeapi.internal.config.parseAs import net.corda.nodeapi.internal.persistence.CheckpointsException import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.SchemaMigration import org.slf4j.LoggerFactory +import picocli.CommandLine.Mixin +import picocli.CommandLine.Option import java.io.File import java.io.FileWriter import java.io.PrintWriter @@ -38,12 +39,8 @@ import java.util.* import javax.sql.DataSource //command line arguments -const val HELP = "help" -const val MODE = "mode" -const val BASE_DIRECTORY = "base-directory" -const val CONFIG = "config-file" const val DOORMAN_JAR_PATH = "doorman-jar-path" -const val RUN_MIGRATION = "execute-migration" +const val EXECUTE_MIGRATION = "execute-migration" const val DRY_RUN = "dry-run" const val CREATE_MIGRATION_CORDAPP = "create-migration-sql-for-cordapp" const val RELEASE_LOCK = "release-lock" @@ -51,195 +48,237 @@ const val RELEASE_LOCK = "release-lock" // output type const val CONSOLE = "CONSOLE" -private val migrationLogger = LoggerFactory.getLogger("migration.tool") -private val errorLogger = LoggerFactory.getLogger("errors") +// initialise loggers lazily as some configuration is changed on startup and if loggers are already initialised it will be ignored +private val migrationLogger by lazy {LoggerFactory.getLogger("migration.tool") } +private val errorLogger by lazy { LoggerFactory.getLogger("errors") } private enum class Mode { NODE, DOORMAN } -private fun initOptionParser(): OptionParser = OptionParser().apply { - accepts(MODE, "Either 'NODE' or 'DOORMAN'. By default 'NODE'") - .withOptionalArg() - .withValuesConvertedBy(object : EnumConverter(Mode::class.java) {}) - .defaultsTo(Mode.NODE) +private class DbManagementToolOptions { + @Option( + names = ["--mode"], + description = ["The operating mode."] + ) + var mode: Mode = Mode.NODE - accepts(BASE_DIRECTORY, "The node or doorman directory") - .withRequiredArg().required() + @Option( + names = ["-b", "--base-directory"], + description = ["The node or doorman directory."] + ) + var baseDirectory: Path? = null - accepts(CONFIG, "The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman.") - .withOptionalArg() + @Option( + names = ["-f", "--config-file"], + description = ["The name of the config file. By default 'node.conf' for a simple node and 'network-management.conf' for a doorman."] + ) + var configFile: String? = null - accepts(DOORMAN_JAR_PATH, "The path to the doorman JAR") - .withOptionalArg() + @Option( + names = ["--$DOORMAN_JAR_PATH"], + description = ["The path to the doorman JAR."] + ) + var doormanJarPath: Path? = null - val runMig = accepts(RUN_MIGRATION, - "This option will run the db migration on the configured database. This is the only command that will actually write to the database.") + @Option( + names = ["--$EXECUTE_MIGRATION"], + description = ["This option will run the db migration on the configured database. This is the only command that will actually write to the database."] + ) + var executeMigration: Boolean = false - val dryRun = accepts(DRY_RUN, """Output the database migration to the specified output file. - |The output directory is the base-directory. - |You can specify a file name or 'CONSOLE' if you want to send the output to the console.""".trimMargin()) + @Option( + names = ["--$DRY_RUN"], + arity = "0..1", + description = ["Output the database migration to the specified output file.", + "The output directory is the base-directory.", + "You can specify a file name or 'CONSOLE' if you want to send the output to the console."] + ) + var dryRun: String? = null - dryRun.withOptionalArg() - dryRun.availableUnless(runMig) + @Option( + names = ["--$CREATE_MIGRATION_CORDAPP"], + arity = "0..1", + description = ["Create migration files for a CorDapp.", + "You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations.", + "The output directory is the base-directory, where a `migration` folder is created."] + ) + var createMigrationSqlForCordappPath: String? = null - accepts(CREATE_MIGRATION_CORDAPP, """Create migration files for a CorDapp. - |You can specify the fully qualified name of the `MappedSchema` class. If not specified it will generate the migration for all schemas that don't have migrations. - |The output directory is the base-directory, where a `migration` folder is created.""".trimMargin()) - .withOptionalArg() + val createMigrationSqlForCordapp : Boolean get() = createMigrationSqlForCordappPath != null - accepts(RELEASE_LOCK, "Releases whatever locks are on the database change log table, in case shutdown failed.") - - accepts(HELP).forHelp() + @Option( + names = ["--$RELEASE_LOCK"], + description = ["Releases whatever locks are on the database change log table, in case shutdown failed."] + ) + var releaseLock: Boolean = false } fun main(args: Array) { - val parser = initOptionParser() - try { - val options = parser.parse(*args) - runCommand(options, parser) - } catch (e: OptionException) { - errorAndExit(e.message) - } + DbManagementTool().start(args) } data class Configuration(val dataSourceProperties: Properties, val database: DatabaseConfig, val jarDirs: List = emptyList()) -private fun runCommand(options: OptionSet, parser: OptionParser) { +private class DbManagementTool : CordaCliWrapper("database-manager", "The Corda database management tool.") { + @Mixin + var cmdLineOptions = DbManagementToolOptions() - fun baseDirectory() = Paths.get(options.valueOf(BASE_DIRECTORY) as String).toAbsolutePath().normalize() - val mode = options.valueOf(MODE) as Mode - fun configFile(defaultCfgName: String) = baseDirectory() / ((options.valueOf(CONFIG) as String?) ?: defaultCfgName) - - when { - options.has(HELP) -> parser.printHelpOn(System.out) - mode == Mode.NODE -> { - val baseDirectory = baseDirectory() - if (!baseDirectory.exists()) { - errorAndExit("Could not find base-directory: '$baseDirectory'.") - } - val config = configFile("node.conf") - if (!config.exists()) { - errorAndExit("Not a valid node folder. Could not find the config file: '$config'.") - } - val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration() - val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps")) - - val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null) - - handleCommand(options, baseDirectory, config, mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys) - } - mode == Mode.DOORMAN -> { - if (!options.has(DOORMAN_JAR_PATH)) { - errorAndExit("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.") - } - val fatJarPath = Paths.get(options.valueOf(DOORMAN_JAR_PATH) as String) - if (!fatJarPath.exists()) { - errorAndExit("Could not find the doorman jar in location: '$fatJarPath'.") - } - val doormanClassloader = classLoaderFromJar(fatJarPath) - val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1" - val schema = loadMappedSchema(doormanSchema, doormanClassloader) - handleCommand(options, baseDirectory(), configFile("network-management.conf"), mode, doormanClassloader, setOf(schema)) - } - } - migrationLogger.info("Done") -} - -private fun handleCommand(options: OptionSet, baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set) { - val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let { - if (mode == Mode.NODE) { - it.withFallback(configOf("baseDirectory" to baseDirectory.toString())) - .withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true))) - .resolve() - } else { - it - } - } - val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle) - - fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource -> - withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource) + private fun checkOnlyOneCommandSelected() { + val selectedOptions = mutableListOf() + if (cmdLineOptions.dryRun != null) selectedOptions.add(DRY_RUN) + if (cmdLineOptions.executeMigration) selectedOptions.add(EXECUTE_MIGRATION) + if (cmdLineOptions.createMigrationSqlForCordapp) selectedOptions.add(CREATE_MIGRATION_CORDAPP) + if (cmdLineOptions.releaseLock) selectedOptions.add(RELEASE_LOCK) + require(selectedOptions.count() != 0) {"You must call database-manager with a command option. See --help for further info."} + require(selectedOptions.count() == 1) {"You cannot call more than one of: ${selectedOptions.joinToString(", ")}. See --help for further info."} } - when { - options.has(RELEASE_LOCK) -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve().parseAs(Configuration::class), baseDirectory, classLoader) { - SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock() - } - options.has(DRY_RUN) -> { - val writer = getMigrationOutput(baseDirectory, options) - migrationLogger.info("Exporting the current db migrations ...") - runMigrationCommand { migration, _ -> - migration.generateMigrationScript(writer) - } - } - options.has(RUN_MIGRATION) -> { - migrationLogger.info("Running the database migration on $baseDirectory") - runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) } - } - options.has(CREATE_MIGRATION_CORDAPP) && (mode == Mode.NODE) -> { - - fun generateMigrationFileForSchema(schemaClass: String) { - migrationLogger.info("Creating database migration files for schema: $schemaClass into ${baseDirectory / "migration"}") - try { - runWithDataSource(config, baseDirectory, classLoader) { - MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass) - } - } catch (e: Exception) { - e.printStackTrace() - errorAndExit("Could not generate migration for $schemaClass: ${e.message}") + override fun runProgram(): Int { + checkOnlyOneCommandSelected() + require(cmdLineOptions.baseDirectory != null) {"You must specify a base directory"} + fun baseDirectory() = cmdLineOptions.baseDirectory?.toAbsolutePath()?.normalize()!! + fun configFile(defaultCfgName: String) = baseDirectory() / (cmdLineOptions.configFile ?: defaultCfgName) + when { + cmdLineOptions.mode == Mode.NODE -> { + val baseDirectory = baseDirectory() + if (!baseDirectory.exists()) { + error("Could not find base-directory: '$baseDirectory'.") } - } + val config = configFile("node.conf") + if (!config.exists()) { + error("Not a valid node folder. Could not find the config file: '$config'.") + } + val nodeConfig = ConfigHelper.loadConfig(baseDirectory, config).parseAsNodeConfiguration() + val cordappLoader = JarScanningCordappLoader.fromDirectories(setOf(baseDirectory, baseDirectory / "cordapps")) - if (options.hasArgument(CREATE_MIGRATION_CORDAPP)) { - val schemaClass = options.valueOf(CREATE_MIGRATION_CORDAPP) as String - generateMigrationFileForSchema(schemaClass) + val schemaService = NodeSchemaService(extraSchemas = cordappLoader.cordappSchemas, includeNotarySchemas = nodeConfig.notary != null) + + handleCommand(baseDirectory, config, cmdLineOptions.mode, cordappLoader.appClassLoader, schemaService.schemaOptions.keys) + } + cmdLineOptions.mode == Mode.DOORMAN -> { + if (cmdLineOptions.doormanJarPath != null) { + error("The $DOORMAN_JAR_PATH argument is required when running in doorman mode.") + } + val fatJarPath = cmdLineOptions.doormanJarPath!! + if (!fatJarPath.exists()) { + error("Could not find the doorman jar in location: '$fatJarPath'.") + } + val doormanClassloader = classLoaderFromJar(fatJarPath) + val doormanSchema = "com.r3.corda.networkmanage.common.persistence.NetworkManagementSchemaServices\$SchemaV1" + val schema = loadMappedSchema(doormanSchema, doormanClassloader) + handleCommand(baseDirectory(), configFile("network-management.conf"), cmdLineOptions.mode, doormanClassloader, setOf(schema)) + } + } + migrationLogger.info("Done") + return ExitCodes.SUCCESS + } + + private fun handleCommand(baseDirectory: Path, configFile: Path, mode: Mode, classLoader: ClassLoader, schemas: Set) { + val parsedConfig = ConfigFactory.parseFile(configFile.toFile()).resolve().let { + if (mode == Mode.NODE) { + it.withFallback(configOf("baseDirectory" to baseDirectory.toString())) + .withFallback(ConfigFactory.parseResources("reference.conf", ConfigParseOptions.defaults().setAllowMissing(true))) + .resolve() } else { - schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach { - generateMigrationFileForSchema(it.javaClass.name) - } + it } } - else -> errorAndExit("Please specify a correct command") - } -} + val config = parsedConfig.parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle) -private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray()) + fun runMigrationCommand(withMigration: (SchemaMigration, DataSource) -> Unit): Unit = runWithDataSource(config, baseDirectory, classLoader) { dataSource -> + withMigration(SchemaMigration(schemas, dataSource, config.database, classLoader), dataSource) + } -private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema + when { + cmdLineOptions.releaseLock -> runWithDataSource(ConfigFactory.parseFile(configFile.toFile()).resolve( ).parseAs(Configuration::class, UnknownConfigKeysPolicy.IGNORE::handle), baseDirectory, classLoader) { + SchemaMigration(emptySet(), it, config.database, Thread.currentThread().contextClassLoader).forceReleaseMigrationLock() + } + cmdLineOptions.dryRun != null -> { + val writer = getMigrationOutput(baseDirectory) + migrationLogger.info("Exporting the current db migrations ...") + runMigrationCommand { migration, _ -> + migration.generateMigrationScript(writer) + } + } + cmdLineOptions.executeMigration -> { + migrationLogger.info("Running the database migration on $baseDirectory") + runMigrationCommand { migration, dataSource -> migration.runMigration(dataSource.connection.use { DBCheckpointStorage().getCheckpointCount(it) != 0L }) } + } + cmdLineOptions.createMigrationSqlForCordapp && (mode == Mode.NODE) -> { + fun generateMigrationFileForSchema(schemaClass: String) { + migrationLogger.info("Creating database migration files for schema: $schemaClass into ${(baseDirectory / "migration").toString().trim()}") + try { + runWithDataSource(config, baseDirectory, classLoader) { + MigrationExporter(baseDirectory, config.dataSourceProperties, classLoader, it).generateMigrationForCorDapp(schemaClass) + } + } catch (e: Exception) { + wrappedError("Could not generate migration for $schemaClass: ${e.message}", e) + } + } -private fun getMigrationOutput(baseDirectory: Path, options: OptionSet): Writer { - val option = options.valueOf(DRY_RUN) as String? - return when (option) { - null -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql")) - CONSOLE -> PrintWriter(System.out) - else -> FileWriter(File(baseDirectory.toFile(), option)) - } -} - -private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) { - val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() } - val jarDirs = config.jarDirs.map { Paths.get(it) } - for (jarDir in jarDirs) { - if (!jarDir.exists()) { - errorAndExit("Could not find the configured JDBC driver directory: '$jarDir'.") + if (cmdLineOptions.createMigrationSqlForCordappPath != "") { + generateMigrationFileForSchema(cmdLineOptions.createMigrationSqlForCordappPath!!) + } else { + schemas.filter { MigrationHelpers.getMigrationResource(it, classLoader) == null }.forEach { + generateMigrationFileForSchema(it.javaClass.name) + } + } + } + else -> error("Please specify a correct command") } } - return try { - withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs)) - } catch (e: CheckpointsException) { - errorAndExit(e.message) - } catch (e: Exception) { - errorAndExit("""Failed to create datasource. - |Please check that the correct JDBC driver is installed in one of the following folders: - |${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")} - |Caused By $e""".trimMargin(), e) + private fun classLoaderFromJar(jarPath: Path): ClassLoader = URLClassLoader(listOf(jarPath.toUri().toURL()).toTypedArray()) + + private fun loadMappedSchema(schemaName: String, classLoader: ClassLoader) = classLoader.loadClass(schemaName).kotlin.objectInstance as MappedSchema + + private fun getMigrationOutput(baseDirectory: Path): Writer { + return when (cmdLineOptions.dryRun) { + "" -> FileWriter(File(baseDirectory.toFile(), "migration${SimpleDateFormat("yyyyMMddHHmmss").format(Date())}.sql")) + CONSOLE -> PrintWriter(System.out) + else -> FileWriter(File(baseDirectory.toFile(), cmdLineOptions.dryRun)) + } + } + + private fun runWithDataSource(config: Configuration, baseDirectory: Path, classLoader: ClassLoader, withDatasource: (DataSource) -> Unit) { + val driversFolder = (baseDirectory / "drivers").let { if (it.exists()) listOf(it) else emptyList() } + val jarDirs = config.jarDirs.map { Paths.get(it) } + for (jarDir in jarDirs) { + if (!jarDir.exists()) { + error("Could not find the configured JDBC driver directory: '$jarDir'.") + } + } + + return try { + withDatasource(createDatasourceFromDriverJarFolders(config.dataSourceProperties, classLoader, driversFolder + jarDirs)) + } catch (e: CheckpointsException) { + error(e) + } catch (e: ClassNotFoundException) { + wrappedError("Class not found in CorDapp", e) + } catch (e: Exception) { + wrappedError("""Failed to create datasource. + |Please check that the correct JDBC driver is installed in one of the following folders: + |${(driversFolder + jarDirs).joinToString("\n\t - ", "\t - ")} + |Caused By $e""".trimMargin(), e) + } + } + + private fun wrappedError(message: String, innerException: Exception) { + errorLogger.error(message, innerException) + throw WrappedConfigurationException(message, innerException) + } + + private fun error(exception: Exception) { + errorLogger.error(exception.message, exception) + throw exception + } + + private fun error(message: String) { + errorLogger.error(message) + throw ConfigurationException(message) } } -private fun errorAndExit(message: String?, exception: Exception? = null) { - errorLogger.error(message, exception) - System.err.println(message) - System.exit(1) -} +class ConfigurationException(message: String): Exception(message) +class WrappedConfigurationException(message: String, val innerException: Exception): Exception(message) \ No newline at end of file diff --git a/tools/dbmigration/src/main/resources/log4j2.xml b/tools/dbmigration/src/main/resources/log4j2.xml index b525d8672d..a6b2aec453 100644 --- a/tools/dbmigration/src/main/resources/log4j2.xml +++ b/tools/dbmigration/src/main/resources/log4j2.xml @@ -2,15 +2,21 @@ - info - debug + ${sys:consoleLogLevel:-error} + ${sys:defaultLogLevel:-info} - + + + + + + + @@ -30,7 +36,8 @@ - + +