CORDA-319: Shell: use ExternalResolver to load our commands

This eliminates JIT java compilation and the consequent need for
tools.jar (which doesn't get shipped in DemoBench). It also makes
development more pleasant by avoiding weird IDE integration issues
that came from having java-in-resources.

node/build.gradle

@@ -141,7 +141,9 @@ dependencies {
     compile "io.netty:netty-all:$netty_version"
 
     // CRaSH: An embeddable monitoring and admin shell with support for adding new commands written in Groovy.
-    compile "com.github.corda:crash:A2"
+    compile("com.github.corda.crash:crash.shell:9d242da2a10e686f33a3aefc69e4768824ad0716") {
+        exclude group: "org.slf4j", module: "slf4j-jdk14"
+    }
 
     // OkHTTP: Simple HTTP library.
     compile "com.squareup.okhttp3:okhttp:$okhttp_version"

node/src/main/java/net/corda/node/shell/FlowShellCommand.java

@@ -1,4 +1,4 @@
-package net.corda.node;
+package net.corda.node.shell;
 
 // See the comments at the top of run.java
 
@@ -8,7 +8,7 @@ import org.crsh.text.*;
 
 import java.util.*;
 
-import static net.corda.node.InteractiveShell.*;
+import static net.corda.node.shell.InteractiveShell.*;
 
 @Man(
     "Allows you to list and start flows. This is the primary way in which you command the node to change the ledger.\n\n" +
@@ -17,7 +17,7 @@ import static net.corda.node.InteractiveShell.*;
     "flow constructors (the right one is picked automatically) are then specified using the same syntax as for the run command."
 )
 @Usage("Start a (work)flow on the node. This is how you can change the ledger.")
-public class flow extends InteractiveShellCommand {
+public class FlowShellCommand extends InteractiveShellCommand {
     // Note that the class name is deliberately lower case, because we want the command the user types to be
     // lower case. CRaSH should ideally lowercase the command names for us, but it doesn't.
 

node/src/main/java/net/corda/node/shell/RunShellCommand.java

@@ -1,24 +1,16 @@
-package net.corda.node;
+package net.corda.node.shell;
 
 import net.corda.core.messaging.*;
 import net.corda.jackson.*;
 import org.crsh.cli.*;
 import org.crsh.command.*;
-import org.crsh.text.*;
 
-import java.lang.reflect.*;
 import java.util.*;
-import java.util.concurrent.*;
 
-import static net.corda.node.InteractiveShell.*;
+// Note that this class cannot be converted to Kotlin because CRaSH does not understand InvocationContext<Map<?, ?>> which
+// is the closest you can get in Kotlin to raw types.
 
-// This file is actually compiled at runtime with a bundled Java compiler by CRaSH. That's pretty weak: being able
-// to do this is a neat party trick and means people can write new commands in Java then just drop them into
-// their node directory, but it makes the first usage of the command slower for no good reason. There is a PR
-// in the upstream CRaSH project that adds an ExternalResolver which might be useful. Then we could convert this
-// file to Kotlin too.
-
-public class run extends InteractiveShellCommand {
+public class RunShellCommand extends InteractiveShellCommand {
     @Command
     @Man(
         "Runs a method from the CordaRPCOps interface, which is the same interface exposed to RPC clients.\n\n" +

node/src/main/kotlin/net/corda/node/Corda.kt

@@ -11,6 +11,7 @@ import net.corda.core.node.Version
 import net.corda.core.utilities.Emoji
 import net.corda.node.internal.Node
 import net.corda.node.services.config.FullNodeConfiguration
+import net.corda.node.shell.InteractiveShell
 import net.corda.node.utilities.registration.HTTPNetworkRegistrationService
 import net.corda.node.utilities.registration.NetworkRegistrationHelper
 import org.fusesource.jansi.Ansi
@@ -133,7 +134,11 @@ fun main(args: Array<String>) {
             // Don't start the shell if there's no console attached.
             val runShell = !cmdlineOptions.noLocalShell && System.console() != null
             node.startupComplete then {
-                InteractiveShell.startShell(dir, runShell, cmdlineOptions.sshdServer, node)
+                try {
+                    InteractiveShell.startShell(dir, runShell, cmdlineOptions.sshdServer, node)
+                } catch(e: Throwable) {
+                    log.error("Shell failed to start", e)
+                }
             }
         } failure {
             log.error("Error during network map registration", it)

node/src/main/kotlin/net/corda/node/shell/InteractiveShell.kt

@@ -1,4 +1,4 @@
-package net.corda.node
+package net.corda.node.shell
 
 import com.fasterxml.jackson.core.JsonFactory
 import com.fasterxml.jackson.core.JsonGenerator
@@ -17,6 +17,7 @@ import net.corda.core.utilities.Emoji
 import net.corda.jackson.JacksonSupport
 import net.corda.jackson.StringToMethodCallParser
 import net.corda.node.internal.Node
+import net.corda.node.printBasicNodeInfo
 import net.corda.node.services.statemachine.FlowStateMachineImpl
 import net.corda.node.utilities.ANSIProgressRenderer
 import net.corda.nodeapi.ArtemisMessagingComponent
@@ -26,8 +27,14 @@ import org.crsh.command.InvocationContext
 import org.crsh.console.jline.JLineProcessor
 import org.crsh.console.jline.TerminalFactory
 import org.crsh.console.jline.console.ConsoleReader
+import org.crsh.lang.impl.java.JavaLanguage
+import org.crsh.plugin.CRaSHPlugin
+import org.crsh.plugin.PluginContext
+import org.crsh.plugin.PluginLifeCycle
+import org.crsh.plugin.ServiceLoaderDiscovery
+import org.crsh.shell.Shell
 import org.crsh.shell.ShellFactory
-import org.crsh.standalone.Bootstrap
+import org.crsh.shell.impl.command.ExternalResolver
 import org.crsh.text.Color
 import org.crsh.text.RenderPrintWriter
 import org.crsh.util.InterruptHandler
@@ -53,7 +60,6 @@ import kotlin.concurrent.thread
 
 // TODO: Add command history.
 // TODO: Command completion.
-// TODO: Find a way to inject this directly into CRaSH as a command, without needing JIT source compilation.
 // TODO: Do something sensible with commands that return a future.
 // TODO: Configure default renderers, send objects down the pipeline, add commands to do json/xml/yaml outputs.
 // TODO: Add a command to view last N lines/tail/control log4j2 loggers.
@@ -76,25 +82,6 @@ object InteractiveShell {
 
         Logger.getLogger("").level = Level.OFF   // TODO: Is this really needed?
 
-        val classpathDriver = ClassPathMountFactory(Thread.currentThread().contextClassLoader)
-        val fileDriver = FileMountFactory(Utils.getCurrentDirectory())
-
-        val extraCommandsPath = (dir / "shell-commands").toAbsolutePath().createDirectories()
-        val commandsFS = FS.Builder()
-                .register("file", fileDriver)
-                .mount("file:" + extraCommandsPath)
-                .register("classpath", classpathDriver)
-                .mount("classpath:/net/corda/node/shell/")
-                .mount("classpath:/crash/commands/")
-                .build()
-        // TODO: Re-point to our own conf path.
-        val confFS = FS.Builder()
-                .register("classpath", classpathDriver)
-                .mount("classpath:/crash")
-                .build()
-
-        val bootstrap = Bootstrap(Thread.currentThread().contextClassLoader, confFS, commandsFS)
-
         val config = Properties()
         if (runSSH) {
             // TODO: Finish and enable SSH access.
@@ -117,16 +104,9 @@ object InteractiveShell {
             config["crash.auth.simple.password"] = "admin"
         }
 
-        bootstrap.config = config
-        bootstrap.setAttributes(mapOf(
-                "node" to node,
-                "services" to node.services,
-                "ops" to node.rpcOps,
-                "mapper" to yamlInputMapper
-        ))
-        bootstrap.bootstrap()
-
-        // TODO: Automatically set up the JDBC sub-command with a connection to the database.
+        ExternalResolver.INSTANCE.addCommand("run", "Runs a method from the CordaRPCOps interface on the node.", RunShellCommand::class.java)
+        ExternalResolver.INSTANCE.addCommand("flow", "Start a (work)flow on the node. This is how you can change the ledger.", FlowShellCommand::class.java)
+        val shell = ShellLifecycle(dir).start(config)
 
         if (runSSH) {
             // printBasicNodeInfo("SSH server listening on address", node.configuration.sshdAddress.toString())
@@ -135,7 +115,7 @@ object InteractiveShell {
         // Possibly bring up a local shell in the launching terminal window, unless it's disabled.
         if (!runLocalShell)
             return
-        val shell = bootstrap.context.getPlugin(ShellFactory::class.java).create(null)
+        // TODO: Automatically set up the JDBC sub-command with a connection to the database.
         val terminal = TerminalFactory.create()
         val consoleReader = ConsoleReader("Corda", FileInputStream(FileDescriptor.`in`), System.out, terminal)
         val jlineProcessor = JLineProcessor(terminal.isAnsiSupported, shell, consoleReader, System.out)
@@ -155,6 +135,47 @@ object InteractiveShell {
         }
     }
 
+    class ShellLifecycle(val dir: Path) : PluginLifeCycle() {
+        fun start(config: Properties): Shell {
+            val classLoader = this.javaClass.classLoader
+            val classpathDriver = ClassPathMountFactory(classLoader)
+            val fileDriver = FileMountFactory(Utils.getCurrentDirectory())
+
+            val extraCommandsPath = (dir / "shell-commands").toAbsolutePath().createDirectories()
+            val commandsFS = FS.Builder()
+                    .register("file", fileDriver)
+                    .mount("file:" + extraCommandsPath)
+                    .register("classpath", classpathDriver)
+                    .mount("classpath:/net/corda/node/shell/")
+                    .mount("classpath:/crash/commands/")
+                    .build()
+            val confFS = FS.Builder()
+                    .register("classpath", classpathDriver)
+                    .mount("classpath:/crash")
+                    .build()
+
+            val discovery = object : ServiceLoaderDiscovery(classLoader) {
+                override fun getPlugins(): Iterable<CRaSHPlugin<*>> {
+                    // Don't use the Java language plugin (we may not have tools.jar available at runtime), this
+                    // will cause any commands using JIT Java compilation to be suppressed. In CRaSH upstream that
+                    // is only the 'jmx' command.
+                    return super.getPlugins().filterNot { it is JavaLanguage }
+                }
+            }
+            val attributes = mapOf(
+                    "node" to node,
+                    "services" to node.services,
+                    "ops" to node.rpcOps,
+                    "mapper" to yamlInputMapper
+            )
+            val context = PluginContext(discovery, attributes, commandsFS, confFS, classLoader)
+            context.refresh()
+            this.config = config
+            start(context)
+            return context.getPlugin(ShellFactory::class.java).create(null)
+        }
+    }
+
     private val yamlInputMapper: ObjectMapper by lazy {
         // Return a standard Corda Jackson object mapper, configured to use YAML by default and with extra
         // serializers.

node/src/main/kotlin/net/corda/node/shell/InteractiveShellCommand.kt

@@ -1,4 +1,4 @@
-package net.corda.node
+package net.corda.node.shell
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import net.corda.core.messaging.CordaRPCOps

node/src/test/kotlin/net/corda/node/InteractiveShellTest.kt

@@ -14,6 +14,7 @@ import net.corda.core.utilities.DUMMY_PUBKEY_1
 import net.corda.core.utilities.UntrustworthyData
 import net.corda.jackson.JacksonSupport
 import net.corda.node.services.identity.InMemoryIdentityService
+import net.corda.node.shell.InteractiveShell
 import org.junit.Test
 import org.slf4j.Logger
 import java.util.*