CORDA-1456: Change output format in the shell (#4318)

* [CORDA-1456] Add support and switch for JSON/YAML output format

docs/source/changelog.rst

@@ -310,6 +310,8 @@ Unreleased
 * Finance CorDapp is now build as a sealed and signed JAR file.
   Custom classes can no longer be placed in the packages defined in Finance Cordapp or access it's non-public members.
 
+* The format of the shell commands' output can now be customized via the node shell, using the ``output-format`` command.
+
 Version 3.3
 -----------
 

docs/source/shell.rst

@@ -206,6 +206,18 @@ You can shut the node down via shell:
 * ``gracefulShutdown`` will put node into draining mode, and shut down when there are no flows running
 * ``shutdown`` will shut the node down immediately
 
+Output Formats
+**********************
+
+You can choose the format in which the output of the commands will be shown.
+
+To see what is the format that's currently used, you can type ``output-format get``.
+
+To update the format, you can type ``output-format set json``.
+
+The currently supported formats are ``json``, ``yaml``. The default format is ``yaml``.
+
+
 Flow commands
 *************
 

tools/shell/src/main/java/net/corda/tools/shell/OutputFormatCommand.java

@@ -0,0 +1,59 @@
+package net.corda.tools.shell;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableBiMap;
+import net.corda.tools.shell.InteractiveShell.OutputFormat;
+import org.crsh.cli.Argument;
+import org.crsh.cli.Command;
+import org.crsh.cli.Man;
+import org.crsh.cli.Usage;
+import org.crsh.command.InvocationContext;
+import org.crsh.command.ScriptException;
+import org.crsh.text.RenderPrintWriter;
+
+import java.util.Map;
+
+@Man("Allows you to see and update the format that's currently used for the commands' output.")
+public class OutputFormatCommand extends InteractiveShellCommand {
+
+    public OutputFormatCommand() {}
+
+    @VisibleForTesting
+    OutputFormatCommand(final RenderPrintWriter printWriter) {
+        this.out = printWriter;
+    }
+
+    private static final BiMap<String, OutputFormat> OUTPUT_FORMAT_MAPPING = ImmutableBiMap.of(
+            "json", OutputFormat.JSON,
+            "yaml", OutputFormat.YAML
+    );
+
+    @Command
+    @Man("Sets the output format of the commands.")
+    @Usage("sets the output format of the commands.")
+    public void set(InvocationContext<Map> context,
+                    @Usage("The format of the commands output. Supported values: json, yaml.") @Argument String format) {
+        OutputFormat outputFormat = parseFormat(format);
+
+        InteractiveShell.setOutputFormat(outputFormat);
+    }
+
+    @Command
+    @Man("Shows the output format of the commands.")
+    @Usage("shows the output format of the commands.")
+    public void get(InvocationContext<Map> context) {
+        OutputFormat outputFormat = InteractiveShell.getOutputFormat();
+        final String format = OUTPUT_FORMAT_MAPPING.inverse().get(outputFormat);
+
+        out.println(format);
+    }
+
+    private OutputFormat parseFormat(String format) {
+        if (!OUTPUT_FORMAT_MAPPING.containsKey(format)) {
+            throw new ScriptException("The provided format is not supported: " + format);
+        }
+
+        return OUTPUT_FORMAT_MAPPING.get(format);
+    }
+}

tools/shell/src/main/java/net/corda/tools/shell/RunShellCommand.java

@@ -1,5 +1,6 @@
 package net.corda.tools.shell;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import net.corda.client.jackson.StringToMethodCallParser;
@@ -36,7 +37,8 @@ public class RunShellCommand extends InteractiveShellCommand {
                     "consulting the developer guide at https://docs.corda.net/api/kotlin/corda/net.corda.core.messaging/-corda-r-p-c-ops/index.html"
     )
     @Usage("runs a method from the CordaRPCOps interface on the node.")
-    public Object main(InvocationContext<Map> context, @Usage("The command to run") @Argument(unquote = false) List<String> command) {
+    public Object main(InvocationContext<Map> context,
+                       @Usage("The command to run") @Argument(unquote = false) List<String> command) {
         logger.info("Executing command \"run {}\",", (command != null) ? command.stream().collect(joining(" ")) : "<no arguments>");
         StringToMethodCallParser<CordaRPCOps> parser = new StringToMethodCallParser<>(CordaRPCOps.class, objectMapper(InteractiveShell.getCordappsClassloader()));
 
@@ -44,6 +46,7 @@ public class RunShellCommand extends InteractiveShellCommand {
             emitHelp(context, parser);
             return null;
         }
+
         return InteractiveShell.runRPCFromString(command, out, context, ops(), objectMapper(InteractiveShell.getCordappsClassloader()));
     }
 

tools/shell/src/main/kotlin/net/corda/tools/shell/InteractiveShell.kt

@@ -1,9 +1,11 @@
 package net.corda.tools.shell
 
+import com.fasterxml.jackson.core.JsonFactory
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.databind.SerializationFeature
 import com.fasterxml.jackson.databind.module.SimpleModule
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator
 import net.corda.client.jackson.JacksonSupport
 import net.corda.client.jackson.StringToMethodCallParser
 import net.corda.client.rpc.CordaRPCClientConfiguration
@@ -61,7 +63,7 @@ import kotlin.concurrent.thread
 // TODO: Add command history.
 // TODO: Command completion.
 // TODO: Do something sensible with commands that return a future.
-// TODO: Configure default renderers, send objects down the pipeline, add commands to do json/xml/yaml outputs.
+// TODO: Configure default renderers, send objects down the pipeline, add support for xml output format.
 // TODO: Add a command to view last N lines/tail/control log4j2 loggers.
 // TODO: Review or fix the JVM commands which have bitrotted and some are useless.
 // TODO: Get rid of the 'java' command, it's kind of worthless.
@@ -82,6 +84,11 @@ object InteractiveShell {
     @JvmStatic
     fun getCordappsClassloader() = classLoader
 
+    enum class OutputFormat {
+        JSON,
+        YAML
+    }
+
     /**
      * Starts an interactive shell connected to the local terminal. This shell gives administrator access to the node
      * internals.
@@ -117,6 +124,7 @@ object InteractiveShell {
             }
         }
 
+        ExternalResolver.INSTANCE.addCommand("output-format", "Commands to inspect and update the output format.", OutputFormatCommand::class.java)
         ExternalResolver.INSTANCE.addCommand("run", "Runs a method from the CordaRPCOps interface on the node.", RunShellCommand::class.java)
         ExternalResolver.INSTANCE.addCommand("flow", "Commands to work with flows. Flows are how you can change the ledger.", FlowShellCommand::class.java)
         ExternalResolver.INSTANCE.addCommand("start", "An alias for 'flow start'", StartShellCommand::class.java)
@@ -191,6 +199,16 @@ object InteractiveShell {
         throw e.cause ?: e
     }
 
+    @JvmStatic
+    fun setOutputFormat(outputFormat: OutputFormat) {
+        this.outputFormat = outputFormat
+    }
+
+    @JvmStatic
+    fun getOutputFormat(): OutputFormat {
+        return outputFormat
+    }
+
     fun createYamlInputMapper(rpcOps: CordaRPCOps): ObjectMapper {
         // Return a standard Corda Jackson object mapper, configured to use YAML by default and with extra
         // serializers.
@@ -203,8 +221,13 @@ object InteractiveShell {
         }
     }
 
-    private fun createOutputMapper(): ObjectMapper {
-        return JacksonSupport.createNonRpcMapper().apply {
+    private fun createOutputMapper(outputFormat: OutputFormat): ObjectMapper {
+        val factory = when(outputFormat) {
+            OutputFormat.JSON -> JsonFactory()
+            OutputFormat.YAML -> YAMLFactory().disable(YAMLGenerator.Feature.WRITE_DOC_START_MARKER)
+        }
+
+        return JacksonSupport.createNonRpcMapper(factory).apply {
             // Register serializers for stateful objects from libraries that are special to the RPC system and don't
             // make sense to print out to the screen. For classes we own, annotations can be used instead.
             val rpcModule = SimpleModule().apply {
@@ -218,8 +241,8 @@ object InteractiveShell {
         }
     }
 
-    // TODO: This should become the default renderer rather than something used specifically by commands.
-    private val outputMapper by lazy { createOutputMapper() }
+    // TODO: A default renderer could be used, instead of an object mapper. See: http://www.crashub.org/1.3/reference.html#_renderers
+    private var outputFormat = OutputFormat.YAML
 
     @VisibleForTesting
     lateinit var latch: CountDownLatch
@@ -236,7 +259,7 @@ object InteractiveShell {
                               output: RenderPrintWriter,
                               rpcOps: CordaRPCOps,
                               ansiProgressRenderer: ANSIProgressRenderer,
-                              om: ObjectMapper = outputMapper) {
+                              inputObjectMapper: ObjectMapper = createYamlInputMapper(rpcOps)) {
         val matches = try {
             rpcOps.registeredFlows().filter { nameFragment in it }
         } catch (e: PermissionException) {
@@ -261,7 +284,7 @@ object InteractiveShell {
         try {
             // Show the progress tracker on the console until the flow completes or is interrupted with a
             // Ctrl-C keypress.
-            val stateObservable = runFlowFromString({ clazz, args -> rpcOps.startTrackedFlowDynamic(clazz, *args) }, inputData, flowClazz, om)
+            val stateObservable = runFlowFromString({ clazz, args -> rpcOps.startTrackedFlowDynamic(clazz, *args) }, inputData, flowClazz, inputObjectMapper)
 
             latch = CountDownLatch(1)
             ansiProgressRenderer.render(stateObservable, latch::countDown)
@@ -415,7 +438,8 @@ object InteractiveShell {
     }
 
     @JvmStatic
-    fun runRPCFromString(input: List<String>, out: RenderPrintWriter, context: InvocationContext<out Any>, cordaRPCOps: CordaRPCOps, om: ObjectMapper): Any? {
+    fun runRPCFromString(input: List<String>, out: RenderPrintWriter, context: InvocationContext<out Any>, cordaRPCOps: CordaRPCOps,
+                         inputObjectMapper: ObjectMapper): Any? {
         val cmd = input.joinToString(" ").trim { it <= ' ' }
         if (cmd.startsWith("startflow", ignoreCase = true)) {
             // The flow command provides better support and startFlow requires special handling anyway due to
@@ -430,11 +454,11 @@ object InteractiveShell {
         var result: Any? = null
         try {
             InputStreamSerializer.invokeContext = context
-            val parser = StringToMethodCallParser(CordaRPCOps::class.java, om)
+            val parser = StringToMethodCallParser(CordaRPCOps::class.java, inputObjectMapper)
             val call = parser.parse(cordaRPCOps, cmd)
             result = call.call()
             if (result != null && result !== kotlin.Unit && result !is Void) {
-                result = printAndFollowRPCResponse(result, out)
+                result = printAndFollowRPCResponse(result, out, outputFormat)
             }
             if (result is Future<*>) {
                 if (!result.isDone) {
@@ -530,19 +554,11 @@ object InteractiveShell {
         }
     }
 
-    private fun printAndFollowRPCResponse(response: Any?, out: PrintWriter): CordaFuture<Unit> {
+    private fun printAndFollowRPCResponse(response: Any?, out: PrintWriter, outputFormat: OutputFormat): CordaFuture<Unit> {
+        val outputMapper = createOutputMapper(outputFormat)
 
         val mapElement: (Any?) -> String = { element -> outputMapper.writerWithDefaultPrettyPrinter().writeValueAsString(element) }
-        val mappingFunction: (Any?) -> String = { value ->
-            if (value is Collection<*>) {
-                value.joinToString(",${System.lineSeparator()}  ", "[${System.lineSeparator()}  ", "${System.lineSeparator()}]") { element ->
-                    mapElement(element)
-                }
-            } else {
-                mapElement(value)
-            }
-        }
-        return maybeFollow(response, mappingFunction, out)
+        return maybeFollow(response, mapElement, out)
     }
 
     private class PrintingSubscriber(private val printerFun: (Any?) -> String, private val toStream: PrintWriter) : Subscriber<Any>() {

tools/shell/src/test/java/net/corda/tools/shell/OutputFormatCommandTest.java

@@ -0,0 +1,52 @@
+package net.corda.tools.shell;
+
+import org.crsh.command.InvocationContext;
+import org.crsh.command.ScriptException;
+import org.crsh.text.RenderPrintWriter;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+public class OutputFormatCommandTest {
+
+    private InvocationContext mockInvocationContext;
+    private RenderPrintWriter printWriter;
+
+    private OutputFormatCommand outputFormatCommand;
+
+    private static final String JSON_FORMAT_STRING = "json";
+    private static final String YAML_FORMAT_STRING = "yaml";
+
+    @Before
+    public void setup() {
+        mockInvocationContext = mock(InvocationContext.class);
+        printWriter = mock(RenderPrintWriter.class);
+
+        outputFormatCommand = new OutputFormatCommand(printWriter);
+    }
+
+    @Test
+    public void testValidUpdateToJson() {
+        outputFormatCommand.set(mockInvocationContext, JSON_FORMAT_STRING);
+        outputFormatCommand.get(mockInvocationContext);
+
+        verify(printWriter).println(JSON_FORMAT_STRING);
+    }
+
+    @Test
+    public void testValidUpdateToYaml() {
+        outputFormatCommand.set(mockInvocationContext, YAML_FORMAT_STRING);
+        outputFormatCommand.get(mockInvocationContext);
+
+        verify(printWriter).println(YAML_FORMAT_STRING);
+    }
+
+    @Test
+    public void testInvalidUpdate() {
+        assertThatExceptionOfType(ScriptException.class).isThrownBy(() -> outputFormatCommand.set(mockInvocationContext, "some-invalid-format"))
+                .withMessage("The provided format is not supported: some-invalid-format");
+    }
+}

tools/shell/src/test/kotlin/net/corda/tools/shell/InteractiveShellTest.kt

@@ -1,21 +1,36 @@
 package net.corda.tools.shell
 
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.type.TypeFactory
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import com.nhaarman.mockito_kotlin.mock
+import com.nhaarman.mockito_kotlin.verify
+import com.nhaarman.mockito_kotlin.whenever
 import net.corda.client.jackson.JacksonSupport
 import net.corda.client.jackson.internal.ToStringSerialize
 import net.corda.core.contracts.Amount
 import net.corda.core.crypto.SecureHash
+import net.corda.core.crypto.generateKeyPair
 import net.corda.core.flows.FlowLogic
 import net.corda.core.flows.StateMachineRunId
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.internal.concurrent.openFuture
+import net.corda.core.messaging.CordaRPCOps
 import net.corda.core.messaging.FlowProgressHandleImpl
+import net.corda.core.node.NodeInfo
+import net.corda.core.utilities.NetworkHostAndPort
 import net.corda.core.utilities.ProgressTracker
 import net.corda.node.services.identity.InMemoryIdentityService
+import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.BOB_NAME
 import net.corda.testing.core.TestIdentity
+import net.corda.testing.core.getTestPartyAndCertificate
 import net.corda.testing.internal.DEV_ROOT_CA
+import org.crsh.command.InvocationContext
+import org.crsh.text.RenderPrintWriter
+import org.junit.Before
 import org.junit.Test
 import rx.Observable
 import java.util.*
@@ -23,8 +38,75 @@ import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 
 class InteractiveShellTest {
+    lateinit var inputObjectMapper: ObjectMapper
+    lateinit var cordaRpcOps: CordaRPCOps
+    lateinit var invocationContext: InvocationContext<Map<Any, Any>>
+    lateinit var printWriter: RenderPrintWriter
+
+    @Before
+    fun setup() {
+        inputObjectMapper = objectMapperWithClassLoader(InteractiveShell.getCordappsClassloader())
+        cordaRpcOps = mock()
+        invocationContext = mock()
+        printWriter = mock()
+    }
+
     companion object {
         private val megaCorp = TestIdentity(CordaX500Name("MegaCorp", "London", "GB"))
+
+        private val ALICE = getTestPartyAndCertificate(ALICE_NAME, generateKeyPair().public)
+        private val BOB = getTestPartyAndCertificate(BOB_NAME, generateKeyPair().public)
+        private val ALICE_NODE_INFO = NodeInfo(listOf(NetworkHostAndPort("localhost", 8080)), listOf(ALICE), 1, 1)
+        private val BOB_NODE_INFO = NodeInfo(listOf(NetworkHostAndPort("localhost", 80)), listOf(BOB), 1, 1)
+        private val NODE_INFO_JSON_PAYLOAD =
+                """
+                {
+                  "addresses" : [ "localhost:8080" ],
+                  "legalIdentitiesAndCerts" : [ "O=Alice Corp, L=Madrid, C=ES" ],
+                  "platformVersion" : 1,
+                  "serial" : 1
+                }
+                """.trimIndent()
+        private val NODE_INFO_YAML_PAYLOAD =
+                """
+                    addresses:
+                    - "localhost:8080"
+                    legalIdentitiesAndCerts:
+                    - "O=Alice Corp, L=Madrid, C=ES"
+                    platformVersion: 1
+                    serial: 1
+
+                """.trimIndent()
+        private val NETWORK_MAP_JSON_PAYLOAD =
+                """
+                    [ {
+                      "addresses" : [ "localhost:8080" ],
+                      "legalIdentitiesAndCerts" : [ "O=Alice Corp, L=Madrid, C=ES" ],
+                      "platformVersion" : 1,
+                      "serial" : 1
+                    }, {
+                      "addresses" : [ "localhost:80" ],
+                      "legalIdentitiesAndCerts" : [ "O=Bob Plc, L=Rome, C=IT" ],
+                      "platformVersion" : 1,
+                      "serial" : 1
+                    } ]
+                """.trimIndent()
+        private val NETWORK_MAP_YAML_PAYLOAD =
+                """
+                    - addresses:
+                      - "localhost:8080"
+                      legalIdentitiesAndCerts:
+                      - "O=Alice Corp, L=Madrid, C=ES"
+                      platformVersion: 1
+                      serial: 1
+                    - addresses:
+                      - "localhost:80"
+                      legalIdentitiesAndCerts:
+                      - "O=Bob Plc, L=Rome, C=IT"
+                      platformVersion: 1
+                      serial: 1
+
+                """.trimIndent()
     }
 
     @Suppress("UNUSED")
@@ -58,6 +140,14 @@ class InteractiveShellTest {
         assertEquals(expected, output!!, input)
     }
 
+    private fun objectMapperWithClassLoader(classLoader: ClassLoader?): ObjectMapper {
+        val objectMapper = ObjectMapper()
+        val tf = TypeFactory.defaultInstance().withClassLoader(classLoader)
+        objectMapper.typeFactory = tf
+
+        return objectMapper
+    }
+    
     @Test
     fun flowStartSimple() {
         check("a: Hi there", "Hi there")
@@ -125,6 +215,35 @@ class InteractiveShellTest {
     @Test
     fun party() = check("party: \"${megaCorp.name}\"", megaCorp.name.toString())
 
+    @Test
+    fun runRpcFromStringWithCustomTypeResult() {
+        val command = listOf("nodeInfo")
+        whenever(cordaRpcOps.nodeInfo()).thenReturn(ALICE_NODE_INFO)
+
+        InteractiveShell.setOutputFormat(InteractiveShell.OutputFormat.YAML)
+        InteractiveShell.runRPCFromString(command, printWriter, invocationContext, cordaRpcOps, inputObjectMapper)
+        verify(printWriter).println(NODE_INFO_YAML_PAYLOAD)
+
+
+        InteractiveShell.setOutputFormat(InteractiveShell.OutputFormat.JSON)
+        InteractiveShell.runRPCFromString(command, printWriter, invocationContext, cordaRpcOps, inputObjectMapper)
+        verify(printWriter).println(NODE_INFO_JSON_PAYLOAD)
+    }
+
+    @Test
+    fun runRpcFromStringWithCollectionsResult() {
+        val command = listOf("networkMapSnapshot")
+        whenever(cordaRpcOps.networkMapSnapshot()).thenReturn(listOf(ALICE_NODE_INFO, BOB_NODE_INFO))
+
+        InteractiveShell.setOutputFormat(InteractiveShell.OutputFormat.YAML)
+        InteractiveShell.runRPCFromString(command, printWriter, invocationContext, cordaRpcOps, inputObjectMapper)
+        verify(printWriter).println(NETWORK_MAP_YAML_PAYLOAD)
+
+        InteractiveShell.setOutputFormat(InteractiveShell.OutputFormat.JSON)
+        InteractiveShell.runRPCFromString(command, printWriter, invocationContext, cordaRpcOps, inputObjectMapper)
+        verify(printWriter).println(NETWORK_MAP_JSON_PAYLOAD)
+    }
+
     @ToStringSerialize
     data class UserValue(@JsonProperty("label") val label: String) {
         override fun toString() = label