diff --git a/.idea/compiler.xml b/.idea/compiler.xml index a28d15edcc..e2620b8b2d 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -14,6 +14,8 @@ + + @@ -172,4 +174,4 @@ - \ No newline at end of file + diff --git a/build.gradle b/build.gradle index f8270f2808..fe6a9308eb 100644 --- a/build.gradle +++ b/build.gradle @@ -76,6 +76,7 @@ buildscript { ext.ghostdriver_version = '2.1.0' ext.eaagentloader_version = '1.0.3' ext.jsch_version = '0.1.54' + ext.commons_cli_version = '1.4' // Update 121 is required for ObjectInputFilter and at time of writing 131 was latest: ext.java8_minUpdateVersion = '131' diff --git a/experimental/blobinspector/build.gradle b/experimental/blobinspector/build.gradle new file mode 100644 index 0000000000..2862ff6fae --- /dev/null +++ b/experimental/blobinspector/build.gradle @@ -0,0 +1,52 @@ +apply plugin: 'java' +apply plugin: 'kotlin' +apply plugin: 'application' + +mainClassName = 'net.corda.blobinspector.MainKt' + +dependencies { + compile project(':core') + compile project(':node-api') + + compile "commons-cli:commons-cli:$commons_cli_version" + + testCompile project(':test-utils') + + testCompile "junit:junit:$junit_version" +} + +/** + * To run from within gradle use + * + * ./gradlew -PrunArgs=" " :experimental:blobinspector:run + * + * For example, to parse a file from the command line and print out the deserialized properties + * + * ./gradlew -PrunArgs="-f -d" :experimental:blobinspector:run + * + * at the command line. + */ +run { + if (project.hasProperty('runArgs')) { + args = [ project.findProperty('runArgs').toString().split(" ") ].flatten() + } + + if (System.properties.getProperty('consoleLogLevel') != null) { + logging.captureStandardOutput(LogLevel.valueOf(System.properties.getProperty('consoleLogLevel'))) + logging.captureStandardError(LogLevel.valueOf(System.properties.getProperty('consoleLogLevel'))) + systemProperty "consoleLogLevel", System.properties.getProperty('consoleLogLevel') + } +} + +/** + * Build a executable jar + */ +jar { + baseName 'blobinspector' + manifest { + attributes( + 'Automatic-Module-Name': 'net.corda.experimental.blobinspector', + 'Main-Class': 'net.corda.blobinspector.MainKt' + ) + } +} diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobInspector.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobInspector.kt new file mode 100644 index 0000000000..8a30c2319f --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobInspector.kt @@ -0,0 +1,399 @@ +package net.corda.blobinspector + +import net.corda.core.crypto.SecureHash +import net.corda.core.serialization.EncodingWhitelist +import net.corda.core.serialization.SerializationEncoding +import net.corda.core.utilities.ByteSequence +import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl +import net.corda.nodeapi.internal.serialization.amqp.* +import org.apache.qpid.proton.amqp.Binary +import org.apache.qpid.proton.amqp.DescribedType +import org.apache.qpid.proton.amqp.Symbol + +/** + * Print a string to the console only if the verbose config option is set. + */ +fun String.debug(config: Config) { + if (config.verbose) { + println(this) + } +} + +/** + * + */ +interface Stringify { + fun stringify(sb: IndentingStringBuilder) +} + +/** + * Makes classnames easier to read by stripping off the package names from the class and separating nested + * classes + * + * For example: + * + * net.corda.blobinspector.Class1 + * Class1 + * + * net.corda.blobinspector.Class1 + * Class1 + * + * net.corda.blobinspector.Class1> + * Class1 > + * + * net.corda.blobinspector.Class1> + * Class1 :: C > + */ +fun String.simplifyClass(): String { + + return if (this.endsWith('>')) { + val templateStart = this.indexOf('<') + val clazz = (this.substring(0, templateStart)) + val params = this.substring(templateStart+1, this.length-1).split(',').map { it.simplifyClass() }.joinToString() + + "${clazz.simplifyClass()} <$params>" + } + else { + substring(this.lastIndexOf('.') + 1).replace("$", " :: ") + } +} + +/** + * Represents the deserialized form of the property of an Object + * + * @param name + * @param type + */ +abstract class Property( + val name: String, + val type: String) : Stringify + +/** + * Derived class of [Property], represents properties of an object that are non compelex, such + * as any POD type or String + */ +class PrimProperty( + name: String, + type: String, + private val value: String) : Property(name, type) { + override fun toString(): String = "$name : $type : $value" + + override fun stringify(sb: IndentingStringBuilder) { + sb.appendln("$name : $type : $value") + } +} + +/** + * Derived class of [Property] that represents a binary blob. Specifically useful because printing + * a stream of bytes onto the screen isn't very use friendly + */ +class BinaryProperty( + name: String, + type: String, + val value: ByteArray) : Property(name, type) { + override fun toString(): String = "$name : $type : <<>>" + + override fun stringify(sb: IndentingStringBuilder) { + sb.appendln("$name : $type : <<>>") + } +} + +/** + * Derived class of [Property] that represent a list property. List could be either PoD types or + * composite types. + */ +class ListProperty( + name: String, + type: String, + private val values: MutableList = mutableListOf()) : Property(name, type) { + override fun stringify(sb: IndentingStringBuilder) { + sb.apply { + if (values.isEmpty()) { + appendln("$name : $type : [ << EMPTY LIST >> ]") + } else if (values.first() is Stringify) { + appendln("$name : $type : [") + values.forEach { + (it as Stringify).stringify(this) + } + appendln("]") + } else { + appendln("$name : $type : [") + values.forEach { + appendln(it.toString()) + } + appendln("]") + } + } + } +} + +class MapProperty( + name: String, + type: String, + private val map: MutableMap<*, *> +) : Property(name, type) { + override fun stringify(sb: IndentingStringBuilder) { + if (map.isEmpty()) { + sb.appendln("$name : $type : { << EMPTY MAP >> }") + return + } + + // TODO this will not produce pretty output + sb.apply { + appendln("$name : $type : {") + map.forEach { + try { + (it.key as Stringify).stringify(this) + } catch (e: ClassCastException) { + append (it.key.toString() + " : ") + } + try { + (it.value as Stringify).stringify(this) + } catch (e: ClassCastException) { + appendln("\"${it.value.toString()}\"") + } + } + appendln("}") + } + } +} + +/** + * Derived class of [Property] that represents class properties that are themselves instances of + * some complex type. + */ +class InstanceProperty( + name: String, + type: String, + val value: Instance) : Property(name, type) { + override fun stringify(sb: IndentingStringBuilder) { + sb.append("$name : ") + value.stringify(sb) + } +} + +/** + * Represents an instance of a composite type. + */ +class Instance( + val name: String, + val type: String, + val fields: MutableList = mutableListOf()) : Stringify { + override fun stringify(sb: IndentingStringBuilder) { + sb.apply { + appendln("${name.simplifyClass()} : {") + fields.forEach { + it.stringify(this) + } + appendln("}") + } + } +} + +/** + * + */ +fun inspectComposite( + config: Config, + typeMap: Map, + obj: DescribedType): Instance { + if (obj.described !is List<*>) throw MalformedBlob("") + + val name = (typeMap[obj.descriptor] as CompositeType).name + "composite: $name".debug(config) + + val inst = Instance( + typeMap[obj.descriptor]?.name ?: "", + typeMap[obj.descriptor]?.label ?: "") + + (typeMap[obj.descriptor] as CompositeType).fields.zip(obj.described as List<*>).forEach { + " field: ${it.first.name}".debug(config) + inst.fields.add( + if (it.second is DescribedType) { + " - is described".debug(config) + val d = inspectDescribed(config, typeMap, it.second as DescribedType) + + when (d) { + is Instance -> + InstanceProperty( + it.first.name, + it.first.type, + d) + is List<*> -> { + " - List".debug(config) + ListProperty( + it.first.name, + it.first.type, + d as MutableList) + } + is Map<*, *> -> { + MapProperty( + it.first.name, + it.first.type, + d as MutableMap<*, *>) + } + else -> { + " skip it".debug(config) + return@forEach + } + } + + } else { + " - is prim".debug(config) + when (it.first.type) { + // Note, as in the case of SHA256 we can treat particular binary types + // as different properties with a little coercion + "binary" -> { + if (name == "net.corda.core.crypto.SecureHash\$SHA256") { + PrimProperty( + it.first.name, + it.first.type, + SecureHash.SHA256((it.second as Binary).array).toString()) + } else { + BinaryProperty(it.first.name, it.first.type, (it.second as Binary).array) + } + } + else -> PrimProperty(it.first.name, it.first.type, it.second.toString()) + } + }) + } + + return inst +} + +fun inspectRestricted( + config: Config, + typeMap: Map, + obj: DescribedType): Any { + return when ((typeMap[obj.descriptor] as RestrictedType).source) { + "list" -> inspectRestrictedList(config, typeMap, obj) + "map" -> inspectRestrictedMap(config, typeMap, obj) + else -> throw NotImplementedError() + } +} + + +fun inspectRestrictedList( + config: Config, + typeMap: Map, + obj: DescribedType +) : List { + if (obj.described !is List<*>) throw MalformedBlob("") + + return mutableListOf().apply { + (obj.described as List<*>).forEach { + when (it) { + is DescribedType -> add(inspectDescribed(config, typeMap, it)) + is RestrictedType -> add(inspectRestricted(config, typeMap, it)) + else -> add (it.toString()) + } + } + } +} + +fun inspectRestrictedMap( + config: Config, + typeMap: Map, + obj: DescribedType +) : Map { + if (obj.described !is Map<*,*>) throw MalformedBlob("") + + return mutableMapOf().apply { + (obj.described as Map<*, *>).forEach { + val key = when (it.key) { + is DescribedType -> inspectDescribed(config, typeMap, it.key as DescribedType) + is RestrictedType -> inspectRestricted(config, typeMap, it.key as RestrictedType) + else -> it.key.toString() + } + + val value = when (it.value) { + is DescribedType -> inspectDescribed(config, typeMap, it.value as DescribedType) + is RestrictedType -> inspectRestricted(config, typeMap, it.value as RestrictedType) + else -> it.value.toString() + } + + this[key] = value + } + } +} + + +/** + * Every element of the blob stream will be a ProtonJ [DescribedType]. When inspecting the blob stream + * the two custom Corda types we're interested in are [CompositeType]'s, representing the instance of + * some object (class), and [RestrictedType]'s, representing containers and enumerations. + * + * @param config The configuration object that controls the behaviour of the BlobInspector + * @param typeMap + * @param obj + */ +fun inspectDescribed( + config: Config, + typeMap: Map, + obj: DescribedType): Any { + "${obj.descriptor} in typeMap? = ${obj.descriptor in typeMap}".debug(config) + + return when (typeMap[obj.descriptor]) { + is CompositeType -> { + "* It's composite".debug(config) + inspectComposite(config, typeMap, obj) + } + is RestrictedType -> { + "* It's restricted".debug(config) + inspectRestricted(config, typeMap, obj) + } + else -> { + "${typeMap[obj.descriptor]?.name} is neither Composite or Restricted".debug(config) + } + } + +} + +internal object NullEncodingWhitelist : EncodingWhitelist { + override fun acceptEncoding(encoding: SerializationEncoding) = false +} + +// TODO : Refactor to generically poerate on arbitrary blobs, not a single workflow +fun inspectBlob(config: Config, blob: ByteArray) { + val bytes = ByteSequence.of(blob) + + val headerSize = SerializationFactoryImpl.magicSize + + // TODO written to only understand one version, when we support multiple this will need to change + val headers = listOf(ByteSequence.of(amqpMagic.bytes)) + + val blobHeader = bytes.take(headerSize) + + if (blobHeader !in headers) { + throw MalformedBlob("Blob is not a Corda AMQP serialised object graph") + } + + + val e = DeserializationInput.getEnvelope(bytes, NullEncodingWhitelist) + + if (config.schema) { + println(e.schema) + } + + if (config.transforms) { + println(e.transformsSchema) + } + + val typeMap = e.schema.types.associateBy({ it.descriptor.name }, { it }) + + if (config.data) { + val inspected = inspectDescribed(config, typeMap, e.obj as DescribedType) + + println("\n${IndentingStringBuilder().apply { (inspected as Instance).stringify(this) }}") + + (inspected as Instance).fields.find { + it.type.startsWith("net.corda.core.serialization.SerializedBytes<") + }?.let { + "Found field of SerializedBytes".debug(config) + (it as InstanceProperty).value.fields.find { it.name == "bytes" }?.let { raw -> + inspectBlob(config, (raw as BinaryProperty).value) + } + } + } +} + diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobLoader.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobLoader.kt new file mode 100644 index 0000000000..a027249079 --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/BlobLoader.kt @@ -0,0 +1,40 @@ +package net.corda.blobinspector + +import java.io.File +import java.net.URL + +/** + * + */ +class FileBlobHandler(config_: Config) : BlobHandler(config_) { + private val path = File(URL((config_ as FileConfig).file).toURI()) + + override fun getBytes(): ByteArray { + return path.readBytes() + } +} + +/** + * + */ +class InMemoryBlobHandler(config_: Config) : BlobHandler(config_) { + private val localBytes = (config_ as InMemoryConfig).blob?.bytes ?: kotlin.ByteArray(0) + override fun getBytes(): ByteArray = localBytes +} + +/** + * + */ +abstract class BlobHandler (val config: Config) { + companion object { + fun make(config: Config) : BlobHandler { + return when (config.mode) { + Mode.file -> FileBlobHandler(config) + Mode.inMem -> InMemoryBlobHandler(config) + } + } + } + + abstract fun getBytes() : ByteArray +} + diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Config.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Config.kt new file mode 100644 index 0000000000..376331ec2b --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Config.kt @@ -0,0 +1,137 @@ +package net.corda.blobinspector + +import org.apache.commons.cli.CommandLine +import net.corda.core.serialization.SerializedBytes +import org.apache.commons.cli.Option +import org.apache.commons.cli.Options + +/** + * Enumeration of the modes in which the blob inspector can be run. + * + * @property make lambda function that takes no parameters and returns a specific instance of the configuration + * object for that mode. + * + * @property options A lambda function that takes no parameters and returns an [Options] instance that define + * the command line flags related to this mode. For example ``file`` mode would have an option to pass in + * the name of the file to read. + * + */ +enum class Mode( + val make : () -> Config, + val options : (Options) -> Unit +) { + file( + { + FileConfig(Mode.file) + }, + { o -> + o.apply{ + addOption( + Option ("f", "file", true, "path to file").apply { + isRequired = true + } + ) + } + } + ), + inMem( + { + InMemoryConfig(Mode.inMem) + }, + { + // The in memory only mode has no specific option assocaited with it as it's intended for + // testing purposes only within the unit test framework and not use on the command line + } + ) +} + +/** + * Configuration data class for the Blob Inspector. + * + * @property mode + */ +abstract class Config (val mode: Mode) { + var schema: Boolean = false + var transforms: Boolean = false + var data: Boolean = false + var verbose: Boolean = false + + abstract fun populateSpecific(cmdLine: CommandLine) + abstract fun withVerbose() : Config + + fun populate(cmdLine: CommandLine) { + schema = cmdLine.hasOption('s') + transforms = cmdLine.hasOption('t') + data = cmdLine.hasOption('d') + verbose = cmdLine.hasOption('v') + + populateSpecific(cmdLine) + } + + fun options() = Options().apply { + // install generic options + addOption(Option("s", "schema", false, "print the blob's schema").apply { + isRequired = false + }) + + addOption(Option("t", "transforms", false, "print the blob's transforms schema").apply { + isRequired = false + }) + + addOption(Option("d", "data", false, "Display the serialised data").apply { + isRequired = false + }) + + addOption(Option("v", "verbose", false, "Enable debug output").apply { + isRequired = false + }) + + // install the mode specific options + mode.options(this) + } +} + + +/** + * Configuration object when running in "File" mode, i.e. the object has been specified at + * the command line + */ +class FileConfig ( + mode: Mode +) : Config(mode) { + + var file: String = "unset" + + override fun populateSpecific(cmdLine : CommandLine) { + file = cmdLine.getParsedOptionValue("f") as String + } + + override fun withVerbose() : FileConfig { + return FileConfig(mode).apply { + this.schema = schema + this.transforms = transforms + this.data = data + this.verbose = true + } + } +} + + +/** + * Placeholder config objet used when running unit tests and the inspected blob is being fed in + * via some mechanism directly. Normally this will be the direct serialisation of an object in a unit + * test and then dumping that blob into the inspector for visual comparison of the output + */ +class InMemoryConfig ( + mode: Mode +) : Config(mode) { + var blob: SerializedBytes<*>? = null + + override fun populateSpecific(cmdLine: CommandLine) { + throw UnsupportedOperationException("In memory config is for testing only and cannot set specific flags") + } + + override fun withVerbose(): Config { + throw UnsupportedOperationException("In memory config is for testing headlessly, cannot be verbose") + } +} diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Errors.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Errors.kt new file mode 100644 index 0000000000..888ef1e302 --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Errors.kt @@ -0,0 +1,3 @@ +package net.corda.blobinspector + +class MalformedBlob(msg: String) : Exception(msg) diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/IndentingStringBuilder.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/IndentingStringBuilder.kt new file mode 100644 index 0000000000..48d81a8eb7 --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/IndentingStringBuilder.kt @@ -0,0 +1,45 @@ +package net.corda.blobinspector + +/** + * Wrapper around a [StringBuilder] that automates the indenting of lines as they're appended to facilitate + * pretty printing of deserialized blobs. + * + * @property sb The wrapped [StringBuilder] + * @property indenting Boolean flag that indicates weather we need to pad the start of whatever text + * currently being added to the string. + * @property indent How deeply the next line should be offset from the first column + */ +class IndentingStringBuilder(s : String = "", private val offset : Int = 4) { + private val sb = StringBuilder(s) + private var indenting = true + private var indent = 0 + + private fun wrap(ln: String, appender: (String) -> Unit) { + if ((ln.endsWith("}") || ln.endsWith("]")) && indent > 0 && ln.length == 1) { + indent -= offset + } + + appender(ln) + + if (ln.endsWith("{") || ln.endsWith("[")){ + indent += offset + } + } + + fun appendln(ln: String) { + wrap(ln) { s -> sb.appendln("${"".padStart(if (indenting) indent else 0, ' ')}$s") } + + indenting = true + } + + + fun append(ln: String) { + indenting = false + + wrap(ln) { s -> sb.append("${"".padStart(indent, ' ')}$s") } + } + + override fun toString(): String { + return sb.toString() + } +} \ No newline at end of file diff --git a/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt new file mode 100644 index 0000000000..0e13b9e087 --- /dev/null +++ b/experimental/blobinspector/src/main/kotlin/net/corda/blobinspector/Main.kt @@ -0,0 +1,81 @@ +package net.corda.blobinspector + +import org.apache.commons.cli.* +import java.lang.IllegalArgumentException + +/** + * Mode isn't a required property as we default it to [Mode.file] + */ +private fun modeOption() = Option("m", "mode", true, "mode, file is the default").apply { + isRequired = false +} + +/** + * + * Parse the command line arguments looking for the main mode into which the application is + * being put. Note, this defaults to [Mode.file] if not set meaning we will look for a file path + * being passed as a parameter and parse that file. + * + * @param args reflects the command line arguments + * + * @return An instantiated but unpopulated [Config] object instance suitable for the mode into + * which we've been placed. This Config object should be populated via [loadModeSpecificOptions] + */ +fun getMode(args: Array) : Config { + // For now we only care what mode we're being put in, we can build the rest of the args and parse them + // later + val options = Options().apply { + addOption(modeOption()) + } + + val cmd = try { + DefaultParser().parse(options, args, true) + } catch (e: org.apache.commons.cli.ParseException) { + println (e) + HelpFormatter().printHelp("blobinspector", options) + throw IllegalArgumentException("OH NO!!!") + } + + return try { + Mode.valueOf(cmd.getParsedOptionValue("m") as? String ?: "file") + } catch (e: IllegalArgumentException) { + Mode.file + }.make() +} + +/** + * + * @param config an instance of a [Config] specialisation suitable for the mode into which + * the application has been put. + * @param args The command line arguments + */ +fun loadModeSpecificOptions(config: Config, args: Array) { + config.apply { + // load that modes specific command line switches, needs to include the mode option + val modeSpecificOptions = config.options().apply { + addOption(modeOption()) + } + + populate (try { + DefaultParser().parse(modeSpecificOptions, args, false) + } catch (e: org.apache.commons.cli.ParseException) { + println ("Error: ${e.message}") + HelpFormatter().printHelp("blobinspector", modeSpecificOptions) + System.exit(1) + return + }) + } +} + +/** + * Executable entry point + */ +fun main(args: Array) { + println ("<<< WARNING: this tool is experimental and under active development >>>") + getMode(args).let { mode -> + loadModeSpecificOptions(mode, args) + BlobHandler.make(mode) + }.apply { + inspectBlob(config, getBytes()) + } +} diff --git a/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/FileParseTests.kt b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/FileParseTests.kt new file mode 100644 index 0000000000..a018baaf49 --- /dev/null +++ b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/FileParseTests.kt @@ -0,0 +1,84 @@ +package net.corda.blobinspector + +import java.net.URI + +import org.junit.Test +import net.corda.testing.common.internal.ProjectStructure.projectRootDir + + +class FileParseTests { + @Suppress("UNUSED") + var localPath : URI = projectRootDir.toUri().resolve( + "tools/blobinspector/src/test/resources/net/corda/blobinspector") + + fun setupArgsWithFile(path: String) = Array(5) { + when (it) { + 0 -> "-m" + 1 -> "file" + 2 -> "-f" + 3 -> path + 4 -> "-d" + else -> "error" + } + } + + private val filesToTest = listOf ( + "FileParseTests.1Int", + "FileParseTests.2Int", + "FileParseTests.3Int", + "FileParseTests.1String", + "FileParseTests.1Composite", + "FileParseTests.2Composite", + "FileParseTests.IntList", + "FileParseTests.StringList", + "FileParseTests.MapIntString", + "FileParseTests.MapIntClass" + ) + + fun testFile(file : String) { + val path = FileParseTests::class.java.getResource(file) + val args = setupArgsWithFile(path.toString()) + + val handler = getMode(args).let { mode -> + loadModeSpecificOptions(mode, args) + BlobHandler.make(mode) + } + + inspectBlob(handler.config, handler.getBytes()) + } + + @Test + fun simpleFiles() { + filesToTest.forEach { testFile(it) } + } + + @Test + fun specificTest() { + testFile(filesToTest[4]) + testFile(filesToTest[5]) + testFile(filesToTest[6]) + } + + @Test + fun networkParams() { + val file = "networkParams" + val path = FileParseTests::class.java.getResource(file) + val verbose = false + + val args = verbose.let { + if (it) + Array(4) { when (it) { 0 -> "-f" ; 1 -> path.toString(); 2 -> "-d"; 3 -> "-vs"; else -> "error" } } + else + Array(3) { when (it) { 0 -> "-f" ; 1 -> path.toString(); 2 -> "-d"; else -> "error" } } + } + + val handler = getMode(args).let { mode -> + loadModeSpecificOptions(mode, args) + BlobHandler.make(mode) + } + + inspectBlob(handler.config, handler.getBytes()) + + } + +} diff --git a/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/InMemoryTests.kt b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/InMemoryTests.kt new file mode 100644 index 0000000000..d8451df92d --- /dev/null +++ b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/InMemoryTests.kt @@ -0,0 +1,89 @@ +package net.corda.blobinspector + +import net.corda.core.serialization.SerializedBytes +import net.corda.nodeapi.internal.serialization.AllWhitelist +import net.corda.nodeapi.internal.serialization.amqp.SerializationOutput +import net.corda.nodeapi.internal.serialization.amqp.SerializerFactory +import org.junit.Test + + +class InMemoryTests { + private val factory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()) + + private fun inspect (b: SerializedBytes<*>) { + BlobHandler.make( + InMemoryConfig(Mode.inMem).apply { blob = b; data = true} + ).apply { + inspectBlob(config, getBytes()) + } + } + + @Test + fun test1() { + data class C (val a: Int, val b: Long, val c: String) + inspect (SerializationOutput(factory).serialize(C(100, 567L, "this is a test"))) + } + + @Test + fun test2() { + data class C (val i: Int, val c: C?) + inspect (SerializationOutput(factory).serialize(C(1, C(2, C(3, C(4, null)))))) + } + + @Test + fun test3() { + data class C (val a: IntArray, val b: Array) + + val a = IntArray(10) { i -> i } + val c = C(a, arrayOf("aaa", "bbb", "ccc")) + + inspect (SerializationOutput(factory).serialize(c)) + } + + @Test + fun test4() { + data class Elem(val e1: Long, val e2: String) + data class Wrapper (val name: String, val elementes: List) + + inspect (SerializationOutput(factory).serialize( + Wrapper("Outer Class", + listOf( + Elem(1L, "First element"), + Elem(2L, "Second element"), + Elem(3L, "Third element") + )))) + } + + @Test + fun test4b() { + data class Elem(val e1: Long, val e2: String) + data class Wrapper (val name: String, val elementes: List>) + + inspect (SerializationOutput(factory).serialize( + Wrapper("Outer Class", + listOf ( + listOf( + Elem(1L, "First element"), + Elem(2L, "Second element"), + Elem(3L, "Third element") + ), + listOf( + Elem(4L, "Fourth element"), + Elem(5L, "Fifth element"), + Elem(6L, "Sixth element") + ) + )))) + } + + @Test + fun test5() { + data class C (val a: Map) + + inspect (SerializationOutput(factory).serialize( + C(mapOf( + "a" to "a a a", + "b" to "b b b", + "c" to "c c c")) + )) + } +} \ No newline at end of file diff --git a/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/ModeParse.kt b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/ModeParse.kt new file mode 100644 index 0000000000..9b69363386 --- /dev/null +++ b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/ModeParse.kt @@ -0,0 +1,77 @@ +package net.corda.blobinspector + +import org.junit.Test +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import kotlin.test.assertFalse + +class ModeParse { + @Test + fun fileIsSetToFile() { + val opts1 = Array(2) { + when (it) { + 0 -> "-m" + 1 -> "file" + else -> "error" + } + } + + assertEquals(Mode.file, getMode(opts1).mode) + } + + @Test + fun nothingIsSetToFile() { + val opts1 = Array(0) { "" } + + assertEquals(Mode.file, getMode(opts1).mode) + } + + @Test + fun filePathIsSet() { + val opts1 = Array(4) { + when (it) { + 0 -> "-m" + 1 -> "file" + 2 -> "-f" + 3 -> "path/to/file" + else -> "error" + } + } + + val config = getMode(opts1) + assertTrue (config is FileConfig) + assertEquals(Mode.file, config.mode) + assertEquals("unset", (config as FileConfig).file) + + loadModeSpecificOptions(config, opts1) + + assertEquals("path/to/file", config.file) + } + + @Test + fun schemaIsSet() { + Array(2) { when (it) { 0 -> "-f"; 1 -> "path/to/file"; else -> "error" } }.let { options -> + getMode(options).apply { + loadModeSpecificOptions(this, options) + assertFalse (schema) + } + } + + Array(3) { when (it) { 0 -> "--schema"; 1 -> "-f"; 2 -> "path/to/file"; else -> "error" } }.let { + getMode(it).apply { + loadModeSpecificOptions(this, it) + assertTrue (schema) + } + } + + Array(3) { when (it) { 0 -> "-f"; 1 -> "path/to/file"; 2 -> "-s"; else -> "error" } }.let { + getMode(it).apply { + loadModeSpecificOptions(this, it) + assertTrue (schema) + } + } + + } + + +} \ No newline at end of file diff --git a/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/SimplifyClassTests.kt b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/SimplifyClassTests.kt new file mode 100644 index 0000000000..10d470685b --- /dev/null +++ b/experimental/blobinspector/src/test/kotlin/net/corda/blobinspector/SimplifyClassTests.kt @@ -0,0 +1,28 @@ +package net.corda.blobinspector + +import org.junit.Test + +class SimplifyClassTests { + + @Test + fun test1() { + data class A(val a: Int) + + println (A::class.java.name) + println (A::class.java.name.simplifyClass()) + } + + @Test + fun test2() { + val p = this.javaClass.`package`.name + + println("$p.Class1<$p.Class2>") + println("$p.Class1<$p.Class2>".simplifyClass()) + println("$p.Class1<$p.Class2, $p.Class3>") + println("$p.Class1<$p.Class2, $p.Class3>".simplifyClass()) + println("$p.Class1<$p.Class2<$p.Class3>>") + println("$p.Class1<$p.Class2<$p.Class3>>".simplifyClass()) + println("$p.Class1<$p.Class2<$p.Class3>>") + println("$p.Class1\$C<$p.Class2<$p.Class3>>".simplifyClass()) + } +} \ No newline at end of file diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Composite b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Composite new file mode 100644 index 0000000000..450e6970da Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Composite differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Int b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Int new file mode 100644 index 0000000000..25dcb48d65 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1Int differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1String b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1String new file mode 100644 index 0000000000..9676f0375f Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.1String differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Composite b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Composite new file mode 100644 index 0000000000..0bf3a5c475 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Composite differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Int b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Int new file mode 100644 index 0000000000..118a23f37b Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.2Int differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.3Int b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.3Int new file mode 100644 index 0000000000..9f00d59068 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.3Int differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.IntList b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.IntList new file mode 100644 index 0000000000..d762a9e821 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.IntList differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntClass b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntClass new file mode 100644 index 0000000000..175949d9aa Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntClass differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntString b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntString new file mode 100644 index 0000000000..67ba352ec4 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.MapIntString differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.StringList b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.StringList new file mode 100644 index 0000000000..5758d9fa62 Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/FileParseTests.StringList differ diff --git a/experimental/blobinspector/src/test/resources/net/corda/blobinspector/networkParams b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/networkParams new file mode 100644 index 0000000000..dcdbaa7b5f Binary files /dev/null and b/experimental/blobinspector/src/test/resources/net/corda/blobinspector/networkParams differ diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt index 369978bb62..fa67d13c09 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt @@ -80,7 +80,7 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe open class SerializationFactoryImpl : SerializationFactory() { companion object { - private val magicSize = sequenceOf(kryoMagic, amqpMagic).map { it.size }.distinct().single() + val magicSize = sequenceOf(kryoMagic, amqpMagic).map { it.size }.distinct().single() } private val creator: List = Exception().stackTrace.asList() diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt index 04c85e4926..fdfaf01d81 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt @@ -35,7 +35,7 @@ class DeserializationInput @JvmOverloads constructor(private val serializerFacto private val encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist) { private val objectHistory: MutableList = mutableListOf() - internal companion object { + companion object { private val BYTES_NEEDED_TO_PEEK: Int = 23 fun peekSize(bytes: ByteArray): Int { @@ -60,7 +60,7 @@ class DeserializationInput @JvmOverloads constructor(private val serializerFacto @VisibleForTesting @Throws(NotSerializableException::class) - internal fun withDataBytes(byteSequence: ByteSequence, encodingWhitelist: EncodingWhitelist, task: (ByteBuffer) -> T): T { + fun withDataBytes(byteSequence: ByteSequence, encodingWhitelist: EncodingWhitelist, task: (ByteBuffer) -> T): T { // Check that the lead bytes match expected header val amqpSequence = amqpMagic.consume(byteSequence) ?: throw NotSerializableException("Serialization header does not match.") var stream: InputStream = ByteBufferInputStream(amqpSequence) @@ -79,8 +79,22 @@ class DeserializationInput @JvmOverloads constructor(private val serializerFacto stream.close() } } + + @Throws(NotSerializableException::class) + fun getEnvelope(byteSequence: ByteSequence, encodingWhitelist: EncodingWhitelist = NullEncodingWhitelist): Envelope { + return withDataBytes(byteSequence, encodingWhitelist) { dataBytes -> + val data = Data.Factory.create() + val expectedSize = dataBytes.remaining() + if (data.decode(dataBytes) != expectedSize.toLong()) throw NotSerializableException("Unexpected size of data") + Envelope.get(data) + } + } } + + @Throws(NotSerializableException::class) + fun getEnvelope(byteSequence: ByteSequence) = Companion.getEnvelope(byteSequence, encodingWhitelist) + @Throws(NotSerializableException::class) inline fun deserialize(bytes: SerializedBytes): T = deserialize(bytes, T::class.java) @@ -88,16 +102,6 @@ class DeserializationInput @JvmOverloads constructor(private val serializerFacto inline internal fun deserializeAndReturnEnvelope(bytes: SerializedBytes): ObjectAndEnvelope = deserializeAndReturnEnvelope(bytes, T::class.java) - @Throws(NotSerializableException::class) - internal fun getEnvelope(byteSequence: ByteSequence): Envelope { - return withDataBytes(byteSequence, encodingWhitelist) { dataBytes -> - val data = Data.Factory.create() - val expectedSize = dataBytes.remaining() - if (data.decode(dataBytes) != expectedSize.toLong()) throw NotSerializableException("Unexpected size of data") - Envelope.get(data) - } - } - @Throws(NotSerializableException::class) private fun des(generator: () -> R): R { try { @@ -118,13 +122,13 @@ class DeserializationInput @JvmOverloads constructor(private val serializerFacto */ @Throws(NotSerializableException::class) fun deserialize(bytes: ByteSequence, clazz: Class): T = des { - val envelope = getEnvelope(bytes) + val envelope = getEnvelope(bytes, encodingWhitelist) clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)) } @Throws(NotSerializableException::class) fun deserializeAndReturnEnvelope(bytes: SerializedBytes, clazz: Class): ObjectAndEnvelope = des { - val envelope = getEnvelope(bytes) + val envelope = getEnvelope(bytes, encodingWhitelist) // Now pick out the obj and schema from the envelope. ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)), envelope) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Envelope.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Envelope.kt index 5b489c5d84..10fcc8966b 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Envelope.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Envelope.kt @@ -29,7 +29,7 @@ data class Envelope(val obj: Any?, val schema: Schema, val transformsSchema: Tra fun get(data: Data): Envelope { val describedType = data.`object` as DescribedType if (describedType.descriptor != DESCRIPTOR) { - throw NotSerializableException("Unexpected descriptor ${describedType.descriptor}.") + throw NotSerializableException("Unexpected descriptor ${describedType.descriptor}, should be $DESCRIPTOR.") } val list = describedType.described as List<*> diff --git a/settings.gradle b/settings.gradle index 335d58496f..113b5fbaa5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,6 +20,7 @@ include 'experimental:behave' include 'experimental:sandbox' include 'experimental:quasar-hook' include 'experimental:kryo-hook' +include 'experimental:blobinspector' include 'test-common' include 'test-utils' include 'smoke-test-utils'