From 8f0c7c947a4a48d1cc4f5b21fd9a17b004ad3256 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Sun, 11 Aug 2019 23:32:35 +0100 Subject: [PATCH 01/12] Allow custom serializers to be registered with type aliases for deserializing. --- .../internal/amqp/CustomSerializer.kt | 6 +++++- .../internal/amqp/CustomSerializerRegistry.kt | 15 ++++++++++++--- .../corda/serialization/internal/amqp/Schema.kt | 5 +++-- .../internal/amqp/SerializerFactoryBuilder.kt | 2 +- .../internal/model/LocalTypeModel.kt | 4 +--- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt index 0be0de6c23..ef9134e9e4 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt @@ -29,6 +29,10 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { */ open val additionalSerializers: Iterable> = emptyList() + /** + * This custom serializer is also allowed to deserialize these classes. + */ + open val deserializationAliases: Set> = emptySet() protected abstract val descriptor: Descriptor /** @@ -110,7 +114,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { */ abstract class CustomSerializerImp(protected val clazz: Class, protected val withInheritance: Boolean) : CustomSerializer() { override val type: Type get() = clazz - override val typeDescriptor: Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(clazz)}") + override val typeDescriptor: Symbol = typeDescriptorFor(clazz) override fun writeClassInfo(output: SerializationOutput) {} override val descriptor: Descriptor = Descriptor(typeDescriptor) override fun isSerializerFor(clazz: Class<*>): Boolean = if (withInheritance) this.clazz.isAssignableFrom(clazz) else this.clazz == clazz diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt index 2081a8162f..48486315fa 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt @@ -84,7 +84,7 @@ class CachingCustomSerializerRegistry( } private val customSerializersCache: MutableMap = DefaultCacheProvider.createCache() - private var customSerializers: List = emptyList() + private val customSerializers: MutableList = mutableListOf() /** * Register a custom serializer for any type that cannot be serialized or deserialized by the default serializer @@ -93,7 +93,7 @@ class CachingCustomSerializerRegistry( override fun register(customSerializer: CustomSerializer) { logger.trace("action=\"Registering custom serializer\", class=\"${customSerializer.type}\"") - if (!customSerializersCache.isEmpty()) { + if (customSerializersCache.isNotEmpty()) { logger.warn("Attempting to register custom serializer $customSerializer.type} in an active cache." + "All serializers should be registered before the cache comes into use.") } @@ -103,14 +103,23 @@ class CachingCustomSerializerRegistry( for (additional in customSerializer.additionalSerializers) { register(additional) } + + for (alias in customSerializer.deserializationAliases) { + val aliasDescriptor = typeDescriptorFor(alias) + if (aliasDescriptor != customSerializer.typeDescriptor) { + descriptorBasedSerializerRegistry[aliasDescriptor.toString()] = customSerializer + } + } + customSerializer } + } override fun registerExternal(customSerializer: CorDappCustomSerializer) { logger.trace("action=\"Registering external serializer\", class=\"${customSerializer.type}\"") - if (!customSerializersCache.isEmpty()) { + if (customSerializersCache.isNotEmpty()) { logger.warn("Attempting to register custom serializer ${customSerializer.type} in an active cache." + "All serializers must be registered before the cache comes into use.") } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt index f1bb00f7c0..81584378a6 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt @@ -9,12 +9,13 @@ import org.apache.qpid.proton.amqp.UnsignedInteger import org.apache.qpid.proton.amqp.UnsignedLong import org.apache.qpid.proton.codec.DescribedTypeConstructor import java.io.NotSerializableException -import net.corda.serialization.internal.carpenter.Field as CarpenterField -import net.corda.serialization.internal.carpenter.Schema as CarpenterSchema +import java.lang.reflect.Type const val DESCRIPTOR_DOMAIN: String = "net.corda" val amqpMagic = CordaSerializationMagic("corda".toByteArray() + byteArrayOf(1, 0)) +fun typeDescriptorFor(type: Type): Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(type)}") + /** * This and the classes below are OO representations of the AMQP XML schema described in the specification. Their * [toString] representations generate the associated XML form. diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index 7c5ec79c49..82de1c6c7c 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -92,7 +92,7 @@ object SerializerFactoryBuilder { customSerializerRegistry, onlyCustomSerializers) - val typeLoader = ClassCarpentingTypeLoader( + val typeLoader: TypeLoader = ClassCarpentingTypeLoader( SchemaBuildingRemoteTypeCarpenter(classCarpenter), classCarpenter.classloader) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/LocalTypeModel.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/LocalTypeModel.kt index 45bdc8794f..fb801c52d7 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/LocalTypeModel.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/LocalTypeModel.kt @@ -1,7 +1,5 @@ package net.corda.serialization.internal.model -import net.corda.core.serialization.ClassWhitelist -import net.corda.serialization.internal.amqp.* import java.lang.reflect.* /** @@ -54,7 +52,7 @@ class ConfigurableLocalTypeModel(private val typeModelConfiguration: LocalTypeMo private val typeInformationCache = DefaultCacheProvider.createCache() /** - * We need to provide the [TypeInformationBuilder] with a temporary local cache, so that it doesn't leak + * We need to provide the [LocalTypeInformationBuilder] with a temporary local cache, so that it doesn't leak * [LocalTypeInformation] with unpatched cycles into the global cache where other threads can access them * before we've patched the cycles up. */ From e4f38d19451a415fef31d2cbd92512c8062514f4 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 12 Aug 2019 18:12:58 +0100 Subject: [PATCH 02/12] Ensure that described properties are associated with a descriptor. --- serialization-deterministic/build.gradle | 4 +-- .../internal/amqp/ArraySerializer.kt | 4 ++- .../amqp/ComposableTypePropertySerializer.kt | 10 +++---- .../internal/amqp/CustomSerializer.kt | 4 +++ .../serialization/internal/amqp/Schema.kt | 28 ++++++++++++++++--- .../internal/model/TypeIdentifier.kt | 5 +++- 6 files changed, 42 insertions(+), 13 deletions(-) diff --git a/serialization-deterministic/build.gradle b/serialization-deterministic/build.gradle index 497745157b..229b2e179a 100644 --- a/serialization-deterministic/build.gradle +++ b/serialization-deterministic/build.gradle @@ -22,12 +22,12 @@ dependencies { // Configure these by hand. It should be a minimal subset of dependencies, // and without any obviously non-deterministic ones such as Hibernate. - // This dependency will become "compile" scoped in our published POM. + // These dependencies will become "compile" scoped in our published POM. // See publish.dependenciesFrom.defaultScope. deterministicLibraries project(path: ':core-deterministic', configuration: 'deterministicArtifacts') + deterministicLibraries "org.apache.qpid:proton-j:$protonj_version" // These "implementation" dependencies will become "runtime" scoped in our published POM. - implementation "org.apache.qpid:proton-j:$protonj_version" implementation "org.iq80.snappy:snappy:$snappy_version" implementation "com.google.guava:guava:$guava_version" } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt index 3e2266f63a..63b6ce73b7 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt @@ -88,7 +88,9 @@ open class ArraySerializer(override val type: Type, factory: LocalSerializerFact context: SerializationContext ): Any { if (obj is List<*>) { - return obj.map { input.readObjectOrNull(it, schemas, elementType, context) }.toArrayOfType(elementType) + return obj.map { + input.readObjectOrNull(redescribe(it, elementType), schemas, elementType, context) + }.toArrayOfType(elementType) } else throw AMQPNotSerializableException(type, "Expected a List but found $obj") } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt index aa9764fd1e..f384dfbabd 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt @@ -1,6 +1,7 @@ package net.corda.serialization.internal.amqp import net.corda.core.serialization.SerializationContext +import net.corda.serialization.internal.amqp.AMQPTypeIdentifiers.isPrimitive import net.corda.serialization.internal.model.* import org.apache.qpid.proton.amqp.Binary import org.apache.qpid.proton.codec.Data @@ -18,7 +19,7 @@ interface PropertyReadStrategy { * Select the correct strategy for reading properties, based on the property type. */ fun make(name: String, typeIdentifier: TypeIdentifier, type: Type): PropertyReadStrategy = - if (AMQPTypeIdentifiers.isPrimitive(typeIdentifier)) { + if (isPrimitive(typeIdentifier)) { when (typeIdentifier) { in characterTypes -> AMQPCharPropertyReadStrategy else -> AMQPPropertyReadStrategy @@ -47,7 +48,7 @@ interface PropertyWriteStrategy { fun make(name: String, propertyInformation: LocalPropertyInformation, factory: LocalSerializerFactory): PropertyWriteStrategy { val reader = PropertyReader.make(propertyInformation) val type = propertyInformation.type - return if (AMQPTypeIdentifiers.isPrimitive(type.typeIdentifier)) { + return if (isPrimitive(type.typeIdentifier)) { when (type.typeIdentifier) { in characterTypes -> AMQPCharPropertyWriteStategy(reader) else -> AMQPPropertyWriteStrategy(reader) @@ -191,15 +192,14 @@ object EvolutionPropertyWriteStrategy : PropertyWriteStrategy { * Read a type that comes with its own [TypeDescriptor], by calling back into [RemoteSerializerFactory] to obtain a suitable * serializer for that descriptor. */ -class DescribedTypeReadStrategy(name: String, - typeIdentifier: TypeIdentifier, +class DescribedTypeReadStrategy(name: String, typeIdentifier: TypeIdentifier, private val type: Type): PropertyReadStrategy { private val nameForDebug = "$name(${typeIdentifier.prettyPrint(false)})" override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput, context: SerializationContext): Any? = ifThrowsAppend({ nameForDebug }) { - input.readObjectOrNull(obj, schemas, type, context) + input.readObjectOrNull(redescribe(obj, type), schemas, type, context) } } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt index ef9134e9e4..d12495e4fd 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt @@ -123,11 +123,13 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { /** * Additional base features for a custom serializer for a particular class, that excludes subclasses. */ + @KeepForDJVM abstract class Is(clazz: Class) : CustomSerializerImp(clazz, false) /** * Additional base features for a custom serializer for all implementations of a particular interface or super class. */ + @KeepForDJVM abstract class Implements(clazz: Class) : CustomSerializerImp(clazz, true) /** @@ -137,6 +139,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { * The proxy class must use only types which are either native AMQP or other types for which there are pre-registered * custom serializers. */ + @KeepForDJVM abstract class Proxy(clazz: Class, protected val proxyClass: Class

, protected val factory: LocalSerializerFactory, @@ -195,6 +198,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { * @param maker A lambda for constructing an instance, that defaults to calling a constructor that expects a string. * @param unmaker A lambda that extracts the string value for an instance, that defaults to the [toString] method. */ + @KeepForDJVM abstract class ToString(clazz: Class, withInheritance: Boolean = false, private val maker: (String) -> T = clazz.getConstructor(String::class.java).let { `constructor` -> { string -> `constructor`.newInstance(string) } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt index 81584378a6..91d0537e71 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt @@ -3,10 +3,10 @@ package net.corda.serialization.internal.amqp import net.corda.core.KeepForDJVM import net.corda.core.internal.uncheckedCast import net.corda.serialization.internal.CordaSerializationMagic -import org.apache.qpid.proton.amqp.DescribedType -import org.apache.qpid.proton.amqp.Symbol -import org.apache.qpid.proton.amqp.UnsignedInteger -import org.apache.qpid.proton.amqp.UnsignedLong +import net.corda.serialization.internal.amqp.AMQPTypeIdentifiers.isPrimitive +import net.corda.serialization.internal.model.TypeIdentifier.TopType +import net.corda.serialization.internal.model.TypeIdentifier.Companion.forGenericType +import org.apache.qpid.proton.amqp.* import org.apache.qpid.proton.codec.DescribedTypeConstructor import java.io.NotSerializableException import java.lang.reflect.Type @@ -16,6 +16,26 @@ val amqpMagic = CordaSerializationMagic("corda".toByteArray() + byteArrayOf(1, 0 fun typeDescriptorFor(type: Type): Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(type)}") +fun redescribe(obj: Any?, type: Type): Any? { + return if (obj == null || obj is DescribedType || obj is Binary || forGenericType(type).run { isPrimitive(this) || this == TopType }) { + obj + } else { + /** + * This must be a primitive [obj] that has a non-primitive [type]. + * Rewrap it with the required descriptor for further deserialization. + */ + RedescribedType(typeDescriptorFor(type), obj) + } +} + +private class RedescribedType( + private val descriptor: Symbol, + private val described: Any? +) : DescribedType { + override fun getDescriptor(): Symbol = descriptor + override fun getDescribed(): Any? = described +} + /** * This and the classes below are OO representations of the AMQP XML schema described in the specification. Their * [toString] representations generate the associated XML form. diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt index 43f0329ab9..6e1dd4953f 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt @@ -206,7 +206,10 @@ sealed class TypeIdentifier { override fun toString() = "Parameterised(${prettyPrint()})" override fun getLocalType(classLoader: ClassLoader): Type { - val rawType = Class.forName(name, false, classLoader) + // We need to invoke ClassLoader.loadClass() directly, because + // the JVM will complain if Class.forName() returns a class + // that has a name other than the requested one. + val rawType = classLoader.loadClass(name) if (rawType.typeParameters.size != parameters.size) { throw IncompatibleTypeIdentifierException( "Class $rawType expects ${rawType.typeParameters.size} type arguments, " + From be64c895158fd310c04c9069d1a74239dcd13c09 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Thu, 15 Aug 2019 14:53:15 +0100 Subject: [PATCH 03/12] Implement generic CustomerSerializers that create more specific AMQPSerializer instances at runtime. --- .../serialization/internal/amqp/CustomSerializer.kt | 2 ++ .../internal/amqp/CustomSerializerRegistry.kt | 13 ++++++++++--- .../internal/amqp/DeserializationInput.kt | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt index d12495e4fd..8eb4c6c60a 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt @@ -57,6 +57,8 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { abstract fun writeDescribedObject(obj: T, data: Data, type: Type, output: SerializationOutput, context: SerializationContext) + open fun specialiseFor(declaredType: Type): AMQPSerializer? = this + /** * This custom serializer represents a sort of symbolic link from a subclass to a super class, where the super * class custom serializer is responsible for the "on the wire" format but we want to create a reference to the diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt index 48486315fa..4ef4fe2307 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt @@ -129,7 +129,7 @@ class CachingCustomSerializerRegistry( customSerializer } } - + override fun findCustomSerializer(clazz: Class<*>, declaredType: Type): AMQPSerializer? { val typeIdentifier = CustomSerializerIdentifier( TypeIdentifier.forClass(clazz), @@ -173,7 +173,13 @@ class CachingCustomSerializerRegistry( throw IllegalCustomSerializerException(declaredSerializers.first(), clazz) } - return declaredSerializers.first() + return declaredSerializers.first().let { + if (it is CustomSerializer) { + it.specialiseFor(declaredType) + } else { + it + } + } } private val Class<*>.isCustomSerializationForbidden: Boolean get() = when { @@ -182,4 +188,5 @@ class CachingCustomSerializerRegistry( isAnnotationPresent(CordaSerializable::class.java) -> true else -> false } -} \ No newline at end of file +} + diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/DeserializationInput.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/DeserializationInput.kt index 503d370fd6..c21050bc8d 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/DeserializationInput.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/DeserializationInput.kt @@ -142,12 +142,12 @@ class DeserializationInput constructor( envelope) } - internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type, context: SerializationContext + fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type, context: SerializationContext ): Any? { return if (obj == null) null else readObject(obj, schema, type, context) } - internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, context: SerializationContext): Any = + fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, context: SerializationContext): Any = if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) { // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference. val objectIndex = (obj.described as UnsignedInteger).toInt() From aa2f1029a65543b9bfc5e8a791b92ecd19c98ada Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Sun, 18 Aug 2019 19:45:00 +0100 Subject: [PATCH 04/12] Use LocalTypeIdentifier information where available to lookup CustomSerializer. --- .../internal/amqp/CustomSerializerRegistry.kt | 2 +- .../internal/amqp/LocalSerializerFactory.kt | 17 ++++++++++++--- .../internal/model/TypeIdentifier.kt | 2 ++ .../model/TypeModellingFingerPrinter.kt | 21 +++++++++++++++++-- 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt index 4ef4fe2307..8397fe8018 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt @@ -44,7 +44,7 @@ interface CustomSerializerRegistry { * * @param clazz The actual class to look for a custom serializer for. * @param declaredType The declared type to look for a custom serializer for. - * @return The custom serializer handing the class, if found, or `null`. + * @return The custom serializer handling the class, if found, or `null`. * * @throws IllegalCustomSerializerException If a custom serializer identifies itself as the serializer for * a class annotated with [CordaSerializable], since all such classes should be serializable via standard object diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt index fe34660a68..543a8e8e2f 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt @@ -6,6 +6,8 @@ import net.corda.core.utilities.contextLogger import net.corda.core.utilities.debug import net.corda.core.utilities.trace import net.corda.serialization.internal.model.* +import net.corda.serialization.internal.model.TypeIdentifier.* +import net.corda.serialization.internal.model.TypeIdentifier.Companion.classLoaderFor import org.apache.qpid.proton.amqp.Symbol import java.lang.reflect.ParameterizedType import java.lang.reflect.Type @@ -137,9 +139,18 @@ class DefaultLocalSerializerFactory( serializersByTypeId.getOrPut(localTypeInformation.typeIdentifier) { val declaredClass = declaredType.asClass() + // Any Custom Serializer cached for a ParameterizedType can only be + // found by searching for that exact same type. Searching for its raw + // class will not work! + val declaredGenericType = if (declaredType !is ParameterizedType && localTypeInformation.typeIdentifier is Parameterised) { + localTypeInformation.typeIdentifier.getLocalType(classLoaderFor(declaredClass)) + } else { + declaredType + } + // can be useful to enable but will be *extremely* chatty if you do - logger.trace { "Get Serializer for $declaredClass ${declaredType.typeName}" } - customSerializerRegistry.findCustomSerializer(declaredClass, declaredType)?.apply { return@get this } + logger.trace { "Get Serializer for $declaredClass ${declaredGenericType.typeName}" } + customSerializerRegistry.findCustomSerializer(declaredClass, declaredGenericType)?.apply { return@get this } return when (localTypeInformation) { is LocalTypeInformation.ACollection -> makeDeclaredCollection(localTypeInformation) @@ -250,4 +261,4 @@ class DefaultLocalSerializerFactory( } } -} \ No newline at end of file +} diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt index 6e1dd4953f..274cd50dda 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt @@ -63,6 +63,8 @@ sealed class TypeIdentifier { // This method has locking. So we memo the value here. private val systemClassLoader: ClassLoader = ClassLoader.getSystemClassLoader() + fun classLoaderFor(clazz: Class<*>): ClassLoader = clazz.classLoader ?: systemClassLoader + /** * Obtain the [TypeIdentifier] for an erased Java class. * diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt index b0314bbc2a..7077eecce1 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt @@ -4,7 +4,9 @@ import com.google.common.hash.Hashing import net.corda.core.utilities.contextLogger import net.corda.core.utilities.toBase64 import net.corda.serialization.internal.amqp.* -import java.io.NotSerializableException +import net.corda.serialization.internal.model.TypeIdentifier.* +import net.corda.serialization.internal.model.TypeIdentifier.Companion.classLoaderFor +import java.lang.reflect.ParameterizedType /** * A fingerprinter that fingerprints [LocalTypeInformation]. @@ -224,7 +226,22 @@ private class FingerPrintingState( // Give any custom serializers loaded into the factory the chance to supply their own type-descriptors private fun fingerprintWithCustomSerializerOrElse(type: LocalTypeInformation, defaultAction: () -> Unit) { - val customTypeDescriptor = customSerializerRegistry.findCustomSerializer(type.observedType.asClass(), type.observedType)?.typeDescriptor?.toString() + val observedType = type.observedType + val observedClass = observedType.asClass() + + // Any Custom Serializer cached for a ParameterizedType can only be + // found by searching for that exact same type. Searching for its raw + // class will not work! + val observedGenericType = if (observedType !is ParameterizedType && type.typeIdentifier is Parameterised) { + type.typeIdentifier.getLocalType(classLoaderFor(observedClass)) + } else { + observedType + } + + val customTypeDescriptor = customSerializerRegistry.findCustomSerializer( + clazz = observedClass, + declaredType = observedGenericType + )?.typeDescriptor?.toString() if (customTypeDescriptor != null) writer.write(customTypeDescriptor) else defaultAction() } From bdd5d136551a837193bf4174a6701d4e3121a6d5 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 19 Aug 2019 11:51:49 +0100 Subject: [PATCH 05/12] Provide a map of Java primitive types as a configuration value. --- .../internal/amqp/AMQPSerializerFactories.kt | 7 ++++++- .../internal/amqp/AMQPSerializerFactories.kt | 9 ++++++++- .../amqp/EvolutionSerializerFactory.kt | 13 ++++++++++-- .../internal/amqp/SerializerFactoryBuilder.kt | 20 ++++++++++++++++++- 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt b/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt index 2b1d66ab76..19a59bc9ba 100644 --- a/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt +++ b/serialization-deterministic/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt @@ -13,11 +13,16 @@ import net.corda.serialization.internal.carpenter.Schema @Suppress("UNUSED") fun createSerializerFactoryFactory(): SerializerFactoryFactory = DeterministicSerializerFactoryFactory() +/** + * Creates a [ClassCarpenter] suitable for the DJVM, i.e. one that doesn't work. + */ +fun createClassCarpenter(context: SerializationContext): ClassCarpenter = DummyClassCarpenter(context.whitelist, context.deserializationClassLoader) + private class DeterministicSerializerFactoryFactory : SerializerFactoryFactory { override fun make(context: SerializationContext) = SerializerFactoryBuilder.build( whitelist = context.whitelist, - classCarpenter = DummyClassCarpenter(context.whitelist, context.deserializationClassLoader)) + classCarpenter = createClassCarpenter(context)) } private class DummyClassCarpenter( diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt index 878a294695..071dbf74b7 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPSerializerFactories.kt @@ -3,14 +3,21 @@ package net.corda.serialization.internal.amqp import net.corda.core.serialization.SerializationContext +import net.corda.serialization.internal.carpenter.ClassCarpenter import net.corda.serialization.internal.carpenter.ClassCarpenterImpl fun createSerializerFactoryFactory(): SerializerFactoryFactory = SerializerFactoryFactoryImpl() +fun createClassCarpenter(context: SerializationContext): ClassCarpenter = ClassCarpenterImpl( + whitelist = context.whitelist, + cl = context.deserializationClassLoader, + lenient = context.lenientCarpenterEnabled +) + open class SerializerFactoryFactoryImpl : SerializerFactoryFactory { override fun make(context: SerializationContext): SerializerFactory { return SerializerFactoryBuilder.build(context.whitelist, - ClassCarpenterImpl(context.whitelist, context.deserializationClassLoader, context.lenientCarpenterEnabled), + createClassCarpenter(context), mustPreserveDataWhenEvolving = context.preventDataLoss ) } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EvolutionSerializerFactory.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EvolutionSerializerFactory.kt index 8821860355..4755289cf6 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EvolutionSerializerFactory.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EvolutionSerializerFactory.kt @@ -17,6 +17,13 @@ interface EvolutionSerializerFactory { fun getEvolutionSerializer( remote: RemoteTypeInformation, local: LocalTypeInformation): AMQPSerializer? + + /** + * A mapping between Java object types and their equivalent Java primitive types. + * Predominantly for the sake of the DJVM sandbox where e.g. `char` will map to + * sandbox.java.lang.Character instead of java.lang.Character. + */ + val primitiveTypes: Map, Class<*>> } class EvolutionSerializationException(remoteTypeInformation: RemoteTypeInformation, reason: String) @@ -32,7 +39,9 @@ class EvolutionSerializationException(remoteTypeInformation: RemoteTypeInformati class DefaultEvolutionSerializerFactory( private val localSerializerFactory: LocalSerializerFactory, private val classLoader: ClassLoader, - private val mustPreserveDataWhenEvolving: Boolean): EvolutionSerializerFactory { + private val mustPreserveDataWhenEvolving: Boolean, + override val primitiveTypes: Map, Class<*>> +): EvolutionSerializerFactory { override fun getEvolutionSerializer(remote: RemoteTypeInformation, local: LocalTypeInformation): AMQPSerializer? = @@ -77,7 +86,7 @@ class DefaultEvolutionSerializerFactory( val localClass = localProperty.type.observedType.asClass() val remoteClass = remoteProperty.type.typeIdentifier.getLocalType(classLoader).asClass() - if (!localClass.isAssignableFrom(remoteClass) && remoteClass != localClass.kotlin.javaPrimitiveType) { + if (!localClass.isAssignableFrom(remoteClass) && remoteClass != primitiveTypes[localClass]) { throw EvolutionSerializationException(this, "Local type $localClass of property $name is not assignable from remote type $remoteClass") } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index 82de1c6c7c..1af772c712 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -7,9 +7,24 @@ import net.corda.serialization.internal.carpenter.ClassCarpenter import net.corda.serialization.internal.carpenter.ClassCarpenterImpl import net.corda.serialization.internal.model.* import java.io.NotSerializableException +import java.util.Collections.unmodifiableMap @KeepForDJVM object SerializerFactoryBuilder { + /** + * The standard mapping of Java object types to Java primitive types. + * The DJVM will need to override these, but probably not anyone else. + */ + private val javaPrimitiveTypes: Map, Class<*>> = unmodifiableMap(mapOf?, Class?>( + Boolean::class.javaObjectType to Boolean::class.javaPrimitiveType, + Byte::class.javaObjectType to Byte::class.javaPrimitiveType, + Char::class.javaObjectType to Char::class.javaPrimitiveType, + Double::class.javaObjectType to Double::class.javaPrimitiveType, + Float::class.javaObjectType to Float::class.javaPrimitiveType, + Int::class.javaObjectType to Int::class.javaPrimitiveType, + Long::class.javaObjectType to Long::class.javaPrimitiveType, + Short::class.javaObjectType to Short::class.javaPrimitiveType + )) as Map, Class<*>> @JvmStatic fun build(whitelist: ClassWhitelist, classCarpenter: ClassCarpenter): SerializerFactory { @@ -99,7 +114,8 @@ object SerializerFactoryBuilder { val evolutionSerializerFactory = if (allowEvolution) DefaultEvolutionSerializerFactory( localSerializerFactory, classCarpenter.classloader, - mustPreserveDataWhenEvolving + mustPreserveDataWhenEvolving, + javaPrimitiveTypes ) else NoEvolutionSerializerFactory val remoteSerializerFactory = DefaultRemoteSerializerFactory( @@ -127,4 +143,6 @@ Local: ${localTypeInformation.prettyPrint(false)} """) } + + override val primitiveTypes: Map, Class<*>> = emptyMap() } \ No newline at end of file From a5d5e0d47600c7fe2d484c421a6988a08ec1fde7 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 19 Aug 2019 18:12:04 +0100 Subject: [PATCH 06/12] Allow custom serialization for all subclasses of a configurable set of classes. --- .../internal/amqp/CustomSerializerRegistry.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt index 8397fe8018..d0e8700e08 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializerRegistry.kt @@ -57,8 +57,10 @@ interface CustomSerializerRegistry { } class CachingCustomSerializerRegistry( - private val descriptorBasedSerializerRegistry: DescriptorBasedSerializerRegistry) - : CustomSerializerRegistry { + private val descriptorBasedSerializerRegistry: DescriptorBasedSerializerRegistry, + private val allowedFor: Set> +) : CustomSerializerRegistry { + constructor(descriptorBasedSerializerRegistry: DescriptorBasedSerializerRegistry) : this(descriptorBasedSerializerRegistry, emptySet()) companion object { val logger = contextLogger() @@ -185,6 +187,7 @@ class CachingCustomSerializerRegistry( private val Class<*>.isCustomSerializationForbidden: Boolean get() = when { AMQPTypeIdentifiers.isPrimitive(this) -> true isSubClassOf(CordaThrowable::class.java) -> false + allowedFor.any { it.isAssignableFrom(this) } -> false isAnnotationPresent(CordaSerializable::class.java) -> true else -> false } From 99074b5a49ab56473800921dd9b64152ee0d55e4 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Wed, 21 Aug 2019 14:21:23 +0100 Subject: [PATCH 07/12] Modify the fingerprinter not to use ConcurrentHashMap.computeIfAbsent() because we cannot guarantee that the cache is not reentered by the computation. --- .../internal/model/TypeModellingFingerPrinter.kt | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt index 7077eecce1..b0ac855d34 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeModellingFingerPrinter.kt @@ -36,11 +36,15 @@ class TypeModellingFingerPrinter( private val cache: MutableMap = DefaultCacheProvider.createCache() override fun fingerprint(typeInformation: LocalTypeInformation): String = - cache.computeIfAbsent(typeInformation.typeIdentifier) { - FingerPrintingState( - customTypeDescriptorLookup, - FingerprintWriter(debugEnabled)).fingerprint(typeInformation) - } + /* + * We cannot use ConcurrentMap.computeIfAbsent() here because it requires + * that the map not be re-entered during the computation function. And + * the Fingerprinter cannot guarantee that. + */ + cache.getOrPut(typeInformation.typeIdentifier) { + FingerPrintingState(customTypeDescriptorLookup, FingerprintWriter(debugEnabled)) + .fingerprint(typeInformation) + } } /** From b2d335c5184a9d61ed1ced2e497f0097e02a5a46 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Thu, 22 Aug 2019 18:02:55 +0100 Subject: [PATCH 08/12] Make the choice of AMQP serializer for primitive types configurable. --- .../internal/amqp/LocalSerializerFactory.kt | 4 +++- .../internal/amqp/RemoteSerializerFactory.kt | 1 - .../net/corda/serialization/internal/amqp/Schema.kt | 4 +++- .../internal/amqp/SerializerFactoryBuilder.kt | 9 ++++++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt index 543a8e8e2f..bfd8863441 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/LocalSerializerFactory.kt @@ -12,6 +12,7 @@ import org.apache.qpid.proton.amqp.Symbol import java.lang.reflect.ParameterizedType import java.lang.reflect.Type import java.util.* +import java.util.function.Function import javax.annotation.concurrent.ThreadSafe /** @@ -89,6 +90,7 @@ class DefaultLocalSerializerFactory( private val fingerPrinter: FingerPrinter, override val classloader: ClassLoader, private val descriptorBasedSerializerRegistry: DescriptorBasedSerializerRegistry, + private val primitiveSerializerFactory: Function, AMQPSerializer>, private val customSerializerRegistry: CustomSerializerRegistry, private val onlyCustomSerializers: Boolean) : LocalSerializerFactory { @@ -237,7 +239,7 @@ class DefaultLocalSerializerFactory( throw AMQPNotSerializableException( type, "Serializer does not support synthetic classes") - AMQPTypeIdentifiers.isPrimitive(typeInformation.typeIdentifier) -> AMQPPrimitiveSerializer(clazz) + AMQPTypeIdentifiers.isPrimitive(typeInformation.typeIdentifier) -> primitiveSerializerFactory.apply(clazz) else -> makeNonCustomSerializer(type, typeInformation, clazz) } } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/RemoteSerializerFactory.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/RemoteSerializerFactory.kt index e7b00f618a..6c057bfd71 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/RemoteSerializerFactory.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/RemoteSerializerFactory.kt @@ -3,7 +3,6 @@ package net.corda.serialization.internal.amqp import net.corda.core.serialization.SerializationContext import net.corda.core.utilities.contextLogger import net.corda.serialization.internal.model.* -import org.hibernate.type.descriptor.java.ByteTypeDescriptor import java.io.NotSerializableException /** diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt index 91d0537e71..26dc084587 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt @@ -4,6 +4,7 @@ import net.corda.core.KeepForDJVM import net.corda.core.internal.uncheckedCast import net.corda.serialization.internal.CordaSerializationMagic import net.corda.serialization.internal.amqp.AMQPTypeIdentifiers.isPrimitive +import net.corda.serialization.internal.model.TypeIdentifier import net.corda.serialization.internal.model.TypeIdentifier.TopType import net.corda.serialization.internal.model.TypeIdentifier.Companion.forGenericType import org.apache.qpid.proton.amqp.* @@ -14,7 +15,8 @@ import java.lang.reflect.Type const val DESCRIPTOR_DOMAIN: String = "net.corda" val amqpMagic = CordaSerializationMagic("corda".toByteArray() + byteArrayOf(1, 0)) -fun typeDescriptorFor(type: Type): Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(type)}") +fun typeDescriptorFor(typeId: TypeIdentifier): Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(typeId)}") +fun typeDescriptorFor(type: Type): Symbol = typeDescriptorFor(forGenericType(type)) fun redescribe(obj: Any?, type: Type): Any? { return if (obj == null || obj is DescribedType || obj is Binary || forGenericType(type).run { isPrimitive(this) || this == TopType }) { diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index 1af772c712..d380ebfaf2 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -8,6 +8,7 @@ import net.corda.serialization.internal.carpenter.ClassCarpenterImpl import net.corda.serialization.internal.model.* import java.io.NotSerializableException import java.util.Collections.unmodifiableMap +import java.util.function.Function @KeepForDJVM object SerializerFactoryBuilder { @@ -15,6 +16,7 @@ object SerializerFactoryBuilder { * The standard mapping of Java object types to Java primitive types. * The DJVM will need to override these, but probably not anyone else. */ + @Suppress("unchecked_cast") private val javaPrimitiveTypes: Map, Class<*>> = unmodifiableMap(mapOf?, Class?>( Boolean::class.javaObjectType to Boolean::class.javaPrimitiveType, Byte::class.javaObjectType to Byte::class.javaPrimitiveType, @@ -104,6 +106,7 @@ object SerializerFactoryBuilder { fingerPrinter, classCarpenter.classloader, descriptorBasedSerializerRegistry, + Function { clazz -> AMQPPrimitiveSerializer(clazz) }, customSerializerRegistry, onlyCustomSerializers) @@ -132,15 +135,15 @@ object SerializerFactoryBuilder { } object NoEvolutionSerializerFactory : EvolutionSerializerFactory { - override fun getEvolutionSerializer(remoteTypeInformation: RemoteTypeInformation, localTypeInformation: LocalTypeInformation): AMQPSerializer { + override fun getEvolutionSerializer(remote: RemoteTypeInformation, local: LocalTypeInformation): AMQPSerializer { throw NotSerializableException(""" Evolution not permitted. Remote: -${remoteTypeInformation.prettyPrint(false)} +${remote.prettyPrint(false)} Local: -${localTypeInformation.prettyPrint(false)} +${local.prettyPrint(false)} """) } From 4ebd02bc04c8193add37a6413e1f5b935f0fb3c7 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Fri, 23 Aug 2019 11:11:11 +0100 Subject: [PATCH 09/12] Tidy up changes for review. --- .../amqp/ComposableTypePropertySerializer.kt | 3 ++- .../internal/amqp/SerializerFactoryBuilder.kt | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt index f384dfbabd..254dc1f422 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ComposableTypePropertySerializer.kt @@ -192,7 +192,8 @@ object EvolutionPropertyWriteStrategy : PropertyWriteStrategy { * Read a type that comes with its own [TypeDescriptor], by calling back into [RemoteSerializerFactory] to obtain a suitable * serializer for that descriptor. */ -class DescribedTypeReadStrategy(name: String, typeIdentifier: TypeIdentifier, +class DescribedTypeReadStrategy(name: String, + typeIdentifier: TypeIdentifier, private val type: Type): PropertyReadStrategy { private val nameForDebug = "$name(${typeIdentifier.prettyPrint(false)})" diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt index d380ebfaf2..a6469b443c 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactoryBuilder.kt @@ -17,16 +17,18 @@ object SerializerFactoryBuilder { * The DJVM will need to override these, but probably not anyone else. */ @Suppress("unchecked_cast") - private val javaPrimitiveTypes: Map, Class<*>> = unmodifiableMap(mapOf?, Class?>( - Boolean::class.javaObjectType to Boolean::class.javaPrimitiveType, - Byte::class.javaObjectType to Byte::class.javaPrimitiveType, - Char::class.javaObjectType to Char::class.javaPrimitiveType, - Double::class.javaObjectType to Double::class.javaPrimitiveType, - Float::class.javaObjectType to Float::class.javaPrimitiveType, - Int::class.javaObjectType to Int::class.javaPrimitiveType, - Long::class.javaObjectType to Long::class.javaPrimitiveType, - Short::class.javaObjectType to Short::class.javaPrimitiveType - )) as Map, Class<*>> + private val javaPrimitiveTypes: Map, Class<*>> = unmodifiableMap(listOf( + Boolean::class, + Byte::class, + Char::class, + Double::class, + Float::class, + Int::class, + Long::class, + Short::class + ).associate { + klazz -> klazz.javaObjectType to klazz.javaPrimitiveType + }) as Map, Class<*>> @JvmStatic fun build(whitelist: ClassWhitelist, classCarpenter: ClassCarpenter): SerializerFactory { From 92ae45a9497718d640f787baa6e63fa7ae9cc221 Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Mon, 26 Aug 2019 15:26:36 +0100 Subject: [PATCH 10/12] Fix typo decimal62 -> decimal64. --- .../corda/serialization/internal/amqp/AMQPTypeIdentifiers.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPTypeIdentifiers.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPTypeIdentifiers.kt index e9d99cfe8f..fbbbaddde5 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPTypeIdentifiers.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPTypeIdentifiers.kt @@ -29,7 +29,7 @@ object AMQPTypeIdentifiers { Float::class to "float", Double::class to "double", Decimal32::class to "decimal32", - Decimal64::class to "decimal62", + Decimal64::class to "decimal64", Decimal128::class to "decimal128", Date::class to "timestamp", UUID::class to "uuid", @@ -62,4 +62,4 @@ object AMQPTypeIdentifiers { private val primitiveByteArrayType = TypeIdentifier.ArrayOf(TypeIdentifier.forClass(Byte::class.javaPrimitiveType!!)) fun nameForType(type: Type): String = nameForType(TypeIdentifier.forGenericType(type)) -} \ No newline at end of file +} From e25d9a1d4ea69b4cf1ed95ee679de648710e9aea Mon Sep 17 00:00:00 2001 From: Chris Rankin Date: Tue, 27 Aug 2019 11:24:41 +0100 Subject: [PATCH 11/12] Update KDocs. --- .../serialization/internal/amqp/CustomSerializer.kt | 9 ++++++++- .../net/corda/serialization/internal/amqp/Schema.kt | 5 +++++ .../corda/serialization/internal/model/TypeIdentifier.kt | 3 ++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt index 8eb4c6c60a..471546369e 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt @@ -30,7 +30,8 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { open val additionalSerializers: Iterable> = emptyList() /** - * This custom serializer is also allowed to deserialize these classes. + * This custom serializer is also allowed to deserialize these classes. This allows us + * to deserialize objects into completely different types, e.g. `A` -> `sandbox.A`. */ open val deserializationAliases: Set> = emptySet() @@ -57,6 +58,12 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { abstract fun writeDescribedObject(obj: T, data: Data, type: Type, output: SerializationOutput, context: SerializationContext) + /** + * [CustomSerializerRegistry.findCustomSerializer] will invoke this method on the [CustomSerializer] + * that it selects to give that serializer an opportunity to customise its behaviour. The serializer + * can also return `null` here, in which case [CustomSerializerRegistry] will proceed as if no + * serializer is available for [declaredType]. + */ open fun specialiseFor(declaredType: Type): AMQPSerializer? = this /** diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt index 26dc084587..36ac18bfe6 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt @@ -18,6 +18,11 @@ val amqpMagic = CordaSerializationMagic("corda".toByteArray() + byteArrayOf(1, 0 fun typeDescriptorFor(typeId: TypeIdentifier): Symbol = Symbol.valueOf("$DESCRIPTOR_DOMAIN:${AMQPTypeIdentifiers.nameForType(typeId)}") fun typeDescriptorFor(type: Type): Symbol = typeDescriptorFor(forGenericType(type)) +/** + * Repackages a naked, non-primitive [obj] as a [DescribedType]. If [obj] is primitive, [Binary] or already + * an instance of [DescribedType]] then it is returned unchanged. This allows Corda to search for a serializer + * capable of handling instances of [type]. + */ fun redescribe(obj: Any?, type: Type): Any? { return if (obj == null || obj is DescribedType || obj is Binary || forGenericType(type).run { isPrimitive(this) || this == TopType }) { obj diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt index 274cd50dda..2697b107a8 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/model/TypeIdentifier.kt @@ -210,7 +210,8 @@ sealed class TypeIdentifier { override fun getLocalType(classLoader: ClassLoader): Type { // We need to invoke ClassLoader.loadClass() directly, because // the JVM will complain if Class.forName() returns a class - // that has a name other than the requested one. + // that has a name other than the requested one. This will happen + // for "transformative" class loaders, i.e. `A` -> `sandbox.A`. val rawType = classLoader.loadClass(name) if (rawType.typeParameters.size != parameters.size) { throw IncompatibleTypeIdentifierException( From 90284a614337aa8d3632b0f8fe2c7eb33e8119f8 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Tue, 27 Aug 2019 13:06:28 +0100 Subject: [PATCH 12/12] CORDA-2919: JacksonSupport, for CordaSerializable classes, improved to only uses those properties that are part of Corda serialisation (#5397) --- .idea/compiler.xml | 2 + .../client/jackson/internal/CordaModule.kt | 45 +- .../client/jackson/JacksonSupportTest.kt | 15 + .../node/services/rpc/CheckpointDumper.kt | 10 +- .../shell/InteractiveShellIntegrationTest.kt | 386 ++++++++---------- 5 files changed, 223 insertions(+), 235 deletions(-) diff --git a/.idea/compiler.xml b/.idea/compiler.xml index e311d8edce..2cbf1d59fe 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -23,6 +23,8 @@ + + diff --git a/client/jackson/src/main/kotlin/net/corda/client/jackson/internal/CordaModule.kt b/client/jackson/src/main/kotlin/net/corda/client/jackson/internal/CordaModule.kt index 910f679d67..947a88cd38 100644 --- a/client/jackson/src/main/kotlin/net/corda/client/jackson/internal/CordaModule.kt +++ b/client/jackson/src/main/kotlin/net/corda/client/jackson/internal/CordaModule.kt @@ -3,6 +3,8 @@ package net.corda.client.jackson.internal import com.fasterxml.jackson.annotation.* +import com.fasterxml.jackson.annotation.JsonAutoDetect.Value +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility import com.fasterxml.jackson.annotation.JsonCreator.Mode.DISABLED import com.fasterxml.jackson.annotation.JsonInclude.Include import com.fasterxml.jackson.core.JsonGenerator @@ -12,10 +14,14 @@ import com.fasterxml.jackson.core.JsonToken import com.fasterxml.jackson.databind.* import com.fasterxml.jackson.databind.annotation.JsonDeserialize import com.fasterxml.jackson.databind.annotation.JsonSerialize +import com.fasterxml.jackson.databind.cfg.MapperConfig import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier import com.fasterxml.jackson.databind.deser.ContextualDeserializer import com.fasterxml.jackson.databind.deser.std.DelegatingDeserializer import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer +import com.fasterxml.jackson.databind.introspect.AnnotatedClass +import com.fasterxml.jackson.databind.introspect.BasicClassIntrospector +import com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector import com.fasterxml.jackson.databind.module.SimpleModule import com.fasterxml.jackson.databind.node.IntNode import com.fasterxml.jackson.databind.node.ObjectNode @@ -32,7 +38,6 @@ import net.corda.core.flows.StateMachineRunId import net.corda.core.identity.* import net.corda.core.internal.DigitalSignatureWithCert import net.corda.core.internal.createComponentGroups -import net.corda.core.internal.kotlinObjectInstance import net.corda.core.node.NodeInfo import net.corda.core.serialization.SerializeAsToken import net.corda.core.serialization.SerializedBytes @@ -56,7 +61,13 @@ class CordaModule : SimpleModule("corda-core") { override fun setupModule(context: SetupContext) { super.setupModule(context) + // For classes which are annotated with CordaSerializable we want to use the same set of properties as the Corda serilasation scheme. + // To do that we use CordaSerializableClassIntrospector to first turn on field visibility for these classes (the Jackson default is + // private fields are not included) and then we use CordaSerializableBeanSerializerModifier to remove any extra properties that Jackson + // might pick up. + context.setClassIntrospector(CordaSerializableClassIntrospector(context)) context.addBeanSerializerModifier(CordaSerializableBeanSerializerModifier()) + context.addBeanDeserializerModifier(AmountBeanDeserializerModifier()) context.setMixInAnnotations(PartyAndCertificate::class.java, PartyAndCertificateMixin::class.java) @@ -88,9 +99,22 @@ class CordaModule : SimpleModule("corda-core") { } } -/** - * Use the same properties that AMQP serialization uses if the POJO is @CordaSerializable - */ +private class CordaSerializableClassIntrospector(private val context: Module.SetupContext) : BasicClassIntrospector() { + override fun constructPropertyCollector( + config: MapperConfig<*>?, + ac: AnnotatedClass?, + type: JavaType, + forSerialization: Boolean, + mutatorPrefix: String? + ): POJOPropertiesCollector { + if (hasCordaSerializable(type.rawClass)) { + // Adjust the field visibility of CordaSerializable classes on the fly as they are encountered. + context.configOverride(type.rawClass).visibility = Value.defaultVisibility().withFieldVisibility(Visibility.ANY) + } + return super.constructPropertyCollector(config, ac, type, forSerialization, mutatorPrefix) + } +} + private class CordaSerializableBeanSerializerModifier : BeanSerializerModifier() { // We need to pass in a SerializerFactory when scanning for properties, but don't actually do any serialisation so any will do. private val serializerFactory = SerializerFactoryBuilder.build(AllWhitelist, javaClass.classLoader) @@ -99,17 +123,10 @@ private class CordaSerializableBeanSerializerModifier : BeanSerializerModifier() beanDesc: BeanDescription, beanProperties: MutableList): MutableList { val beanClass = beanDesc.beanClass - if (hasCordaSerializable(beanClass) && beanClass.kotlinObjectInstance == null && !SerializeAsToken::class.java.isAssignableFrom(beanClass)) { + if (hasCordaSerializable(beanClass) && !SerializeAsToken::class.java.isAssignableFrom(beanClass)) { val typeInformation = serializerFactory.getTypeInformation(beanClass) - val properties = typeInformation.propertiesOrEmptyMap - val amqpProperties = properties.mapNotNull { (name, property) -> - if (property.isCalculated) null else name - } - val propertyRenames = beanDesc.findProperties().associateBy({ it.name }, { it.internalName }) - (amqpProperties - propertyRenames.values).let { - check(it.isEmpty()) { "Jackson didn't provide serialisers for $it" } - } - beanProperties.removeIf { propertyRenames[it.name] !in amqpProperties } + val propertyNames = typeInformation.propertiesOrEmptyMap.mapNotNull { if (it.value.isCalculated) null else it.key } + beanProperties.removeIf { it.name !in propertyNames } } return beanProperties } diff --git a/client/jackson/src/test/kotlin/net/corda/client/jackson/JacksonSupportTest.kt b/client/jackson/src/test/kotlin/net/corda/client/jackson/JacksonSupportTest.kt index c047e7f8fd..8f71de6b5d 100644 --- a/client/jackson/src/test/kotlin/net/corda/client/jackson/JacksonSupportTest.kt +++ b/client/jackson/src/test/kotlin/net/corda/client/jackson/JacksonSupportTest.kt @@ -29,6 +29,7 @@ import net.corda.core.node.services.NetworkParametersService import net.corda.core.node.services.TransactionStorage import net.corda.core.serialization.CordaSerializable import net.corda.core.serialization.SerializedBytes +import net.corda.core.serialization.deserialize import net.corda.core.serialization.serialize import net.corda.core.transactions.CoreTransaction import net.corda.core.transactions.SignedTransaction @@ -658,6 +659,15 @@ class JacksonSupportTest(@Suppress("unused") private val name: String, factory: assertThat(mapper.convertValue(json)).isEqualTo(data) } + @Test + fun `LinearState where the linearId property does not match the backing field`() { + val funkyLinearState = FunkyLinearState(UniqueIdentifier()) + // As a sanity check, show that this is a valid CordaSerializable class + assertThat(funkyLinearState.serialize().deserialize()).isEqualTo(funkyLinearState) + val json = mapper.valueToTree(funkyLinearState) + assertThat(mapper.convertValue(json)).isEqualTo(funkyLinearState) + } + @Test fun `kotlin object`() { val json = mapper.valueToTree(KotlinObject) @@ -713,6 +723,11 @@ class JacksonSupportTest(@Suppress("unused") private val name: String, factory: val nonCtor: Int get() = value } + private data class FunkyLinearState(private val linearID: UniqueIdentifier) : LinearState { + override val linearId: UniqueIdentifier get() = linearID + override val participants: List get() = emptyList() + } + private object KotlinObject @CordaSerializable diff --git a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumper.kt b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumper.kt index 8e14b134bc..6d37c48776 100644 --- a/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumper.kt +++ b/node/src/main/kotlin/net/corda/node/services/rpc/CheckpointDumper.kt @@ -1,7 +1,6 @@ package net.corda.node.services.rpc import co.paralleluniverse.fibers.Stack -import co.paralleluniverse.strands.Strand import com.fasterxml.jackson.annotation.JsonAutoDetect import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility import com.fasterxml.jackson.annotation.JsonFormat @@ -363,9 +362,12 @@ class CheckpointDumper(private val checkpointStorage: CheckpointStorage, private override fun changeProperties(config: SerializationConfig, beanDesc: BeanDescription, beanProperties: MutableList): MutableList { - // Remove references to any node singletons - beanProperties.removeIf { it.type.isTypeOrSubTypeOf(SerializeAsToken::class.java) } - if (FlowLogic::class.java.isAssignableFrom(beanDesc.beanClass)) { + if (SerializeAsToken::class.java.isAssignableFrom(beanDesc.beanClass)) { + // Do not serialise node singletons + // TODO This will cause the singleton to appear as an empty object. Ideally we don't want it to appear at all but this will + // have to do for now. + beanProperties.clear() + } else if (FlowLogic::class.java.isAssignableFrom(beanDesc.beanClass)) { beanProperties.removeIf { it.type.isTypeOrSubTypeOf(ProgressTracker::class.java) || it.name == "_stateMachine" || it.name == "deprecatedPartySessionMap" } diff --git a/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt b/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt index ca2ff63d58..8aadc6957a 100644 --- a/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt +++ b/tools/shell/src/integration-test/kotlin/net/corda/tools/shell/InteractiveShellIntegrationTest.kt @@ -3,7 +3,6 @@ package net.corda.tools.shell import co.paralleluniverse.fibers.Suspendable import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.type.TypeFactory -import com.google.common.io.Files import com.jcraft.jsch.ChannelExec import com.jcraft.jsch.JSch import com.nhaarman.mockito_kotlin.any @@ -11,17 +10,25 @@ import com.nhaarman.mockito_kotlin.doAnswer import com.nhaarman.mockito_kotlin.mock import net.corda.client.jackson.JacksonSupport import net.corda.client.rpc.RPCException +import net.corda.core.contracts.* import net.corda.core.flows.* +import net.corda.core.identity.AbstractParty import net.corda.core.identity.Party +import net.corda.core.internal.createDirectories import net.corda.core.internal.div +import net.corda.core.internal.inputStream import net.corda.core.internal.list import net.corda.core.internal.messaging.InternalCordaRPCOps import net.corda.core.messaging.ClientRpcSslOptions import net.corda.core.messaging.CordaRPCOps import net.corda.core.messaging.startFlow +import net.corda.core.node.ServiceHub +import net.corda.core.transactions.LedgerTransaction +import net.corda.core.transactions.SignedTransaction +import net.corda.core.transactions.TransactionBuilder +import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.ProgressTracker import net.corda.core.utilities.getOrThrow -import net.corda.core.utilities.unwrap import net.corda.node.internal.NodeStartup import net.corda.node.services.Permissions import net.corda.node.services.Permissions.Companion.all @@ -34,6 +41,7 @@ import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.BOB_NAME import net.corda.testing.core.singleIdentity import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.NodeHandle import net.corda.testing.driver.driver import net.corda.testing.driver.internal.NodeHandleInternal import net.corda.testing.internal.useSslRpcOverrides @@ -49,9 +57,10 @@ import org.junit.Ignore import org.junit.Rule import org.junit.Test import org.junit.rules.TemporaryFolder -import java.util.zip.ZipFile +import java.util.* +import java.util.zip.ZipInputStream import javax.security.auth.x500.X500Principal -import kotlin.test.assertNotEquals +import kotlin.test.assertEquals import kotlin.test.assertNotNull import kotlin.test.assertTrue @@ -73,14 +82,8 @@ class InteractiveShellIntegrationTest { fun `shell should not log in with invalid credentials`() { val user = User("u", "p", setOf()) driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = "fake", password = "fake", - hostAndPort = node.rpcAddress) - InteractiveShell.startShell(conf) - + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell("fake", "fake", node.rpcAddress) assertThatThrownBy { InteractiveShell.nodeInfo() }.isInstanceOf(ActiveMQSecurityException::class.java) } } @@ -88,15 +91,9 @@ class InteractiveShellIntegrationTest { @Test fun `shell should log in with valid credentials`() { val user = User("u", "p", setOf()) - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress) - - InteractiveShell.startShell(conf) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node) InteractiveShell.nodeInfo() } } @@ -104,7 +101,6 @@ class InteractiveShellIntegrationTest { @Test fun `shell should log in with ssl`() { val user = User("mark", "dadada", setOf(all())) - var successful = false val (keyPair, cert) = createKeyPairAndSelfSignedTLSCertificate(testName) val keyStorePath = saveToKeyStore(tempFolder.root.toPath() / "keystore.jks", keyPair, cert) @@ -114,20 +110,10 @@ class InteractiveShellIntegrationTest { val clientSslOptions = ClientRpcSslOptions(trustStorePath, "password") driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { - startNode(rpcUsers = listOf(user), customOverrides = brokerSslOptions.useSslRpcOverrides()).getOrThrow().use { node -> - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress, - ssl = clientSslOptions) - - InteractiveShell.startShell(conf) - - InteractiveShell.nodeInfo() - successful = true - } + val node = startNode(rpcUsers = listOf(user), customOverrides = brokerSslOptions.useSslRpcOverrides()).getOrThrow() + startShell(node, clientSslOptions) + InteractiveShell.nodeInfo() } - assertThat(successful).isTrue() } @Test @@ -142,47 +128,33 @@ class InteractiveShellIntegrationTest { val clientSslOptions = ClientRpcSslOptions(trustStorePath, "password") driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { - startNode(rpcUsers = listOf(user), customOverrides = brokerSslOptions.useSslRpcOverrides()).getOrThrow().use { node -> - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress, - ssl = clientSslOptions) - - InteractiveShell.startShell(conf) - - assertThatThrownBy { InteractiveShell.nodeInfo() }.isInstanceOf(RPCException::class.java) - } + val node = startNode(rpcUsers = listOf(user), customOverrides = brokerSslOptions.useSslRpcOverrides()).getOrThrow() + startShell(node, clientSslOptions) + assertThatThrownBy { InteractiveShell.nodeInfo() }.isInstanceOf(RPCException::class.java) } } @Test fun `internal shell user should not be able to connect if node started with devMode=false`() { driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { - startNode().getOrThrow().use { node -> - val conf = (node as NodeHandleInternal).configuration.toShellConfig() - InteractiveShell.startShell(conf) - assertThatThrownBy { InteractiveShell.nodeInfo() }.isInstanceOf(ActiveMQSecurityException::class.java) - } + val node = startNode().getOrThrow() + val conf = (node as NodeHandleInternal).configuration.toShellConfig() + InteractiveShell.startShell(conf) + assertThatThrownBy { InteractiveShell.nodeInfo() }.isInstanceOf(ActiveMQSecurityException::class.java) } } @Ignore @Test fun `ssh runs flows via standalone shell`() { - val user = User("u", "p", setOf(Permissions.startFlow(), + val user = User("u", "p", setOf( + Permissions.startFlow(), Permissions.invokeRpc(CordaRPCOps::registeredFlows), - Permissions.invokeRpc(CordaRPCOps::nodeInfo))) - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress, - sshdPort = 2224) - - InteractiveShell.startShell(conf) + Permissions.invokeRpc(CordaRPCOps::nodeInfo) + )) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node, sshdPort = 2224) InteractiveShell.nodeInfo() val session = JSch().getSession("u", "localhost", 2224) @@ -200,7 +172,7 @@ class InteractiveShellIntegrationTest { val response = String(Streams.readAll(channel.inputStream)) - val linesWithDoneCount = response.lines().filter { line -> line.contains("Done") } + val linesWithDoneCount = response.lines().filter { line -> "Done" in line } channel.disconnect() session.disconnect() @@ -213,9 +185,11 @@ class InteractiveShellIntegrationTest { @Ignore @Test fun `ssh run flows via standalone shell over ssl to node`() { - val user = User("mark", "dadada", setOf(Permissions.startFlow(), + val user = User("mark", "dadada", setOf( + Permissions.startFlow(), Permissions.invokeRpc(CordaRPCOps::registeredFlows), - Permissions.invokeRpc(CordaRPCOps::nodeInfo)/*all()*/)) + Permissions.invokeRpc(CordaRPCOps::nodeInfo)/*all()*/ + )) val (keyPair, cert) = createKeyPairAndSelfSignedTLSCertificate(testName) val keyStorePath = saveToKeyStore(tempFolder.root.toPath() / "keystore.jks", keyPair, cert) @@ -226,14 +200,7 @@ class InteractiveShellIntegrationTest { var successful = false driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { startNode(rpcUsers = listOf(user), customOverrides = brokerSslOptions.useSslRpcOverrides()).getOrThrow().use { node -> - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress, - ssl = clientSslOptions, - sshdPort = 2223) - - InteractiveShell.startShell(conf) + startShell(node, clientSslOptions, sshdPort = 2223) InteractiveShell.nodeInfo() val session = JSch().getSession("mark", "localhost", 2223) @@ -251,7 +218,7 @@ class InteractiveShellIntegrationTest { val response = String(Streams.readAll(channel.inputStream)) - val linesWithDoneCount = response.lines().filter { line -> line.contains("Done") } + val linesWithDoneCount = response.lines().filter { line -> "Done" in line } channel.disconnect() session.disconnect() // TODO Simon make sure to close them @@ -263,174 +230,136 @@ class InteractiveShellIntegrationTest { } assertThat(successful).isTrue() - } } @Test fun `shell should start flow with fully qualified class name`() { val user = User("u", "p", setOf(all())) - var successful = false - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress) - InteractiveShell.startShell(conf) - - // setup and configure some mocks required by InteractiveShell.runFlowByNameFragment() - val output = mock { - on { println(any()) } doAnswer { - val line = it.arguments[0] - println("$line") - if ((line is String) && (line.startsWith("Flow completed with result:"))) - successful = true - } - } - val ansiProgressRenderer = mock { - on { render(any(), any()) } doAnswer { InteractiveShell.latch.countDown() } - } - InteractiveShell.runFlowByNameFragment( - "NoOpFlow", - "", output, node.rpc, ansiProgressRenderer) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node) + val (output, lines) = mockRenderPrintWriter() + InteractiveShell.runFlowByNameFragment(NoOpFlow::class.java.name, "", output, node.rpc, mockAnsiProgressRenderer()) + assertThat(lines.last()).startsWith("Flow completed with result:") } - assertThat(successful).isTrue() } @Test fun `shell should start flow with unique un-qualified class name`() { val user = User("u", "p", setOf(all())) - var successful = false - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress) - InteractiveShell.startShell(conf) - - // setup and configure some mocks required by InteractiveShell.runFlowByNameFragment() - val output = mock { - on { println(any()) } doAnswer { - val line = it.arguments[0] - println("$line") - if ((line is String) && (line.startsWith("Flow completed with result:"))) - successful = true - } - } - val ansiProgressRenderer = mock { - on { render(any(), any()) } doAnswer { InteractiveShell.latch.countDown() } - } - InteractiveShell.runFlowByNameFragment( - "NoOpFlowA", - "", output, node.rpc, ansiProgressRenderer) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node) + val (output, lines) = mockRenderPrintWriter() + InteractiveShell.runFlowByNameFragment("NoOpFlowA", "", output, node.rpc, mockAnsiProgressRenderer()) + assertThat(lines.last()).startsWith("Flow completed with result:") } - assertThat(successful).isTrue() } @Test fun `shell should fail to start flow with ambiguous class name`() { val user = User("u", "p", setOf(all())) - var successful = false - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress) - InteractiveShell.startShell(conf) - - // setup and configure some mocks required by InteractiveShell.runFlowByNameFragment() - val output = mock { - on { println(any()) } doAnswer { - val line = it.arguments[0] - println("$line") - if ((line is String) && (line.startsWith("Ambiguous name provided, please be more specific."))) - successful = true - } - } - val ansiProgressRenderer = mock { - on { render(any(), any()) } doAnswer { InteractiveShell.latch.countDown() } - } - InteractiveShell.runFlowByNameFragment( - "NoOpFlo", - "", output, node.rpc, ansiProgressRenderer) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node) + val (output, lines) = mockRenderPrintWriter() + InteractiveShell.runFlowByNameFragment("NoOpFlo", "", output, node.rpc, mockAnsiProgressRenderer()) + assertThat(lines.any { it.startsWith("Ambiguous name provided, please be more specific.") }).isTrue() } - assertThat(successful).isTrue() } @Test fun `shell should start flow with partially matching class name`() { val user = User("u", "p", setOf(all())) - var successful = false - driver(DriverParameters(notarySpecs = emptyList())) { - val nodeFuture = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true) - val node = nodeFuture.getOrThrow() - - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = node.rpcAddress) - InteractiveShell.startShell(conf) - - // setup and configure some mocks required by InteractiveShell.runFlowByNameFragment() - val output = mock { - on { println(any()) } doAnswer { - val line = it.arguments[0] - println("$line") - if ((line is String) && (line.startsWith("Flow completed with result"))) - successful = true - } - } - val ansiProgressRenderer = mock { - on { render(any(), any()) } doAnswer { InteractiveShell.latch.countDown() } - } - InteractiveShell.runFlowByNameFragment( - "Burble", - "", output, node.rpc, ansiProgressRenderer) + driver(DriverParameters(startNodesInProcess = true, notarySpecs = emptyList())) { + val node = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + startShell(node) + val (output, lines) = mockRenderPrintWriter() + InteractiveShell.runFlowByNameFragment("Burble", "", output, node.rpc, mockAnsiProgressRenderer()) + assertThat(lines.last()).startsWith("Flow completed with result") } - assertThat(successful).isTrue() } @Test fun `dumpCheckpoints creates zip with json file for suspended flow`() { val user = User("u", "p", setOf(all())) - driver(DriverParameters(notarySpecs = emptyList())) { - val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user), startInSameProcess = true).getOrThrow() - val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user), startInSameProcess = true).getOrThrow() + driver(DriverParameters(startNodesInProcess = true)) { + val aliceNode = startNode(providedName = ALICE_NAME, rpcUsers = listOf(user)).getOrThrow() + val bobNode = startNode(providedName = BOB_NAME, rpcUsers = listOf(user)).getOrThrow() bobNode.stop() - // create logs directory since the driver is not creating it - (aliceNode.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME).toFile().mkdir() + // Create logs directory since the driver is not creating it + (aliceNode.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME).createDirectories() - val conf = ShellConfiguration(commandsDirectory = Files.createTempDir().toPath(), - user = user.username, password = user.password, - hostAndPort = aliceNode.rpcAddress) - InteractiveShell.startShell(conf) - // setup and configure some mocks required by InteractiveShell.runFlowByNameFragment() - val output = mock { - on { println(any()) } doAnswer { - val line = it.arguments[0] - assertNotEquals("Please try 'man run' to learn what syntax is acceptable", line) - } + startShell(aliceNode) + + val linearId = UniqueIdentifier(id = UUID.fromString("7c0719f0-e489-46e8-bf3b-ee203156fc7c")) + aliceNode.rpc.startFlow( + ::FlowForCheckpointDumping, + MyState( + "some random string", + linearId, + listOf(aliceNode.nodeInfo.singleIdentity(), bobNode.nodeInfo.singleIdentity()) + ), + bobNode.nodeInfo.singleIdentity() + ) + + Thread.sleep(5000) + + val (output) = mockRenderPrintWriter() + InteractiveShell.runRPCFromString(listOf("dumpCheckpoints"), output, mock(), aliceNode.rpc as InternalCordaRPCOps, inputObjectMapper) + + val zipFile = (aliceNode.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME).list().first { "checkpoints_dump-" in it.toString() } + val json = ZipInputStream(zipFile.inputStream()).use { zip -> + zip.nextEntry + ObjectMapper().readTree(zip) } - aliceNode.rpc.startFlow(::SendFlow, bobNode.nodeInfo.singleIdentity()) + assertNotNull(json["flowId"].asText()) + assertEquals(FlowForCheckpointDumping::class.java.name, json["topLevelFlowClass"].asText()) + assertEquals(linearId.id.toString(), json["topLevelFlowLogic"]["myState"]["linearId"]["id"].asText()) + assertEquals(4, json["flowCallStackSummary"].size()) + assertEquals(4, json["flowCallStack"].size()) + val sendAndReceiveJson = json["suspendedOn"]["sendAndReceive"][0] + assertEquals(bobNode.nodeInfo.singleIdentity().toString(), sendAndReceiveJson["session"]["peer"].asText()) + assertEquals(SignedTransaction::class.qualifiedName, sendAndReceiveJson["sentPayloadType"].asText()) + } + } - InteractiveShell.runRPCFromString( - listOf("dumpCheckpoints"), output, mock(), aliceNode.rpc as InternalCordaRPCOps, inputObjectMapper) + private fun startShell(node: NodeHandle, ssl: ClientRpcSslOptions? = null, sshdPort: Int? = null) { + val user = node.rpcUsers[0] + startShell(user.username, user.password, node.rpcAddress, ssl, sshdPort) + } - // assert that the checkpoint dump zip has been created - val zip = (aliceNode.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME).list() - .find { it.toString().contains("checkpoints_dump-") } - assertNotNull(zip) - // assert that a json file has been created for the suspended flow - val json = ZipFile((zip!!).toFile()).entries().asSequence() - .find { it.name.contains(SendFlow::class.simpleName!!) } - assertNotNull(json) + private fun startShell(user: String, password: String, address: NetworkHostAndPort, ssl: ClientRpcSslOptions? = null, sshdPort: Int? = null) { + val conf = ShellConfiguration( + commandsDirectory = tempFolder.newFolder().toPath(), + user = user, + password = password, + hostAndPort = address, + ssl = ssl, + sshdPort = sshdPort + ) + InteractiveShell.startShell(conf) + } + + private fun mockRenderPrintWriter(): Pair> { + val lines = ArrayList() + val writer = mock { + on { println(any()) } doAnswer { + val line = it.getArgument(0, String::class.java) + println(">>> $line") + lines += line + Unit + } + } + return Pair(writer, lines) + } + + private fun mockAnsiProgressRenderer(): ANSIProgressRenderer { + return mock { + on { render(any(), any()) } doAnswer { InteractiveShell.latch.countDown() } } } @@ -438,7 +367,6 @@ class InteractiveShellIntegrationTest { val objectMapper = JacksonSupport.createNonRpcMapper() val tf = TypeFactory.defaultInstance().withClassLoader(classLoader) objectMapper.typeFactory = tf - return objectMapper } } @@ -470,23 +398,47 @@ class BurbleFlow : FlowLogic() { } } -@StartableByRPC @InitiatingFlow -class SendFlow(private val party: Party) : FlowLogic() { - override val progressTracker = ProgressTracker() +@StartableByRPC +class FlowForCheckpointDumping(private val myState: MyState, private val party: Party): FlowLogic() { + // Make sure any SerializeAsToken instances are not serialised + private var services: ServiceHub? = null + @Suspendable override fun call() { - initiateFlow(party).sendAndReceive("hi").unwrap { it } + services = serviceHub + val tx = TransactionBuilder(serviceHub.networkMapCache.notaryIdentities.first()).apply { + addOutputState(myState) + addCommand(MyContract.Create(), listOf(ourIdentity, party).map(Party::owningKey)) + } + val sessions = listOf(initiateFlow(party)) + val stx = serviceHub.signInitialTransaction(tx) + subFlow(CollectSignaturesFlow(stx, sessions)) + throw IllegalStateException("The test should not get here") } } -@InitiatedBy(SendFlow::class) -class ReceiveFlow(private val session: FlowSession) : FlowLogic() { - override val progressTracker = ProgressTracker() - @Suspendable +@InitiatedBy(FlowForCheckpointDumping::class) +class FlowForCheckpointDumpingResponder(private val session: FlowSession): FlowLogic() { override fun call() { - session.receive().unwrap { it } - session.send("hi") + val signTxFlow = object : SignTransactionFlow(session) { + override fun checkTransaction(stx: SignedTransaction) { + + } + } + subFlow(signTxFlow) + throw IllegalStateException("The test should not get here") } } +class MyContract : Contract { + class Create : CommandData + override fun verify(tx: LedgerTransaction) {} +} + +@BelongsToContract(MyContract::class) +data class MyState( + val data: String, + override val linearId: UniqueIdentifier, + override val participants: List +) : LinearState