From 222c5b9db8a85e3286f94da4bc952d078266f2c7 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Thu, 25 Jan 2018 20:48:20 +0000 Subject: [PATCH 1/2] CORDA-943 - Fix trader demo This is a multi issue problem 1. Fingerprinting of generics treats and differently, forcing the evolver to be used when not needed 2. However, the evolver is required sometimes as generics are not guaranteed to fingerprinting bi-directionally (thanks to type erasure of deeply nested generic types). However, with serialization now writing properties in a specific order, we need to ensure they're read back in that order before applying them to an evolved constructor so as to not corrupt the object reference cache --- .../net/corda/core/contracts/ContractState.kt | 1 - .../amqp/AMQPPrimitiveSerializer.kt | 2 +- .../serialization/amqp/AMQPSerializer.kt | 2 +- .../serialization/amqp/ArraySerializer.kt | 32 ++-- .../amqp/CollectionSerializer.kt | 4 +- .../amqp/CorDappCustomSerializer.kt | 2 +- .../serialization/amqp/CustomSerializer.kt | 2 +- .../amqp/DeserializationInput.kt | 74 +++++---- .../amqp/EnumEvolutionSerializer.kt | 2 +- .../serialization/amqp/EnumSerializer.kt | 2 +- .../serialization/amqp/EvolutionSerializer.kt | 79 ++++++---- .../serialization/amqp/MapSerializer.kt | 6 +- .../serialization/amqp/ObjectSerializer.kt | 4 +- .../serialization/amqp/PropertySerializer.kt | 10 +- .../internal/serialization/amqp/Schema.kt | 32 ++-- .../serialization/amqp/SerializationHelper.kt | 9 +- .../serialization/amqp/SerializationOutput.kt | 12 +- .../serialization/amqp/SerializerFactory.kt | 3 +- .../serialization/amqp/SingletonSerializer.kt | 2 +- .../carpenter/AMQPSchemaExtensions.kt | 7 +- .../serialization/amqp/EvolvabilityTests.kt | 1 + .../serialization/amqp/GenericsTests.kt | 146 +++++++++++++++--- 22 files changed, 295 insertions(+), 139 deletions(-) diff --git a/core/src/main/kotlin/net/corda/core/contracts/ContractState.kt b/core/src/main/kotlin/net/corda/core/contracts/ContractState.kt index f797254f21..b9f3d48fc4 100644 --- a/core/src/main/kotlin/net/corda/core/contracts/ContractState.kt +++ b/core/src/main/kotlin/net/corda/core/contracts/ContractState.kt @@ -1,7 +1,6 @@ package net.corda.core.contracts import net.corda.core.identity.AbstractParty -import net.corda.core.identity.Party import net.corda.core.serialization.CordaSerializable // DOCSTART 1 diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt index dc673940b1..8695c630f7 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt @@ -18,7 +18,7 @@ class AMQPPrimitiveSerializer(clazz: Class<*>) : AMQPSerializer { override fun writeClassInfo(output: SerializationOutput) { } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { if (obj is ByteArray) { data.putObject(Binary(obj)) } else { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt index e70b55d8fc..f306b31f21 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt @@ -30,7 +30,7 @@ interface AMQPSerializer { /** * Write the given object, with declared type, to the output. */ - fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) + fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int = 0) /** * Read the given object from the input. The envelope is provided in case the schema is required. diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt index 46046a88f2..7e3e51da97 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt @@ -45,12 +45,12 @@ open class ArraySerializer(override val type: Type, factory: SerializerFactory) } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { // Write described data.withDescribed(typeNotation.descriptor) { withList { for (entry in obj as Array<*>) { - output.writeObjectOrNull(entry, this, elementType) + output.writeObjectOrNull(entry, this, elementType, offset) } } } @@ -109,15 +109,15 @@ abstract class PrimArraySerializer(type: Type, factory: SerializerFactory) : Arr class PrimIntArraySerializer(factory: SerializerFactory) : PrimArraySerializer(IntArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } class PrimCharArraySerializer(factory: SerializerFactory) : PrimArraySerializer(CharArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } override fun List.toArrayOfType(type: Type): Any { @@ -132,35 +132,35 @@ class PrimCharArraySerializer(factory: SerializerFactory) : class PrimBooleanArraySerializer(factory: SerializerFactory) : PrimArraySerializer(BooleanArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } class PrimDoubleArraySerializer(factory: SerializerFactory) : PrimArraySerializer(DoubleArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } class PrimFloatArraySerializer(factory: SerializerFactory) : PrimArraySerializer(FloatArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } class PrimShortArraySerializer(factory: SerializerFactory) : PrimArraySerializer(ShortArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } class PrimLongArraySerializer(factory: SerializerFactory) : PrimArraySerializer(LongArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { - localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt index d39456cb9e..90a3b8fe54 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt @@ -66,12 +66,12 @@ class CollectionSerializer(val declaredType: ParameterizedType, factory: Seriali } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) { // Write described data.withDescribed(typeNotation.descriptor) { withList { for (entry in obj as Collection<*>) { - output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0]) + output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0], offset) } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt index 003da13268..12508753f6 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt @@ -64,7 +64,7 @@ class CorDappCustomSerializer( override fun writeClassInfo(output: SerializationOutput) {} - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { val proxy = uncheckedCast, SerializationCustomSerializer>(serializer).toProxy(obj) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt index fc0891e93a..143b114fdc 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt @@ -40,7 +40,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { */ override val revealSubclassesInSchema: Boolean get() = false - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { data.withDescribed(descriptor) { writeDescribedObject(uncheckedCast(obj), data, type, output) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt index d34ee9c12e..ce353f0140 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt @@ -1,5 +1,6 @@ package net.corda.nodeapi.internal.serialization.amqp +import com.google.common.primitives.Primitives import net.corda.core.internal.getStackTraceAsString import net.corda.core.serialization.SerializedBytes import net.corda.core.utilities.ByteSequence @@ -51,12 +52,11 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) { } @Throws(NotSerializableException::class) - inline fun deserialize(bytes: SerializedBytes): T = - deserialize(bytes, T::class.java) + inline fun deserialize(bytes: SerializedBytes): T = deserialize(bytes, T::class.java) @Throws(NotSerializableException::class) inline internal fun deserializeAndReturnEnvelope(bytes: SerializedBytes): ObjectAndEnvelope = - deserializeAndReturnEnvelope(bytes, T::class.java) + deserializeAndReturnEnvelope(bytes, T::class.java) @Throws(NotSerializableException::class) internal fun getEnvelope(bytes: ByteSequence): Envelope { @@ -106,41 +106,47 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) { ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)), envelope) } - internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type): Any? { - return if (obj == null) null else readObject(obj, schema, type) + internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type, offset: Int = 0): Any? { + return if (obj == null) null else readObject(obj, schema, type, offset) } - internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type): Any = - if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) { - // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference. - val objectIndex = (obj.described as UnsignedInteger).toInt() - if (objectIndex !in 0..objectHistory.size) - throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " + - "is outside of the bounds for the list of size: ${objectHistory.size}") + internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, offset: Int = 0): Any { + return if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) { + // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference. + val objectIndex = (obj.described as UnsignedInteger).toInt() + if (objectIndex !in 0..objectHistory.size) + throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " + + "is outside of the bounds for the list of size: ${objectHistory.size}") - val objectRetrieved = objectHistory[objectIndex] - if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) - throw NotSerializableException("Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}'") - objectRetrieved - } else { - val objectRead = when (obj) { - is DescribedType -> { - // Look up serializer in factory by descriptor - val serializer = serializerFactory.get(obj.descriptor, schemas) - if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) }) - throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " + - "expected to be of type $type but was ${serializer.type}") - serializer.readObject(obj.described, schemas, this) - } - is Binary -> obj.array - else -> obj // this will be the case for primitive types like [boolean] et al. - } - - // Store the reference in case we need it later on. - // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content - if (suitableForObjectReference(objectRead.javaClass)) objectHistory.add(objectRead) - objectRead + val objectRetrieved = objectHistory[objectIndex] + if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) { + throw NotSerializableException( + "Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}' " + + "@ ${objectIndex}") } + objectRetrieved + } else { + val objectRead = when (obj) { + is DescribedType -> { + // Look up serializer in factory by descriptor + val serializer = serializerFactory.get(obj.descriptor, schemas) + if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) }) + throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " + + "expected to be of type $type but was ${serializer.type}") + serializer.readObject(obj.described, schemas, this) + } + is Binary -> obj.array + else -> obj // this will be the case for primitive types like [boolean] et al. + } + + // Store the reference in case we need it later on. + // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content + if (suitableForObjectReference(objectRead.javaClass)) { + objectHistory.add(objectRead) + } + objectRead + } + } /** * Currently performs checks aimed at: diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt index 6e96d4bad8..e7d7010338 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt @@ -130,7 +130,7 @@ class EnumEvolutionSerializer( throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt index 6aaaf8701e..203c1f8ed9 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt @@ -39,7 +39,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Seria return fromOrd } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { if (obj !is Enum<*>) throw NotSerializableException("Serializing $obj as enum when it isn't") data.withDescribed(typeNotation.descriptor) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt index e3357b84d9..103e0d082e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt @@ -9,15 +9,24 @@ import kotlin.reflect.KFunction import kotlin.reflect.full.findAnnotation import kotlin.reflect.jvm.javaType + /** * Serializer for deserializing objects whose definition has changed since they * were serialised. + * + * @property oldReaders A linked map representing the properties of the object as they were serialized. Note + * this may contain properties that are no longer needed by the class. These *must* be read however to ensure + * any refferenced objects in the object stream are captured properly + * @property kotlinConstructor + * @property constructorArgs used to hold the properties as sent to the object's constructor. Passed in as a + * pre populated array as properties not present on the old constructor must be initialised in the factory */ class EvolutionSerializer( clazz: Type, factory: SerializerFactory, - val readers: List, - override val kotlinConstructor: KFunction?) : ObjectSerializer(clazz, factory) { + private val oldReaders: Map, + override val kotlinConstructor: KFunction?, + private val constructorArgs: Array) : ObjectSerializer(clazz, factory) { // explicitly set as empty to indicate it's unused by this type of serializer override val propertySerializers = PropertySerializersEvolution() @@ -27,13 +36,17 @@ class EvolutionSerializer( * when it was serialised and NOT how that class appears now * * @param type The jvm type of the parameter - * @param idx where in the parameter list this parameter falls. Required as the parameter - * order may have been changed and we need to know where into the list to look + * @param resultsIndex index into the constructor argument list where the read property + * should be placed * @param property object to read the actual property value */ - data class OldParam(val type: Type, val idx: Int, val property: PropertySerializer) { - fun readProperty(paramValues: List<*>, schemas: SerializationSchemas, input: DeserializationInput) = - property.readProperty(paramValues[idx], schemas, input) + data class OldParam(val type: Type, var resultsIndex: Int, val property: PropertySerializer) { + fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput, new: Array) = + property.readProperty(obj, schemas, input).apply { + if(resultsIndex >= 0) { + new[resultsIndex] = this + } + } } companion object { @@ -47,11 +60,11 @@ class EvolutionSerializer( * TODO: Type evolution * TODO: rename annotation */ - private fun getEvolverConstructor(type: Type, oldArgs: Map): KFunction? { + private fun getEvolverConstructor(type: Type, oldArgs: Map): KFunction? { val clazz: Class<*> = type.asClass()!! if (!isConcrete(clazz)) return null - val oldArgumentSet = oldArgs.map { Pair(it.key, it.value) } + val oldArgumentSet = oldArgs.map { Pair(it.key as String?, it.value.type) } var maxConstructorVersion = Integer.MIN_VALUE var constructor: KFunction? = null @@ -83,34 +96,42 @@ class EvolutionSerializer( fun make(old: CompositeType, new: ObjectSerializer, factory: SerializerFactory): AMQPSerializer { - val oldFieldToType = old.fields.map { - it.name as String? to it.getTypeAsClass(factory.classloader) as Type - }.toMap() + val readersAsSerialized = linkedMapOf( + *(old.fields.map { + val returnType = try { + it.getTypeAsClass(factory.classloader) + } catch (e: ClassNotFoundException) { + throw NotSerializableException(e.message) + } - val constructor = getEvolverConstructor(new.type, oldFieldToType) ?: + it.name to OldParam( + returnType, + -1, + PropertySerializer.make( + it.name, PublicPropertyReader(null), returnType, factory)) + }.toTypedArray()) + ) + + val constructor = getEvolverConstructor(new.type, readersAsSerialized) ?: throw NotSerializableException( "Attempt to deserialize an interface: ${new.type}. Serialized form is invalid.") - val oldArgs = mutableMapOf() - var idx = 0 - old.fields.forEach { - val returnType = it.getTypeAsClass(factory.classloader) - oldArgs[it.name] = OldParam( - returnType, idx++, PropertySerializer.make(it.name, PublicPropertyReader(null), returnType, factory)) - } + val constructorArgs = arrayOfNulls(constructor.parameters.size) - val readers = constructor.parameters.map { - oldArgs[it.name!!] ?: if (!it.type.isMarkedNullable) { + constructor.parameters.withIndex().forEach { + readersAsSerialized.get(it.value.name!!)?.apply { + this.resultsIndex = it.index + } ?: if (!it.value.type.isMarkedNullable) { throw NotSerializableException( - "New parameter ${it.name} is mandatory, should be nullable for evolution to worK") - } else null + "New parameter ${it.value.name} is mandatory, should be nullable for evolution to worK") + } } - return EvolutionSerializer(new.type, factory, readers, constructor) + return EvolutionSerializer(new.type, factory, readersAsSerialized, constructor, constructorArgs) } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } @@ -126,7 +147,11 @@ class EvolutionSerializer( override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any { if (obj !is List<*>) throw NotSerializableException("Body of described type is unexpected $obj") - return construct(readers.map { it?.readProperty(obj, schemas, input) }) + // *must* read all the parameters in the order they were serialized + oldReaders.values.zip(obj).map { it.first.readProperty(it.second, schemas, input, constructorArgs) } + + return javaConstructor?.newInstance(*(constructorArgs)) ?: + throw NotSerializableException("Attempt to deserialize an interface: $clazz. Serialized form is invalid.") } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt index ae7f73e6c9..bf7f9c6e84 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt @@ -73,7 +73,7 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) { obj.javaClass.checkSupportedMapType() // Write described data.withDescribed(typeNotation.descriptor) { @@ -81,8 +81,8 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial data.putMap() data.enter() for ((key, value) in obj as Map<*, *>) { - output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0]) - output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1]) + output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0], offset) + output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1], offset) } data.exit() // exit map } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt index 034d4b7f93..ad969bea35 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt @@ -52,13 +52,13 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ clazz.typeName }) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ clazz.typeName }) { // Write described data.withDescribed(typeNotation.descriptor) { // Write list withList { propertySerializers.serializationOrder.forEach { property -> - property.getter.writeProperty(obj, this, output) + property.getter.writeProperty(obj, this, output, offset+4) } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt index f749be0ca8..e0df115ff5 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt @@ -9,7 +9,7 @@ import java.lang.reflect.Type */ sealed class PropertySerializer(val name: String, val propertyReader: PropertyReader, val resolvedType: Type) { abstract fun writeClassInfo(output: SerializationOutput) - abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) + abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int = 0) abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? val type: String = generateType() @@ -80,8 +80,8 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe input.readObjectOrNull(obj, schemas, resolvedType) } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) = ifThrowsAppend({ nameForDebug }) { - output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType) + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) = ifThrowsAppend({ nameForDebug }) { + output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType, offset) } private val nameForDebug = "$name(${resolvedType.typeName})" @@ -100,7 +100,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe return if (obj is Binary) obj.array else obj } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) { + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) { val value = propertyReader.read(obj) if (value is ByteArray) { data.putObject(Binary(value)) @@ -123,7 +123,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe return if (obj == null) null else (obj as Short).toChar() } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) { + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) { val input = propertyReader.read(obj) if (input != null) data.putShort((input as Char).toShort()) else data.putNull() } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt index 596d04a2be..6d344a0f98 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt @@ -350,13 +350,15 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact // creating a unique string for a type which we then hash in the calling function above. private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher { - // We don't include Example and Example where type is ? or T in this otherwise we // generate different fingerprints for class Outer(val a: Inner) when serialising // and deserializing (assuming deserialization is occurring in a factory that didn't // serialise the object in the first place (and thus the cache lookup fails). This is also // true of Any, where we need Example and Example to have the same fingerprint - return if (type in alreadySeen && (type !is SerializerFactory.AnyType) && (type !is TypeVariable<*>)) { + return if ((type in alreadySeen) + && (type !is SerializerFactory.AnyType) + && (type !is TypeVariable<*>) + && (type !is WildcardType)) { hasher.putUnencodedChars(ALREADY_SEEN_HASH) } else { alreadySeen += type @@ -379,14 +381,20 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4) } } - // Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when - // looking at A and B from Example (remember we call this function recursively). When - // serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but - // when deserializing we only have the wildcard placeholder ?, or AnyType + // Previously, we drew a distinction between TypeVariable, Wildcard, and AnyType, changing + // the signature of the fingerprinted object. This, however, doesn't work as it breaks bi- + // directional fingerprints. That is, fingerprinting a concrete instance of a generic + // type (Example), creates a different fingerprint from the generic type itself (Example) // - // Note, TypeVariable<*> used to be encoded as TYPE_VARIABLE_HASH but that again produces a - // differing fingerprint on serialisation and deserialization + // On serialization Example is treated as Example, a TypeVariable + // On deserialisation it is seen as Example, A wildcard *and* a TypeVariable + // Note: AnyType is a special case of WildcarcType used in other parts of the + // serializer so both cases need to be dealt with here + // + // If we treat these types as fundamentally different and alter the fingerprint we will + // end up breaking into the evolver when we shouldn't or, worse, evoking the carpenter. is SerializerFactory.AnyType, + is WildcardType, is TypeVariable<*> -> { hasher.putUnencodedChars("?").putUnencodedChars(ANY_TYPE_HASH) } @@ -418,9 +426,10 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta } } // Hash the element type + some array hash - is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen, - hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH) - // TODO: include bounds + is GenericArrayType -> { + fingerprintForType(type.genericComponentType, contextType, alreadySeen, + hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH) + } is WildcardType -> { hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH) } @@ -446,6 +455,7 @@ private fun fingerprintForObject( offset: Int = 0): Hasher { // Hash the class + properties + interfaces val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type") + propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory) .serializationOrder .fold(hasher.putUnencodedChars(name)) { orig, prop -> diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt index 98274bd638..92aa09fe57 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt @@ -62,11 +62,12 @@ internal fun constructorForDeserialization(type: Type): KFunction? { } /** - * Identifies the properties to be used during serialization by attempting to find those that match the parameters to the - * deserialization constructor, if the class is concrete. If it is abstract, or an interface, then use all the properties. + * Identifies the properties to be used during serialization by attempting to find those that match the parameters + * to the deserialization constructor, if the class is concrete. If it is abstract, or an interface, then use all + * the properties. * - * Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters have - * names accessible via reflection. + * Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters + * have names accessible via reflection. */ internal fun propertiesForSerialization( kotlinConstructor: KFunction?, diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt index 901df19ac2..ff20b8ab68 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt @@ -86,15 +86,15 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory data.putObject(transformsSchema) } - internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type) { + internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type, offset: Int) { if (obj == null) { data.putNull() } else { - writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type) + writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type, offset) } } - internal fun writeObject(obj: Any, data: Data, type: Type) { + internal fun writeObject(obj: Any, data: Data, type: Type, offset: Int = 0) { val serializer = serializerFactory.get(obj.javaClass, type) if (serializer !in serializerHistory) { serializerHistory.add(serializer) @@ -103,11 +103,13 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory val retrievedRefCount = objectHistory[obj] if (retrievedRefCount == null) { - serializer.writeObject(obj, data, type, this) + serializer.writeObject(obj, data, type, this, offset) // Important to do it after serialization such that dependent object will have preceding reference numbers // assigned to them first as they will be first read from the stream on receiving end. // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content - if (suitableForObjectReference(obj.javaClass)) objectHistory.put(obj, objectHistory.size) + if (suitableForObjectReference(obj.javaClass)) { + objectHistory.put(obj, objectHistory.size) + } } else { data.writeReferencedObject(ReferencedObject(retrievedRefCount)) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt index 75f9bf0ded..7bd8178a6d 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt @@ -112,7 +112,7 @@ open class SerializerFactory( */ // TODO: test GenericArrayType private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>, - declaredType: Type) : Type? = when (declaredType) { + declaredType: Type): Type? = when (declaredType) { is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType) // Nothing to infer, otherwise we'd have ParameterizedType is Class<*> -> actualClass @@ -218,7 +218,6 @@ open class SerializerFactory( for (typeNotation in schemaAndDescriptor.schemas.schema.types) { try { val serialiser = processSchemaEntry(typeNotation) - // if we just successfully built a serializer for the type but the type fingerprint // doesn't match that of the serialised object then we are dealing with different // instance of the class, as such we need to build an EvolutionSerializer diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt index ac226008f7..7a25baf7cd 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt @@ -22,7 +22,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto output.writeTypeNotations(typeNotation) } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { data.withDescribed(typeNotation.descriptor) { data.putBoolean(false) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt index 6117695b77..b2d3b2d598 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt @@ -124,7 +124,12 @@ fun AMQPField.getTypeAsClass(classloader: ClassLoader) = typeStrToType[Pair(type "string" -> String::class.java "binary" -> ByteArray::class.java "*" -> if (requires.isEmpty()) Any::class.java else classloader.loadClass(requires[0]) - else -> classloader.loadClass(type) + else -> { + classloader.loadClass( + if (type.endsWith("")) { + type.substring(0, type.length-3) + } else type) + } } fun AMQPField.validateType(classloader: ClassLoader) = when (type) { diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt index 6d084408f3..164feb2bab 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt @@ -6,6 +6,7 @@ import net.corda.testing.common.internal.ProjectStructure.projectRootDir import org.junit.Test import java.io.File import java.io.NotSerializableException +import java.net.URI import kotlin.test.assertEquals // To regenerate any of the binary test files do the following diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index bd858bb103..d12a5c4056 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -1,14 +1,30 @@ package net.corda.nodeapi.internal.serialization.amqp +import net.corda.core.contracts.* import net.corda.core.serialization.SerializedBytes import net.corda.nodeapi.internal.serialization.AllWhitelist import net.corda.testing.common.internal.ProjectStructure.projectRootDir import org.junit.Test +import net.corda.core.identity.AbstractParty +import net.corda.core.identity.CordaX500Name +import net.corda.core.identity.Party +import net.corda.core.transactions.WireTransaction +import net.corda.testing.core.TestIdentity +import org.hibernate.Transaction import java.io.File import java.net.URI +import java.util.* import java.util.concurrent.ConcurrentHashMap import kotlin.test.assertEquals +data class TestContractState( + override val participants: List +) : ContractState + +class TestAttachmentConstraint : AttachmentConstraint { + override fun isSatisfiedBy(attachment: Attachment) = true +} + class GenericsTests { companion object { val VERBOSE = false @@ -16,6 +32,8 @@ class GenericsTests { @Suppress("UNUSED") var localPath = projectRootDir.toUri().resolve( "node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp") + + val miniCorp = TestIdentity(CordaX500Name("MiniCorp", "London", "GB")) } private fun printSeparator() = if (VERBOSE) println("\n\n-------------------------------------------\n\n") else Unit @@ -252,29 +270,119 @@ class GenericsTests { File(GenericsTests::class.java.getResource(resource).toURI()).readBytes())).t) } - interface DifferentBounds { - fun go() + data class StateAndString(val state: TransactionState<*>, val ref: String) + data class GenericStateAndString(val state: TransactionState, val ref: String) + + // + // If this doesn't blow up all is fine + private fun fingerprintingDiffersStrip(state: Any) { + class cl : ClassLoader() + + val m = ClassLoader::class.java.getDeclaredMethod("findLoadedClass", *arrayOf>(String::class.java)) + m.isAccessible = true + + val factory1 = testDefaultFactory() + factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state) + + // attempt at having a class loader without some of the derived non core types loaded and thus + // possibly altering how we serialise things + val altClassLoader = cl() + + val factory2 = SerializerFactory(AllWhitelist, altClassLoader) + factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + val ser2 = TestSerializationOutput(VERBOSE, factory2).serializeAndReturnSchema(state) + + // now deserialise those objects + val factory3 = testDefaultFactory() + factory3.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + val des1 = DeserializationInput(factory3).deserializeAndReturnEnvelope(ser1.obj) + + val factory4 = SerializerFactory(AllWhitelist, cl()) + factory4.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + val des2 = DeserializationInput(factory4).deserializeAndReturnEnvelope(ser2.obj) + } @Test - fun differentBounds() { - data class A (val a: Int): DifferentBounds { - override fun go() { - println(a) - } - } + fun fingerprintingDiffers() { + val state = TransactionState ( + TestContractState(listOf(miniCorp.party)), + "wibble", miniCorp.party, + encumbrance = null, + constraint = TestAttachmentConstraint()) - data class G(val b: T) + val sas = StateAndString(state, "wibble") - val factorys = listOf( - SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()), - SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())) - - val ser = SerializationOutput(factorys[0]) - - ser.serialize(G(A(10))).apply { - factorys.forEach { - } - } + fingerprintingDiffersStrip(sas) } + + @Test + fun fingerprintingDiffersList() { + val state = TransactionState ( + TestContractState(listOf(miniCorp.party)), + "wibble", miniCorp.party, + encumbrance = null, + constraint = TestAttachmentConstraint()) + + val sas = StateAndString(state, "wibble") + + fingerprintingDiffersStrip(Collections.singletonList(sas)) + } + + + // + // Force object to be serialised as Example and deserialized as Example + // + @Test + fun fingerprintingDiffersListLoaded() { + // + // using this wrapper class we force the object to be serialised as + // net.corda.core.contracts.TransactionState + // + data class TransactionStateWrapper (val o: List>) + + val state = TransactionState ( + TestContractState(listOf(miniCorp.party)), + "wibble", miniCorp.party, + encumbrance = null, + constraint = TestAttachmentConstraint()) + + val sas = GenericStateAndString(state, "wibble") + + val factory1 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + + factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer) + + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema( + TransactionStateWrapper(Collections.singletonList(sas))) + + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(sas.ref, des1.obj.o.firstOrNull()?.ref ?: "WILL NOT MATCH") + } + + @Test + fun anotherTry() { + open class BaseState(val a : Int) + class DState(a: Int) : BaseState(a) + data class LTransactionState constructor(val data: T) + data class StateWrapper(val state: LTransactionState) + + val factory1 = testDefaultFactoryNoEvolution() + + val state = LTransactionState(DState(1020304)) + val stateAndString = StateWrapper(state) + + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString) + + //val factory2 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.data.a, des1.obj.state.data.a) + } + } From 00b90a98fbabc8b7f6ed55634c0cf08e2fee0105 Mon Sep 17 00:00:00 2001 From: Katelyn Baker Date: Thu, 1 Feb 2018 12:19:32 +0000 Subject: [PATCH 2/2] CORDA-943 - Cope with multiple generics at str->type conversion in AMQP Also fixes an odd bug where the inferred type of a getter wasn't matching the constructor parameter type because that was still unbounded and seen as T, looking at the raw type allows us to inspect this properly --- .../amqp/AMQPPrimitiveSerializer.kt | 2 +- .../serialization/amqp/AMQPSerializer.kt | 2 +- .../serialization/amqp/ArraySerializer.kt | 46 ++++--- .../amqp/CollectionSerializer.kt | 14 +- .../amqp/CorDappCustomSerializer.kt | 2 +- .../serialization/amqp/CustomSerializer.kt | 2 +- .../amqp/DeserializationInput.kt | 68 +++++----- .../amqp/EnumEvolutionSerializer.kt | 2 +- .../serialization/amqp/EnumSerializer.kt | 2 +- .../serialization/amqp/MapSerializer.kt | 11 +- .../serialization/amqp/ObjectSerializer.kt | 9 +- .../serialization/amqp/PropertySerializer.kt | 10 +- .../internal/serialization/amqp/Schema.kt | 27 ++-- .../serialization/amqp/SerializationHelper.kt | 21 ++- .../serialization/amqp/SerializationOutput.kt | 8 +- .../serialization/amqp/SingletonSerializer.kt | 2 +- .../carpenter/AMQPSchemaExtensions.kt | 6 +- .../serialization/amqp/GenericsTests.kt | 120 +++++++++++++++++- 18 files changed, 254 insertions(+), 100 deletions(-) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt index 8695c630f7..27db491d33 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt @@ -18,7 +18,7 @@ class AMQPPrimitiveSerializer(clazz: Class<*>) : AMQPSerializer { override fun writeClassInfo(output: SerializationOutput) { } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { if (obj is ByteArray) { data.putObject(Binary(obj)) } else { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt index f306b31f21..9dfcd03cb7 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt @@ -30,7 +30,7 @@ interface AMQPSerializer { /** * Write the given object, with declared type, to the output. */ - fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int = 0) + fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int = 0) /** * Read the given object from the input. The envelope is provided in case the schema is required. diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt index 7e3e51da97..b6b20a204e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt @@ -45,12 +45,12 @@ open class ArraySerializer(override val type: Type, factory: SerializerFactory) } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { // Write described data.withDescribed(typeNotation.descriptor) { withList { for (entry in obj as Array<*>) { - output.writeObjectOrNull(entry, this, elementType, offset) + output.writeObjectOrNull(entry, this, elementType, debugIndent) } } } @@ -109,15 +109,19 @@ abstract class PrimArraySerializer(type: Type, factory: SerializerFactory) : Arr class PrimIntArraySerializer(factory: SerializerFactory) : PrimArraySerializer(IntArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } class PrimCharArraySerializer(factory: SerializerFactory) : PrimArraySerializer(CharArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { (obj as CharArray).forEach { + output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } override fun List.toArrayOfType(type: Type): Any { @@ -132,35 +136,45 @@ class PrimCharArraySerializer(factory: SerializerFactory) : class PrimBooleanArraySerializer(factory: SerializerFactory) : PrimArraySerializer(BooleanArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } class PrimDoubleArraySerializer(factory: SerializerFactory) : PrimArraySerializer(DoubleArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } class PrimFloatArraySerializer(factory: SerializerFactory) : PrimArraySerializer(FloatArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } class PrimShortArraySerializer(factory: SerializerFactory) : PrimArraySerializer(ShortArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } class PrimLongArraySerializer(factory: SerializerFactory) : PrimArraySerializer(LongArray::class.java, factory) { - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { - localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } } + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { + localWriteObject(data) { + (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) } + } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt index 90a3b8fe54..bd19e98599 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt @@ -66,18 +66,26 @@ class CollectionSerializer(val declaredType: ParameterizedType, factory: Seriali } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) { + override fun writeObject( + obj: Any, + data: Data, + type: Type, + output: SerializationOutput, + debugIndent: Int) = ifThrowsAppend({ declaredType.typeName }) { // Write described data.withDescribed(typeNotation.descriptor) { withList { for (entry in obj as Collection<*>) { - output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0], offset) + output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0], debugIndent) } } } } - override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) { + override fun readObject( + obj: Any, + schemas: SerializationSchemas, + input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) { // TODO: Can we verify the entries in the list? concreteBuilder((obj as List<*>).map { input.readObjectOrNull(it, schemas, declaredType.actualTypeArguments[0]) }) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt index 12508753f6..161c18d4cc 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt @@ -64,7 +64,7 @@ class CorDappCustomSerializer( override fun writeClassInfo(output: SerializationOutput) {} - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { val proxy = uncheckedCast, SerializationCustomSerializer>(serializer).toProxy(obj) diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt index 143b114fdc..f2ee28d01b 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt @@ -40,7 +40,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { */ override val revealSubclassesInSchema: Boolean get() = false - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { data.withDescribed(descriptor) { writeDescribedObject(uncheckedCast(obj), data, type, output) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt index ce353f0140..ac61820218 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt @@ -1,6 +1,5 @@ package net.corda.nodeapi.internal.serialization.amqp -import com.google.common.primitives.Primitives import net.corda.core.internal.getStackTraceAsString import net.corda.core.serialization.SerializedBytes import net.corda.core.utilities.ByteSequence @@ -56,7 +55,7 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) { @Throws(NotSerializableException::class) inline internal fun deserializeAndReturnEnvelope(bytes: SerializedBytes): ObjectAndEnvelope = - deserializeAndReturnEnvelope(bytes, T::class.java) + deserializeAndReturnEnvelope(bytes, T::class.java) @Throws(NotSerializableException::class) internal fun getEnvelope(bytes: ByteSequence): Envelope { @@ -110,43 +109,42 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) { return if (obj == null) null else readObject(obj, schema, type, offset) } - internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, offset: Int = 0): Any { - return if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) { - // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference. - val objectIndex = (obj.described as UnsignedInteger).toInt() - if (objectIndex !in 0..objectHistory.size) - throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " + - "is outside of the bounds for the list of size: ${objectHistory.size}") + internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, debugIndent: Int = 0): Any = + if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) { + // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference. + val objectIndex = (obj.described as UnsignedInteger).toInt() + if (objectIndex !in 0..objectHistory.size) + throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " + + "is outside of the bounds for the list of size: ${objectHistory.size}") - val objectRetrieved = objectHistory[objectIndex] - if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) { - throw NotSerializableException( - "Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}' " + - "@ ${objectIndex}") - } - objectRetrieved - } else { - val objectRead = when (obj) { - is DescribedType -> { - // Look up serializer in factory by descriptor - val serializer = serializerFactory.get(obj.descriptor, schemas) - if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) }) - throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " + - "expected to be of type $type but was ${serializer.type}") - serializer.readObject(obj.described, schemas, this) + val objectRetrieved = objectHistory[objectIndex] + if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) { + throw NotSerializableException( + "Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}' " + + "@ ${objectIndex}") + } + objectRetrieved + } else { + val objectRead = when (obj) { + is DescribedType -> { + // Look up serializer in factory by descriptor + val serializer = serializerFactory.get(obj.descriptor, schemas) + if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) }) + throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " + + "expected to be of type $type but was ${serializer.type}") + serializer.readObject(obj.described, schemas, this) + } + is Binary -> obj.array + else -> obj // this will be the case for primitive types like [boolean] et al. } - is Binary -> obj.array - else -> obj // this will be the case for primitive types like [boolean] et al. - } - // Store the reference in case we need it later on. - // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content - if (suitableForObjectReference(objectRead.javaClass)) { - objectHistory.add(objectRead) + // Store the reference in case we need it later on. + // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content + if (suitableForObjectReference(objectRead.javaClass)) { + objectHistory.add(objectRead) + } + objectRead } - objectRead - } - } /** * Currently performs checks aimed at: diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt index e7d7010338..a9bc0916b1 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt @@ -130,7 +130,7 @@ class EnumEvolutionSerializer( throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { throw UnsupportedOperationException("It should be impossible to write an evolution serializer") } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt index 203c1f8ed9..5678f094ed 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt @@ -39,7 +39,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Seria return fromOrd } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { if (obj !is Enum<*>) throw NotSerializableException("Serializing $obj as enum when it isn't") data.withDescribed(typeNotation.descriptor) { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt index bf7f9c6e84..5472074d62 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt @@ -73,7 +73,12 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) { + override fun writeObject( + obj: Any, + data: Data, + type: Type, + output: SerializationOutput, + debugIndent: Int) = ifThrowsAppend({ declaredType.typeName }) { obj.javaClass.checkSupportedMapType() // Write described data.withDescribed(typeNotation.descriptor) { @@ -81,8 +86,8 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial data.putMap() data.enter() for ((key, value) in obj as Map<*, *>) { - output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0], offset) - output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1], offset) + output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0], debugIndent) + output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1], debugIndent) } data.exit() // exit map } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt index ad969bea35..fa723fd3de 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt @@ -52,13 +52,18 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS } } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ clazz.typeName }) { + override fun writeObject( + obj: Any, + data: Data, + type: Type, + output: SerializationOutput, + debugIndent: Int) = ifThrowsAppend({ clazz.typeName }) { // Write described data.withDescribed(typeNotation.descriptor) { // Write list withList { propertySerializers.serializationOrder.forEach { property -> - property.getter.writeProperty(obj, this, output, offset+4) + property.getter.writeProperty(obj, this, output, debugIndent+1) } } } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt index e0df115ff5..f9fa3d8f83 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt @@ -9,7 +9,7 @@ import java.lang.reflect.Type */ sealed class PropertySerializer(val name: String, val propertyReader: PropertyReader, val resolvedType: Type) { abstract fun writeClassInfo(output: SerializationOutput) - abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int = 0) + abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int = 0) abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? val type: String = generateType() @@ -80,8 +80,8 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe input.readObjectOrNull(obj, schemas, resolvedType) } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) = ifThrowsAppend({ nameForDebug }) { - output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType, offset) + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) = ifThrowsAppend({ nameForDebug }) { + output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType, debugIndent) } private val nameForDebug = "$name(${resolvedType.typeName})" @@ -100,7 +100,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe return if (obj is Binary) obj.array else obj } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) { + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) { val value = propertyReader.read(obj) if (value is ByteArray) { data.putObject(Binary(value)) @@ -123,7 +123,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe return if (obj == null) null else (obj as Short).toChar() } - override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) { + override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) { val input = propertyReader.read(obj) if (input != null) data.putShort((input as Char).toShort()) else data.putNull() } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt index 6d344a0f98..372cfed9be 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt @@ -349,7 +349,7 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact // This method concatenates various elements of the types recursively as unencoded strings into the hasher, effectively // creating a unique string for a type which we then hash in the calling function above. private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet, - hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher { + hasher: Hasher, factory: SerializerFactory, debugIndent: Int = 1): Hasher { // We don't include Example and Example where type is ? or T in this otherwise we // generate different fingerprints for class Outer(val a: Inner) when serialising // and deserializing (assuming deserialization is occurring in a factory that didn't @@ -372,23 +372,23 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta hasher.putUnencodedChars(clazz.name) } else { hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) { - fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4) + fingerprintForObject(type, type, alreadySeen, hasher, factory, debugIndent+1) } } // ... and concatenate the type data for each parameter type. type.actualTypeArguments.fold(startingHash) { orig, paramType -> - fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4) + fingerprintForType(paramType, type, alreadySeen, orig, factory, debugIndent+1) } } - // Previously, we drew a distinction between TypeVariable, Wildcard, and AnyType, changing + // Previously, we drew a distinction between TypeVariable, WildcardType, and AnyType, changing // the signature of the fingerprinted object. This, however, doesn't work as it breaks bi- // directional fingerprints. That is, fingerprinting a concrete instance of a generic // type (Example), creates a different fingerprint from the generic type itself (Example) // // On serialization Example is treated as Example, a TypeVariable - // On deserialisation it is seen as Example, A wildcard *and* a TypeVariable - // Note: AnyType is a special case of WildcarcType used in other parts of the + // On deserialisation it is seen as Example, A WildcardType *and* a TypeVariable + // Note: AnyType is a special case of WildcardType used in other parts of the // serializer so both cases need to be dealt with here // // If we treat these types as fundamentally different and alter the fingerprint we will @@ -400,7 +400,7 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta } is Class<*> -> { if (type.isArray) { - fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, offset+4) + fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, debugIndent+1) .putUnencodedChars(ARRAY_HASH) } else if (SerializerFactory.isPrimitive(type)) { hasher.putUnencodedChars(type.name) @@ -420,7 +420,7 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta // to the CorDapp but maybe reference to the JAR in the short term. hasher.putUnencodedChars(type.name) } else { - fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4) + fingerprintForObject(type, type, alreadySeen, hasher, factory, debugIndent+1) } } } @@ -428,10 +428,7 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta // Hash the element type + some array hash is GenericArrayType -> { fingerprintForType(type.genericComponentType, contextType, alreadySeen, - hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH) - } - is WildcardType -> { - hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH) + hasher, factory, debugIndent+1).putUnencodedChars(ARRAY_HASH) } else -> throw NotSerializableException("Don't know how to hash") } @@ -452,17 +449,17 @@ private fun fingerprintForObject( alreadySeen: MutableSet, hasher: Hasher, factory: SerializerFactory, - offset: Int = 0): Hasher { + debugIndent: Int = 0): Hasher { // Hash the class + properties + interfaces val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type") propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory) .serializationOrder .fold(hasher.putUnencodedChars(name)) { orig, prop -> - fingerprintForType(prop.getter.resolvedType, type, alreadySeen, orig, factory, offset+4) + fingerprintForType(prop.getter.resolvedType, type, alreadySeen, orig, factory, debugIndent+1) .putUnencodedChars(prop.getter.name) .putUnencodedChars(if (prop.getter.mandatory) NOT_NULLABLE_HASH else NULLABLE_HASH) } - interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, offset+4) } + interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, debugIndent+4) } return hasher } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt index 92aa09fe57..9dd0fb30cf 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt @@ -114,8 +114,8 @@ internal fun propertiesForSerializationFromConstructor( val returnType = resolveTypeVariables(getter.genericReturnType, type) if (!constructorParamTakesReturnTypeOfGetter(returnType, getter.genericReturnType, param.value)) { throw NotSerializableException( - "Property type $returnType for $name of $clazz differs from constructor parameter " - + "type ${param.value.type.javaType}") + "Property type '$returnType' for '$name' of '$clazz' differs from constructor parameter " + + "type '${param.value.type.javaType}'") } Pair(PublicPropertyReader(getter), returnType) @@ -166,9 +166,20 @@ private fun propertiesForSerializationFromSetters( } } -private fun constructorParamTakesReturnTypeOfGetter(getterReturnType: Type, rawGetterReturnType: Type, param: KParameter): Boolean { - val typeToken = TypeToken.of(param.type.javaType) - return typeToken.isSupertypeOf(getterReturnType) || typeToken.isSupertypeOf(rawGetterReturnType) +private fun constructorParamTakesReturnTypeOfGetter( + getterReturnType: Type, + rawGetterReturnType: Type, + param: KParameter): Boolean { + val paramToken = TypeToken.of(param.type.javaType) + val rawParamType = TypeToken.of(paramToken.rawType) + + return paramToken.isSupertypeOf(getterReturnType) + || paramToken.isSupertypeOf(rawGetterReturnType) + // cope with the case where the constructor parameter is a generic type (T etc) but we + // can discover it's raw type. When bounded this wil be the bounding type, unbounded + // generics this will be object + || rawParamType.isSupertypeOf(getterReturnType) + || rawParamType.isSupertypeOf(rawGetterReturnType) } private fun propertiesForSerializationFromAbstract( diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt index ff20b8ab68..94b17f6efd 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt @@ -86,15 +86,15 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory data.putObject(transformsSchema) } - internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type, offset: Int) { + internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type, debugIndent: Int) { if (obj == null) { data.putNull() } else { - writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type, offset) + writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type, debugIndent) } } - internal fun writeObject(obj: Any, data: Data, type: Type, offset: Int = 0) { + internal fun writeObject(obj: Any, data: Data, type: Type, debugIndent: Int = 0) { val serializer = serializerFactory.get(obj.javaClass, type) if (serializer !in serializerHistory) { serializerHistory.add(serializer) @@ -103,7 +103,7 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory val retrievedRefCount = objectHistory[obj] if (retrievedRefCount == null) { - serializer.writeObject(obj, data, type, this, offset) + serializer.writeObject(obj, data, type, this, debugIndent) // Important to do it after serialization such that dependent object will have preceding reference numbers // assigned to them first as they will be first read from the stream on receiving end. // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt index 7a25baf7cd..0c70a18935 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt @@ -22,7 +22,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto output.writeTypeNotations(typeNotation) } - override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) { + override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) { data.withDescribed(typeNotation.descriptor) { data.putBoolean(false) } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt index b2d3b2d598..1608a22e83 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt @@ -126,9 +126,9 @@ fun AMQPField.getTypeAsClass(classloader: ClassLoader) = typeStrToType[Pair(type "*" -> if (requires.isEmpty()) Any::class.java else classloader.loadClass(requires[0]) else -> { classloader.loadClass( - if (type.endsWith("")) { - type.substring(0, type.length-3) - } else type) + if (type.endsWith("?>")) { + type.substring(0, type.indexOf('<')) + } else type) } } diff --git a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt index d12a5c4056..60830cda88 100644 --- a/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt +++ b/node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt @@ -27,7 +27,7 @@ class TestAttachmentConstraint : AttachmentConstraint { class GenericsTests { companion object { - val VERBOSE = false + val VERBOSE = true @Suppress("UNUSED") var localPath = projectRootDir.toUri().resolve( @@ -365,7 +365,7 @@ class GenericsTests { } @Test - fun anotherTry() { + fun nestedGenericsWithBound() { open class BaseState(val a : Int) class DState(a: Int) : BaseState(a) data class LTransactionState constructor(val data: T) @@ -385,4 +385,120 @@ class GenericsTests { assertEquals(state.data.a, des1.obj.state.data.a) } + @Test + fun nestedMultiGenericsWithBound() { + open class BaseState(val a : Int) + class DState(a: Int) : BaseState(a) + class EState(a: Int, val msg: String) : BaseState(a) + + data class LTransactionState (val data: T1, val context: T2) + data class StateWrapper(val state: LTransactionState) + + val factory1 = testDefaultFactoryNoEvolution() + + val state = LTransactionState(DState(1020304), EState(5060708, msg = "thigns")) + val stateAndString = StateWrapper(state) + + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString) + + //val factory2 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.data.a, des1.obj.state.data.a) + assertEquals(state.context.a, des1.obj.state.context.a) + } + + @Test + fun nestedMultiGenericsNoBound() { + open class BaseState(val a : Int) + class DState(a: Int) : BaseState(a) + class EState(a: Int, val msg: String) : BaseState(a) + + data class LTransactionState (val data: T1, val context: T2) + data class StateWrapper(val state: LTransactionState) + + val factory1 = testDefaultFactoryNoEvolution() + + val state = LTransactionState(DState(1020304), EState(5060708, msg = "things")) + val stateAndString = StateWrapper(state) + + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString) + + //val factory2 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.data.a, des1.obj.state.data.a) + assertEquals(state.context.a, des1.obj.state.context.a) + assertEquals(state.context.msg, des1.obj.state.context.msg) + } + + @Test + fun baseClassInheritedButNotOverriden() { + val factory1 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + + open class BaseState(open val a : T1, open val b: T2) + class DState(a: T1, b: T2) : BaseState(a, b) + + val state = DState(100, "hello") + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state) + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.a, des1.obj.a) + assertEquals(state.b, des1.obj.b) + + class DState2(a: T1, b: T2, val c: T3) : BaseState(a, b) + + val state2 = DState2(100, "hello", 100L) + val ser2 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state2) + val des2 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser2.obj) + + assertEquals(state2.a, des2.obj.a) + assertEquals(state2.b, des2.obj.b) + assertEquals(state2.c, des2.obj.c) + } + + @Test + fun baseClassInheritedButNotOverridenBounded() { + val factory1 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + + open class Bound(val a: Int) + + open class BaseState(open val a: T1) + class DState(a: T1) : BaseState(a) + + val state = DState(Bound(100)) + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state) + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.a.a, des1.obj.a.a) + } + + @Test + fun nestedMultiGenericsAtBottomWithBound() { + open class BaseState(val a : T1, val b: T2) + class DState(a: T1, b: T2) : BaseState(a, b) + class EState(a: T1, b: T2, val c: Long) : BaseState(a, b) + + data class LTransactionState, out T4: BaseState> (val data: T3, val context: T4) + data class StateWrapper, out T4: BaseState>(val state: LTransactionState) + + val factory1 = testDefaultFactoryNoEvolution() + + val state = LTransactionState(DState(1020304, "Hello"), EState(5060708, "thins", 100L)) + val stateAndString = StateWrapper(state) + + val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString) + + //val factory2 = testDefaultFactoryNoEvolution() + val factory2 = testDefaultFactory() + val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj) + + assertEquals(state.data.a, des1.obj.state.data.a) + assertEquals(state.context.a, des1.obj.state.context.a) + } + }