CORDA-943 - Fix trader demo

This is a multi issue problem
1. Fingerprinting of generics treats <T> and <?> differently, forcing
the evolver to be used when not needed

2. However, the evolver is required sometimes as generics are not
guaranteed to fingerprinting bi-directionally (thanks to type erasure of
deeply nested generic types). However, with serialization now writing
properties in a specific order, we need to ensure they're read back in
that order before applying them to an evolved constructor so as to
not corrupt the object reference cache

core/src/main/kotlin/net/corda/core/contracts/ContractState.kt

@@ -1,7 +1,6 @@
 package net.corda.core.contracts
 
 import net.corda.core.identity.AbstractParty
-import net.corda.core.identity.Party
 import net.corda.core.serialization.CordaSerializable
 
 // DOCSTART 1

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt

@@ -18,7 +18,7 @@ class AMQPPrimitiveSerializer(clazz: Class<*>) : AMQPSerializer<Any> {
     override fun writeClassInfo(output: SerializationOutput) {
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         if (obj is ByteArray) {
             data.putObject(Binary(obj))
         } else {

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt

@@ -30,7 +30,7 @@ interface AMQPSerializer<out T> {
     /**
      * Write the given object, with declared type, to the output.
      */
-    fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput)
+    fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int = 0)
 
     /**
      * Read the given object from the input. The envelope is provided in case the schema is required.

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt

@@ -45,12 +45,12 @@ open class ArraySerializer(override val type: Type, factory: SerializerFactory)
         }
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         // Write described
         data.withDescribed(typeNotation.descriptor) {
             withList {
                 for (entry in obj as Array<*>) {
-                    output.writeObjectOrNull(entry, this, elementType)
+                    output.writeObjectOrNull(entry, this, elementType, offset)
                 }
             }
         }
@@ -109,15 +109,15 @@ abstract class PrimArraySerializer(type: Type, factory: SerializerFactory) : Arr
 
 class PrimIntArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(IntArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }
 
 class PrimCharArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(CharArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 
     override fun <T> List<T>.toArrayOfType(type: Type): Any {
@@ -132,35 +132,35 @@ class PrimCharArraySerializer(factory: SerializerFactory) :
 
 class PrimBooleanArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(BooleanArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }
 
 class PrimDoubleArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(DoubleArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }
 
 class PrimFloatArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(FloatArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }
 
 class PrimShortArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(ShortArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }
 
 class PrimLongArraySerializer(factory: SerializerFactory) :
         PrimArraySerializer(LongArray::class.java, factory) {
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
-        localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
+        localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, offset+4) } }
     }
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt

@@ -66,12 +66,12 @@ class CollectionSerializer(val declaredType: ParameterizedType, factory: Seriali
         }
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) {
         // Write described
         data.withDescribed(typeNotation.descriptor) {
             withList {
                 for (entry in obj as Collection<*>) {
-                    output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0])
+                    output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0], offset)
                 }
             }
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt

@@ -64,7 +64,7 @@ class CorDappCustomSerializer(
 
     override fun writeClassInfo(output: SerializationOutput) {}
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         val proxy = uncheckedCast<SerializationCustomSerializer<*, *>,
                 SerializationCustomSerializer<Any?, Any?>>(serializer).toProxy(obj)
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt

@@ -40,7 +40,7 @@ abstract class CustomSerializer<T : Any> : AMQPSerializer<T>, SerializerFor {
      */
     override val revealSubclassesInSchema: Boolean get() = false
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         data.withDescribed(descriptor) {
             writeDescribedObject(uncheckedCast(obj), data, type, output)
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt

@@ -1,5 +1,6 @@
 package net.corda.nodeapi.internal.serialization.amqp
 
+import com.google.common.primitives.Primitives
 import net.corda.core.internal.getStackTraceAsString
 import net.corda.core.serialization.SerializedBytes
 import net.corda.core.utilities.ByteSequence
@@ -51,12 +52,11 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
     }
 
     @Throws(NotSerializableException::class)
-    inline fun <reified T : Any> deserialize(bytes: SerializedBytes<T>): T =
+    inline fun <reified T : Any> deserialize(bytes: SerializedBytes<T>): T = deserialize(bytes, T::class.java)
-            deserialize(bytes, T::class.java)
 
     @Throws(NotSerializableException::class)
     inline internal fun <reified T : Any> deserializeAndReturnEnvelope(bytes: SerializedBytes<T>): ObjectAndEnvelope<T> =
-            deserializeAndReturnEnvelope(bytes, T::class.java)
+        deserializeAndReturnEnvelope(bytes, T::class.java)
 
     @Throws(NotSerializableException::class)
     internal fun getEnvelope(bytes: ByteSequence): Envelope {
@@ -106,41 +106,47 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
         ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)), envelope)
     }
 
-    internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type): Any? {
+    internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type, offset: Int = 0): Any? {
-        return if (obj == null) null else readObject(obj, schema, type)
+        return if (obj == null) null else readObject(obj, schema, type, offset)
     }
 
-    internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type): Any =
+    internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, offset: Int = 0): Any {
-            if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) {
+        return if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) {
-                // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference.
+            // It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference.
-                val objectIndex = (obj.described as UnsignedInteger).toInt()
+            val objectIndex = (obj.described as UnsignedInteger).toInt()
-                if (objectIndex !in 0..objectHistory.size)
+            if (objectIndex !in 0..objectHistory.size)
-                    throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " +
+                throw NotSerializableException("Retrieval of existing reference failed. Requested index $objectIndex " +
-                            "is outside of the bounds for the list of size: ${objectHistory.size}")
+                        "is outside of the bounds for the list of size: ${objectHistory.size}")
 
-                val objectRetrieved = objectHistory[objectIndex]
+            val objectRetrieved = objectHistory[objectIndex]
-                if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!))
+            if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) {
-                    throw NotSerializableException("Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}'")
+                throw NotSerializableException(
-                objectRetrieved
+                        "Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}' " +
-            } else {
+                                "@ ${objectIndex}")
-                val objectRead = when (obj) {
-                    is DescribedType -> {
-                        // Look up serializer in factory by descriptor
-                        val serializer = serializerFactory.get(obj.descriptor, schemas)
-                        if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) })
-                            throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " +
-                                    "expected to be of type $type but was ${serializer.type}")
-                        serializer.readObject(obj.described, schemas, this)
-                    }
-                    is Binary -> obj.array
-                    else -> obj // this will be the case for primitive types like [boolean] et al.
-                }
-
-                // Store the reference in case we need it later on.
-                // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content
-                if (suitableForObjectReference(objectRead.javaClass)) objectHistory.add(objectRead)
-                objectRead
             }
+            objectRetrieved
+        } else {
+            val objectRead = when (obj) {
+                is DescribedType -> {
+                    // Look up serializer in factory by descriptor
+                    val serializer = serializerFactory.get(obj.descriptor, schemas)
+                    if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) })
+                        throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " +
+                                "expected to be of type $type but was ${serializer.type}")
+                    serializer.readObject(obj.described, schemas, this)
+                }
+                is Binary -> obj.array
+                else -> obj // this will be the case for primitive types like [boolean] et al.
+            }
+
+            // Store the reference in case we need it later on.
+            // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content
+            if (suitableForObjectReference(objectRead.javaClass)) {
+                objectHistory.add(objectRead)
+            }
+            objectRead
+        }
+    }
 
     /**
      * Currently performs checks aimed at:

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt

@@ -130,7 +130,7 @@ class EnumEvolutionSerializer(
         throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
     }
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt

@@ -39,7 +39,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Seria
         return fromOrd
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         if (obj !is Enum<*>) throw NotSerializableException("Serializing $obj as enum when it isn't")
 
         data.withDescribed(typeNotation.descriptor) {

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt

@@ -9,15 +9,24 @@ import kotlin.reflect.KFunction
 import kotlin.reflect.full.findAnnotation
 import kotlin.reflect.jvm.javaType
 
+
 /**
  * Serializer for deserializing objects whose definition has changed since they
  * were serialised.
+ *
+ * @property oldReaders A linked map representing the properties of the object as they were serialized. Note
+ * this may contain properties that are no longer needed by the class. These *must* be read however to ensure
+ * any refferenced objects in the object stream are captured properly
+ * @property kotlinConstructor
+ * @property constructorArgs used to hold the properties as sent to the object's constructor. Passed in as a
+ * pre populated array as properties not present on the old constructor must be initialised in the factory
  */
 class EvolutionSerializer(
         clazz: Type,
         factory: SerializerFactory,
-        val readers: List<OldParam?>,
+        private val oldReaders: Map<String, OldParam>,
-        override val kotlinConstructor: KFunction<Any>?) : ObjectSerializer(clazz, factory) {
+        override val kotlinConstructor: KFunction<Any>?,
+        private val constructorArgs: Array<Any?>) : ObjectSerializer(clazz, factory) {
 
     // explicitly set as empty to indicate it's unused by this type of serializer
     override val propertySerializers = PropertySerializersEvolution()
@@ -27,13 +36,17 @@ class EvolutionSerializer(
      * when it was serialised and NOT how that class appears now
      *
      * @param type The jvm type of the parameter
-     * @param idx where in the parameter list this parameter falls. Required as the parameter
+     * @param resultsIndex index into the constructor argument list where the read property
-     * order may have been changed and we need to know where into the list to look
+     * should be placed
      * @param property object to read the actual property value
      */
-    data class OldParam(val type: Type, val idx: Int, val property: PropertySerializer) {
+    data class OldParam(val type: Type, var resultsIndex: Int, val property: PropertySerializer) {
-        fun readProperty(paramValues: List<*>, schemas: SerializationSchemas, input: DeserializationInput) =
+        fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput, new: Array<Any?>) =
-                property.readProperty(paramValues[idx], schemas, input)
+                property.readProperty(obj, schemas, input).apply {
+                    if(resultsIndex >= 0) {
+                        new[resultsIndex] = this
+                    }
+                }
     }
 
     companion object {
@@ -47,11 +60,11 @@ class EvolutionSerializer(
          * TODO: Type evolution
          * TODO: rename annotation
          */
-        private fun getEvolverConstructor(type: Type, oldArgs: Map<String?, Type>): KFunction<Any>? {
+        private fun getEvolverConstructor(type: Type, oldArgs: Map<String, OldParam>): KFunction<Any>? {
             val clazz: Class<*> = type.asClass()!!
             if (!isConcrete(clazz)) return null
 
-            val oldArgumentSet = oldArgs.map { Pair(it.key, it.value) }
+            val oldArgumentSet = oldArgs.map { Pair(it.key as String?, it.value.type) }
 
             var maxConstructorVersion = Integer.MIN_VALUE
             var constructor: KFunction<Any>? = null
@@ -83,34 +96,42 @@ class EvolutionSerializer(
         fun make(old: CompositeType, new: ObjectSerializer,
                  factory: SerializerFactory): AMQPSerializer<Any> {
 
-            val oldFieldToType = old.fields.map {
+            val readersAsSerialized = linkedMapOf<String, OldParam>(
-                it.name as String? to it.getTypeAsClass(factory.classloader) as Type
+                    *(old.fields.map {
-            }.toMap()
+                        val returnType =  try {
+                            it.getTypeAsClass(factory.classloader)
+                        } catch (e: ClassNotFoundException) {
+                            throw NotSerializableException(e.message)
+                        }
 
-            val constructor = getEvolverConstructor(new.type, oldFieldToType) ?:
+                        it.name to OldParam(
+                                returnType,
+                                -1,
+                                PropertySerializer.make(
+                                        it.name, PublicPropertyReader(null), returnType, factory))
+                    }.toTypedArray())
+            )
+
+            val constructor = getEvolverConstructor(new.type, readersAsSerialized) ?:
                     throw NotSerializableException(
                             "Attempt to deserialize an interface: ${new.type}. Serialized form is invalid.")
 
-            val oldArgs = mutableMapOf<String, OldParam>()
+            val constructorArgs = arrayOfNulls<Any?>(constructor.parameters.size)
-            var idx = 0
-            old.fields.forEach {
-                val returnType = it.getTypeAsClass(factory.classloader)
-                oldArgs[it.name] = OldParam(
-                        returnType, idx++, PropertySerializer.make(it.name, PublicPropertyReader(null), returnType, factory))
-            }
 
-            val readers = constructor.parameters.map {
+            constructor.parameters.withIndex().forEach {
-                oldArgs[it.name!!] ?: if (!it.type.isMarkedNullable) {
+                readersAsSerialized.get(it.value.name!!)?.apply {
+                    this.resultsIndex = it.index
+                } ?: if (!it.value.type.isMarkedNullable) {
                     throw NotSerializableException(
-                            "New parameter ${it.name} is mandatory, should be nullable for evolution to worK")
+                            "New parameter ${it.value.name} is mandatory, should be nullable for evolution to worK")
-                } else null
+                }
             }
 
-            return EvolutionSerializer(new.type, factory, readers, constructor)
+            return EvolutionSerializer(new.type, factory, readersAsSerialized, constructor, constructorArgs)
         }
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
     }
 
@@ -126,7 +147,11 @@ class EvolutionSerializer(
     override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
         if (obj !is List<*>) throw NotSerializableException("Body of described type is unexpected $obj")
 
-        return construct(readers.map { it?.readProperty(obj, schemas, input) })
+        // *must* read all the parameters in the order they were serialized
+        oldReaders.values.zip(obj).map { it.first.readProperty(it.second, schemas, input, constructorArgs) }
+
+        return javaConstructor?.newInstance(*(constructorArgs)) ?:
+                throw NotSerializableException("Attempt to deserialize an interface: $clazz. Serialized form is invalid.")
     }
 }
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt

@@ -73,7 +73,7 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial
         }
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ declaredType.typeName }) {
         obj.javaClass.checkSupportedMapType()
         // Write described
         data.withDescribed(typeNotation.descriptor) {
@@ -81,8 +81,8 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial
             data.putMap()
             data.enter()
             for ((key, value) in obj as Map<*, *>) {
-                output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0])
+                output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0], offset)
-                output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1])
+                output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1], offset)
             }
             data.exit() // exit map
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt

@@ -52,13 +52,13 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS
         }
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ clazz.typeName }) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) = ifThrowsAppend({ clazz.typeName }) {
         // Write described
         data.withDescribed(typeNotation.descriptor) {
             // Write list
             withList {
                 propertySerializers.serializationOrder.forEach { property ->
-                    property.getter.writeProperty(obj, this, output)
+                    property.getter.writeProperty(obj, this, output, offset+4)
                 }
             }
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt

@@ -9,7 +9,7 @@ import java.lang.reflect.Type
  */
 sealed class PropertySerializer(val name: String, val propertyReader: PropertyReader, val resolvedType: Type) {
     abstract fun writeClassInfo(output: SerializationOutput)
-    abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput)
+    abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int = 0)
     abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any?
 
     val type: String = generateType()
@@ -80,8 +80,8 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
             input.readObjectOrNull(obj, schemas, resolvedType)
         }
 
-        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) = ifThrowsAppend({ nameForDebug }) {
+        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) = ifThrowsAppend({ nameForDebug }) {
-            output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType)
+            output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType, offset)
         }
 
         private val nameForDebug = "$name(${resolvedType.typeName})"
@@ -100,7 +100,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
             return if (obj is Binary) obj.array else obj
         }
 
-        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
+        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) {
             val value = propertyReader.read(obj)
             if (value is ByteArray) {
                 data.putObject(Binary(value))
@@ -123,7 +123,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
             return if (obj == null) null else (obj as Short).toChar()
         }
 
-        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
+        override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, offset: Int) {
             val input = propertyReader.read(obj)
             if (input != null) data.putShort((input as Char).toShort()) else data.putNull()
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt

@@ -350,13 +350,15 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact
 // creating a unique string for a type which we then hash in the calling function above.
 private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet<Type>,
                                hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher {
-
     // We don't include Example<?> and Example<T> where type is ? or T in this otherwise we
     // generate different fingerprints for class Outer<T>(val a: Inner<T>) when serialising
     // and deserializing (assuming deserialization is occurring in a factory that didn't
     // serialise the object in the  first place (and thus the cache lookup fails). This is also
     // true of Any, where we need  Example<A, B> and Example<?, ?> to have the same fingerprint
-    return if (type in alreadySeen && (type !is SerializerFactory.AnyType) && (type !is TypeVariable<*>)) {
+    return if ((type in alreadySeen)
+            && (type !is SerializerFactory.AnyType)
+            && (type !is TypeVariable<*>)
+            && (type !is WildcardType)) {
         hasher.putUnencodedChars(ALREADY_SEEN_HASH)
     } else {
         alreadySeen += type
@@ -379,14 +381,20 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
                         fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4)
                     }
                 }
-            // Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when
+            // Previously, we drew a distinction between TypeVariable, Wildcard, and AnyType, changing
-            // looking at A and B from Example<A, B> (remember we call this function recursively). When
+            // the signature of the fingerprinted object. This, however, doesn't work as it breaks bi-
-            // serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but
+            // directional fingerprints. That is, fingerprinting a concrete instance of a generic
-            // when deserializing we only have the wildcard placeholder ?, or AnyType
+            // type (Example<Int>), creates a different fingerprint from the generic type itself (Example<T>)
             //
-            // Note, TypeVariable<*> used to be encoded as TYPE_VARIABLE_HASH but that again produces a
+            // On serialization Example<Int> is treated as Example<T>, a TypeVariable
-            // differing fingerprint on serialisation and deserialization
+            // On deserialisation it is seen as Example<?>, A wildcard *and* a TypeVariable
+            //      Note: AnyType is a special case of WildcarcType used in other parts of the
+            //            serializer so both cases need to be dealt with here
+            //
+            // If we treat these types as fundamentally different and alter the fingerprint we will
+            // end up breaking into the evolver when we shouldn't or, worse, evoking the carpenter.
                 is SerializerFactory.AnyType,
+                is WildcardType,
                 is TypeVariable<*> -> {
                     hasher.putUnencodedChars("?").putUnencodedChars(ANY_TYPE_HASH)
                 }
@@ -418,9 +426,10 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
                     }
                 }
             // Hash the element type + some array hash
-                is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen,
+                is GenericArrayType -> {
-                        hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH)
+                    fingerprintForType(type.genericComponentType, contextType, alreadySeen,
-            // TODO: include bounds
+                            hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH)
+                }
                 is WildcardType -> {
                     hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH)
                 }
@@ -446,6 +455,7 @@ private fun fingerprintForObject(
         offset: Int = 0): Hasher {
     // Hash the class + properties + interfaces
     val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type")
+
     propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory)
             .serializationOrder
             .fold(hasher.putUnencodedChars(name)) { orig, prop ->

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt

@@ -62,11 +62,12 @@ internal fun constructorForDeserialization(type: Type): KFunction<Any>? {
 }
 
 /**
- * Identifies the properties to be used during serialization by attempting to find those that match the parameters to the
+ * Identifies the properties to be used during serialization by attempting to find those that match the parameters
- * deserialization constructor, if the class is concrete.  If it is abstract, or an interface, then use all the properties.
+ * to the deserialization constructor, if the class is concrete.  If it is abstract, or an interface, then use all
+ * the properties.
  *
- * Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters have
+ * Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters
- * names accessible via reflection.
+ * have names accessible via reflection.
  */
 internal fun <T : Any> propertiesForSerialization(
         kotlinConstructor: KFunction<T>?,

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt

@@ -86,15 +86,15 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory
         data.putObject(transformsSchema)
     }
 
-    internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type) {
+    internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type, offset: Int) {
         if (obj == null) {
             data.putNull()
         } else {
-            writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type)
+            writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type, offset)
         }
     }
 
-    internal fun writeObject(obj: Any, data: Data, type: Type) {
+    internal fun writeObject(obj: Any, data: Data, type: Type, offset: Int = 0) {
         val serializer = serializerFactory.get(obj.javaClass, type)
         if (serializer !in serializerHistory) {
             serializerHistory.add(serializer)
@@ -103,11 +103,13 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory
 
         val retrievedRefCount = objectHistory[obj]
         if (retrievedRefCount == null) {
-            serializer.writeObject(obj, data, type, this)
+            serializer.writeObject(obj, data, type, this, offset)
             // Important to do it after serialization such that dependent object will have preceding reference numbers
             // assigned to them first as they will be first read from the stream on receiving end.
             // Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content
-            if (suitableForObjectReference(obj.javaClass)) objectHistory.put(obj, objectHistory.size)
+            if (suitableForObjectReference(obj.javaClass)) {
+                objectHistory.put(obj, objectHistory.size)
+            }
         } else {
             data.writeReferencedObject(ReferencedObject(retrievedRefCount))
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt

@@ -112,7 +112,7 @@ open class SerializerFactory(
      */
     // TODO: test GenericArrayType
     private fun inferTypeVariables(actualClass: Class<*>?, declaredClass: Class<*>,
-                                   declaredType: Type) : Type? = when (declaredType) {
+                                   declaredType: Type): Type? = when (declaredType) {
         is ParameterizedType -> inferTypeVariables(actualClass, declaredClass, declaredType)
     // Nothing to infer, otherwise we'd have ParameterizedType
         is Class<*> -> actualClass
@@ -218,7 +218,6 @@ open class SerializerFactory(
         for (typeNotation in schemaAndDescriptor.schemas.schema.types) {
             try {
                 val serialiser = processSchemaEntry(typeNotation)
-
                 // if we just successfully built a serializer for the type but the type fingerprint
                 // doesn't match that of the serialised object then we are dealing with  different
                 // instance of the class, as such we need to build an EvolutionSerializer

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt

@@ -22,7 +22,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto
         output.writeTypeNotations(typeNotation)
     }
 
-    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
+    override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
         data.withDescribed(typeNotation.descriptor) {
             data.putBoolean(false)
         }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt

@@ -124,7 +124,12 @@ fun AMQPField.getTypeAsClass(classloader: ClassLoader) = typeStrToType[Pair(type
     "string" -> String::class.java
     "binary" -> ByteArray::class.java
     "*" -> if (requires.isEmpty()) Any::class.java else classloader.loadClass(requires[0])
-    else -> classloader.loadClass(type)
+    else -> {
+        classloader.loadClass(
+                if (type.endsWith("<?>")) {
+                    type.substring(0, type.length-3)
+                } else type)
+    }
 }
 
 fun AMQPField.validateType(classloader: ClassLoader) = when (type) {

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt

@@ -6,6 +6,7 @@ import net.corda.testing.common.internal.ProjectStructure.projectRootDir
 import org.junit.Test
 import java.io.File
 import java.io.NotSerializableException
+import java.net.URI
 import kotlin.test.assertEquals
 
 // To regenerate any of the binary test files do the following

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt

@@ -1,14 +1,30 @@
 package net.corda.nodeapi.internal.serialization.amqp
 
+import net.corda.core.contracts.*
 import net.corda.core.serialization.SerializedBytes
 import net.corda.nodeapi.internal.serialization.AllWhitelist
 import net.corda.testing.common.internal.ProjectStructure.projectRootDir
 import org.junit.Test
+import net.corda.core.identity.AbstractParty
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.identity.Party
+import net.corda.core.transactions.WireTransaction
+import net.corda.testing.core.TestIdentity
+import org.hibernate.Transaction
 import java.io.File
 import java.net.URI
+import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 import kotlin.test.assertEquals
 
+data class TestContractState(
+        override val participants: List<AbstractParty>
+) : ContractState
+
+class TestAttachmentConstraint : AttachmentConstraint {
+    override fun isSatisfiedBy(attachment: Attachment) = true
+}
+
 class GenericsTests {
     companion object {
         val VERBOSE = false
@@ -16,6 +32,8 @@ class GenericsTests {
         @Suppress("UNUSED")
         var localPath = projectRootDir.toUri().resolve(
                 "node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp")
+
+        val miniCorp = TestIdentity(CordaX500Name("MiniCorp", "London", "GB"))
     }
 
     private fun printSeparator() = if (VERBOSE) println("\n\n-------------------------------------------\n\n") else Unit
@@ -252,29 +270,119 @@ class GenericsTests {
                         File(GenericsTests::class.java.getResource(resource).toURI()).readBytes())).t)
     }
 
-    interface DifferentBounds {
+    data class StateAndString(val state: TransactionState<*>, val ref: String)
-        fun go()
+    data class GenericStateAndString<out T: ContractState>(val state: TransactionState<T>, val ref: String)
+
+    //
+    // If this doesn't blow up all is fine
+    private fun fingerprintingDiffersStrip(state: Any) {
+        class cl : ClassLoader()
+
+        val m = ClassLoader::class.java.getDeclaredMethod("findLoadedClass", *arrayOf<Class<*>>(String::class.java))
+        m.isAccessible = true
+
+        val factory1 = testDefaultFactory()
+        factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+        val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state)
+
+        // attempt at having a class loader without some of the derived non core types loaded and thus
+        // possibly altering how we serialise things
+        val altClassLoader = cl()
+
+        val factory2 = SerializerFactory(AllWhitelist, altClassLoader)
+        factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+        val ser2 = TestSerializationOutput(VERBOSE, factory2).serializeAndReturnSchema(state)
+
+        //  now deserialise those objects
+        val factory3 = testDefaultFactory()
+        factory3.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+        val des1 = DeserializationInput(factory3).deserializeAndReturnEnvelope(ser1.obj)
+
+        val factory4 = SerializerFactory(AllWhitelist, cl())
+        factory4.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+        val des2 = DeserializationInput(factory4).deserializeAndReturnEnvelope(ser2.obj)
+
     }
 
     @Test
-    fun differentBounds() {
+    fun fingerprintingDiffers() {
-        data class A (val a: Int): DifferentBounds {
+        val state = TransactionState<TestContractState> (
-           override fun go() {
+                TestContractState(listOf(miniCorp.party)),
-               println(a)
+                "wibble", miniCorp.party,
-           }
+                encumbrance = null,
-        }
+                constraint = TestAttachmentConstraint())
 
-        data class G<out T : DifferentBounds>(val b: T)
+        val sas = StateAndString(state, "wibble")
 
-        val factorys = listOf(
+        fingerprintingDiffersStrip(sas)
-                SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()),
-                SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()))
-
-        val ser = SerializationOutput(factorys[0])
-
-        ser.serialize(G(A(10))).apply {
-            factorys.forEach {
-            }
-        }
     }
+
+    @Test
+    fun fingerprintingDiffersList() {
+        val state = TransactionState<TestContractState> (
+                TestContractState(listOf(miniCorp.party)),
+                "wibble", miniCorp.party,
+                encumbrance = null,
+                constraint = TestAttachmentConstraint())
+
+        val sas = StateAndString(state, "wibble")
+
+        fingerprintingDiffersStrip(Collections.singletonList(sas))
+    }
+
+
+    //
+    // Force object to be serialised as Example<T> and deserialized as Example<?>
+    //
+    @Test
+    fun fingerprintingDiffersListLoaded() {
+        //
+        // using this wrapper class we force the object to be serialised as
+        //      net.corda.core.contracts.TransactionState<T>
+        //
+        data class TransactionStateWrapper<out T : ContractState> (val o: List<GenericStateAndString<T>>)
+
+        val state = TransactionState<TestContractState> (
+                TestContractState(listOf(miniCorp.party)),
+                "wibble", miniCorp.party,
+                encumbrance = null,
+                constraint = TestAttachmentConstraint())
+
+        val sas = GenericStateAndString(state, "wibble")
+
+        val factory1 = testDefaultFactoryNoEvolution()
+        val factory2 = testDefaultFactory()
+
+        factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+        factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
+
+        val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(
+                TransactionStateWrapper(Collections.singletonList(sas)))
+
+        val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
+
+        assertEquals(sas.ref, des1.obj.o.firstOrNull()?.ref ?: "WILL NOT MATCH")
+    }
+
+    @Test
+    fun anotherTry() {
+        open class BaseState(val a : Int)
+        class DState(a: Int) : BaseState(a)
+        data class LTransactionState<out T : BaseState> constructor(val data: T)
+        data class StateWrapper<out T : BaseState>(val state: LTransactionState<T>)
+
+        val factory1 = testDefaultFactoryNoEvolution()
+
+        val state = LTransactionState(DState(1020304))
+        val stateAndString = StateWrapper(state)
+
+        val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString)
+
+        //val factory2 = testDefaultFactoryNoEvolution()
+        val factory2 = testDefaultFactory()
+        val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
+
+        assertEquals(state.data.a, des1.obj.state.data.a)
+    }
+
 }