ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2024-12-26 16:11:12 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

CORDA-943 - Deterministic serialization order breaks object ref cache

Browse Source
This commit is contained in:
Katelyn Baker 2018-02-02 10:40:14 +00:00
parent cf37895327
commit e5627622db
22 changed files with 440 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/src/main/kotlin/net/corda/core/contracts/ContractState.kt
View File

@ -1,7 +1,6 @@
package net.corda.core.contracts
import net.corda.core.identity.AbstractParty
import net.corda.core.identity.Party
import net.corda.core.serialization.CordaSerializable
// DOCSTART 1

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPPrimitiveSerializer.kt
View File

@ -18,7 +18,7 @@ class AMQPPrimitiveSerializer(clazz: Class<*>) : AMQPSerializer<Any> {
override fun writeClassInfo(output: SerializationOutput) {
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
if (obj is ByteArray) {
data.putObject(Binary(obj))
} else {

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializer.kt
View File

@ -30,7 +30,7 @@ interface AMQPSerializer<out T> {
/**
* Write the given object, with declared type, to the output.
*/
fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput)
fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int = 0)
/**
* Read the given object from the input. The envelope is provided in case the schema is required.

46
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ArraySerializer.kt
View File

@ -45,12 +45,12 @@ open class ArraySerializer(override val type: Type, factory: SerializerFactory)
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
// Write described
data.withDescribed(typeNotation.descriptor) {
withList {
for (entry in obj as Array<*>) {
output.writeObjectOrNull(entry, this, elementType)
output.writeObjectOrNull(entry, this, elementType, debugIndent)
}
}
}
@ -109,15 +109,19 @@ abstract class PrimArraySerializer(type: Type, factory: SerializerFactory) : Arr
class PrimIntArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(IntArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}
class PrimCharArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(CharArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) { (obj as CharArray).forEach {
output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
override fun <T> List<T>.toArrayOfType(type: Type): Any {
@ -132,35 +136,45 @@ class PrimCharArraySerializer(factory: SerializerFactory) :
class PrimBooleanArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(BooleanArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}
class PrimDoubleArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(DoubleArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}
class PrimFloatArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(FloatArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}
class PrimShortArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(ShortArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}
class PrimLongArraySerializer(factory: SerializerFactory) :
PrimArraySerializer(LongArray::class.java, factory) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
localWriteObject(data) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType) } }
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
localWriteObject(data) {
(obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, debugIndent+1) }
}
}
}

14
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CollectionSerializer.kt
View File

@ -66,18 +66,26 @@ class CollectionSerializer(val declaredType: ParameterizedType, factory: Seriali
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) {
override fun writeObject(
obj: Any,
data: Data,
type: Type,
output: SerializationOutput,
debugIndent: Int) = ifThrowsAppend({ declaredType.typeName }) {
// Write described
data.withDescribed(typeNotation.descriptor) {
withList {
for (entry in obj as Collection<*>) {
output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0])
output.writeObjectOrNull(entry, this, declaredType.actualTypeArguments[0], debugIndent)
}
}
}
}
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
override fun readObject(
obj: Any,
schemas: SerializationSchemas,
input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
// TODO: Can we verify the entries in the list?
concreteBuilder((obj as List<*>).map { input.readObjectOrNull(it, schemas, declaredType.actualTypeArguments[0]) })
}

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CorDappCustomSerializer.kt
View File

@ -64,7 +64,7 @@ class CorDappCustomSerializer(
override fun writeClassInfo(output: SerializationOutput) {}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
val proxy = uncheckedCast<SerializationCustomSerializer<*, *>,
SerializationCustomSerializer<Any?, Any?>>(serializer).toProxy(obj)

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/CustomSerializer.kt
View File

@ -40,7 +40,7 @@ abstract class CustomSerializer<T : Any> : AMQPSerializer<T>, SerializerFor {
*/
override val revealSubclassesInSchema: Boolean get() = false
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
data.withDescribed(descriptor) {
writeDescribedObject(uncheckedCast(obj), data, type, output)
}

20
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/DeserializationInput.kt
View File

@ -51,8 +51,7 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
}
@Throws(NotSerializableException::class)
inline fun <reified T : Any> deserialize(bytes: SerializedBytes<T>): T =
deserialize(bytes, T::class.java)
inline fun <reified T : Any> deserialize(bytes: SerializedBytes<T>): T = deserialize(bytes, T::class.java)
@Throws(NotSerializableException::class)
inline internal fun <reified T : Any> deserializeAndReturnEnvelope(bytes: SerializedBytes<T>): ObjectAndEnvelope<T> =
@ -106,11 +105,11 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)), envelope)
}
internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type): Any? {
return if (obj == null) null else readObject(obj, schema, type)
internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type, offset: Int = 0): Any? {
return if (obj == null) null else readObject(obj, schema, type, offset)
}
internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type): Any =
internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type, debugIndent: Int = 0): Any =
if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) {
// It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference.
val objectIndex = (obj.described as UnsignedInteger).toInt()
@ -119,8 +118,11 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
"is outside of the bounds for the list of size: ${objectHistory.size}")
val objectRetrieved = objectHistory[objectIndex]
if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!))
throw NotSerializableException("Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}'")
if (!objectRetrieved::class.java.isSubClassOf(type.asClass()!!)) {
throw NotSerializableException(
"Existing reference type mismatch. Expected: '$type', found: '${objectRetrieved::class.java}' " +
"@ ${objectIndex}")
}
objectRetrieved
} else {
val objectRead = when (obj) {
@ -138,7 +140,9 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
// Store the reference in case we need it later on.
// Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content
if (suitableForObjectReference(objectRead.javaClass)) objectHistory.add(objectRead)
if (suitableForObjectReference(objectRead.javaClass)) {
objectHistory.add(objectRead)
}
objectRead
}

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumEvolutionSerializer.kt
View File

@ -130,7 +130,7 @@ class EnumEvolutionSerializer(
throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
}
}

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EnumSerializer.kt
View File

@ -39,7 +39,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Seria
return fromOrd
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
if (obj !is Enum<*>) throw NotSerializableException("Serializing $obj as enum when it isn't")
data.withDescribed(typeNotation.descriptor) {

83
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolutionSerializer.kt
View File

@ -9,15 +9,24 @@ import kotlin.reflect.KFunction
import kotlin.reflect.full.findAnnotation
import kotlin.reflect.jvm.javaType
/**
* Serializer for deserializing objects whose definition has changed since they
* were serialised.
*
* @property oldReaders A linked map representing the properties of the object as they were serialized. Note
* this may contain properties that are no longer needed by the class. These *must* be read however to ensure
* any refferenced objects in the object stream are captured properly
* @property kotlinConstructor
* @property constructorArgs used to hold the properties as sent to the object's constructor. Passed in as a
* pre populated array as properties not present on the old constructor must be initialised in the factory
*/
class EvolutionSerializer(
clazz: Type,
factory: SerializerFactory,
val readers: List<OldParam?>,
override val kotlinConstructor: KFunction<Any>?) : ObjectSerializer(clazz, factory) {
private val oldReaders: Map<String, OldParam>,
override val kotlinConstructor: KFunction<Any>?,
private val constructorArgs: Array<Any?>) : ObjectSerializer(clazz, factory) {
// explicitly set as empty to indicate it's unused by this type of serializer
override val propertySerializers = PropertySerializersEvolution()
@ -27,13 +36,17 @@ class EvolutionSerializer(
* when it was serialised and NOT how that class appears now
*
* @param type The jvm type of the parameter
* @param idx where in the parameter list this parameter falls. Required as the parameter
* order may have been changed and we need to know where into the list to look
* @param resultsIndex index into the constructor argument list where the read property
* should be placed
* @param property object to read the actual property value
*/
data class OldParam(val type: Type, val idx: Int, val property: PropertySerializer) {
fun readProperty(paramValues: List<*>, schemas: SerializationSchemas, input: DeserializationInput) =
property.readProperty(paramValues[idx], schemas, input)
data class OldParam(val type: Type, var resultsIndex: Int, val property: PropertySerializer) {
fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput, new: Array<Any?>) =
property.readProperty(obj, schemas, input).apply {
if(resultsIndex >= 0) {
new[resultsIndex] = this
}
}
}
companion object {
@ -47,11 +60,11 @@ class EvolutionSerializer(
* TODO: Type evolution
* TODO: rename annotation
*/
private fun getEvolverConstructor(type: Type, oldArgs: Map<String?, Type>): KFunction<Any>? {
private fun getEvolverConstructor(type: Type, oldArgs: Map<String, OldParam>): KFunction<Any>? {
val clazz: Class<*> = type.asClass()!!
if (!isConcrete(clazz)) return null
val oldArgumentSet = oldArgs.map { Pair(it.key, it.value) }
val oldArgumentSet = oldArgs.map { Pair(it.key as String?, it.value.type) }
var maxConstructorVersion = Integer.MIN_VALUE
var constructor: KFunction<Any>? = null
@ -83,34 +96,42 @@ class EvolutionSerializer(
fun make(old: CompositeType, new: ObjectSerializer,
factory: SerializerFactory): AMQPSerializer<Any> {
val oldFieldToType = old.fields.map {
it.name as String? to it.getTypeAsClass(factory.classloader) as Type
}.toMap()
val readersAsSerialized = linkedMapOf<String, OldParam>(
*(old.fields.map {
val returnType = try {
it.getTypeAsClass(factory.classloader)
} catch (e: ClassNotFoundException) {
throw NotSerializableException(e.message)
}
val constructor = getEvolverConstructor(new.type, oldFieldToType) ?:
it.name to OldParam(
returnType,
-1,
PropertySerializer.make(
it.name, PublicPropertyReader(null), returnType, factory))
}.toTypedArray())
)
val constructor = getEvolverConstructor(new.type, readersAsSerialized) ?:
throw NotSerializableException(
"Attempt to deserialize an interface: ${new.type}. Serialized form is invalid.")
val oldArgs = mutableMapOf<String, OldParam>()
var idx = 0
old.fields.forEach {
val returnType = it.getTypeAsClass(factory.classloader)
oldArgs[it.name] = OldParam(
returnType, idx++, PropertySerializer.make(it.name, PublicPropertyReader(null), returnType, factory))
}
val constructorArgs = arrayOfNulls<Any?>(constructor.parameters.size)
val readers = constructor.parameters.map {
oldArgs[it.name!!] ?: if (!it.type.isMarkedNullable) {
constructor.parameters.withIndex().forEach {
readersAsSerialized.get(it.value.name!!)?.apply {
this.resultsIndex = it.index
} ?: if (!it.value.type.isMarkedNullable) {
throw NotSerializableException(
"New parameter ${it.name} is mandatory, should be nullable for evolution to worK")
} else null
}
return EvolutionSerializer(new.type, factory, readers, constructor)
"New parameter ${it.value.name} is mandatory, should be nullable for evolution to worK")
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
return EvolutionSerializer(new.type, factory, readersAsSerialized, constructor, constructorArgs)
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, offset: Int) {
throw UnsupportedOperationException("It should be impossible to write an evolution serializer")
}
@ -126,7 +147,11 @@ class EvolutionSerializer(
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
if (obj !is List<*>) throw NotSerializableException("Body of described type is unexpected $obj")
return construct(readers.map { it?.readProperty(obj, schemas, input) })
// *must* read all the parameters in the order they were serialized
oldReaders.values.zip(obj).map { it.first.readProperty(it.second, schemas, input, constructorArgs) }
return javaConstructor?.newInstance(*(constructorArgs)) ?:
throw NotSerializableException("Attempt to deserialize an interface: $clazz. Serialized form is invalid.")
}
}

11
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/MapSerializer.kt
View File

@ -73,7 +73,12 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ declaredType.typeName }) {
override fun writeObject(
obj: Any,
data: Data,
type: Type,
output: SerializationOutput,
debugIndent: Int) = ifThrowsAppend({ declaredType.typeName }) {
obj.javaClass.checkSupportedMapType()
// Write described
data.withDescribed(typeNotation.descriptor) {
@ -81,8 +86,8 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial
data.putMap()
data.enter()
for ((key, value) in obj as Map<*, *>) {
output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0])
output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1])
output.writeObjectOrNull(key, data, declaredType.actualTypeArguments[0], debugIndent)
output.writeObjectOrNull(value, data, declaredType.actualTypeArguments[1], debugIndent)
}
data.exit() // exit map
}

9
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/ObjectSerializer.kt
View File

@ -52,13 +52,18 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) = ifThrowsAppend({ clazz.typeName }) {
override fun writeObject(
obj: Any,
data: Data,
type: Type,
output: SerializationOutput,
debugIndent: Int) = ifThrowsAppend({ clazz.typeName }) {
// Write described
data.withDescribed(typeNotation.descriptor) {
// Write list
withList {
propertySerializers.serializationOrder.forEach { property ->
property.getter.writeProperty(obj, this, output)
property.getter.writeProperty(obj, this, output, debugIndent+1)
}
}
}

10
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/PropertySerializer.kt
View File

@ -9,7 +9,7 @@ import java.lang.reflect.Type
*/
sealed class PropertySerializer(val name: String, val propertyReader: PropertyReader, val resolvedType: Type) {
abstract fun writeClassInfo(output: SerializationOutput)
abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput)
abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int = 0)
abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any?
val type: String = generateType()
@ -80,8 +80,8 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
input.readObjectOrNull(obj, schemas, resolvedType)
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) = ifThrowsAppend({ nameForDebug }) {
output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType)
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) = ifThrowsAppend({ nameForDebug }) {
output.writeObjectOrNull(propertyReader.read(obj), data, resolvedType, debugIndent)
}
private val nameForDebug = "$name(${resolvedType.typeName})"
@ -100,7 +100,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
return if (obj is Binary) obj.array else obj
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) {
val value = propertyReader.read(obj)
if (value is ByteArray) {
data.putObject(Binary(value))
@ -123,7 +123,7 @@ sealed class PropertySerializer(val name: String, val propertyReader: PropertyRe
return if (obj == null) null else (obj as Short).toChar()
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput, debugIndent: Int) {
val input = propertyReader.read(obj)
if (input != null) data.putShort((input as Char).toShort()) else data.putNull()
}

49
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/Schema.kt
View File

@ -349,14 +349,16 @@ private fun Hasher.fingerprintWithCustomSerializerOrElse(factory: SerializerFact
// This method concatenates various elements of the types recursively as unencoded strings into the hasher, effectively
// creating a unique string for a type which we then hash in the calling function above.
private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: MutableSet<Type>,
hasher: Hasher, factory: SerializerFactory, offset: Int = 4): Hasher {
hasher: Hasher, factory: SerializerFactory, debugIndent: Int = 1): Hasher {
// We don't include Example<?> and Example<T> where type is ? or T in this otherwise we
// generate different fingerprints for class Outer<T>(val a: Inner<T>) when serialising
// and deserializing (assuming deserialization is occurring in a factory that didn't
// serialise the object in the first place (and thus the cache lookup fails). This is also
// true of Any, where we need Example<A, B> and Example<?, ?> to have the same fingerprint
return if (type in alreadySeen && (type !is SerializerFactory.AnyType) && (type !is TypeVariable<*>)) {
return if ((type in alreadySeen)
&& (type !is SerializerFactory.AnyType)
&& (type !is TypeVariable<*>)
&& (type !is WildcardType)) {
hasher.putUnencodedChars(ALREADY_SEEN_HASH)
} else {
alreadySeen += type
@ -370,29 +372,35 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
hasher.putUnencodedChars(clazz.name)
} else {
hasher.fingerprintWithCustomSerializerOrElse(factory, clazz, type) {
fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4)
fingerprintForObject(type, type, alreadySeen, hasher, factory, debugIndent+1)
}
}
// ... and concatenate the type data for each parameter type.
type.actualTypeArguments.fold(startingHash) { orig, paramType ->
fingerprintForType(paramType, type, alreadySeen, orig, factory, offset+4)
fingerprintForType(paramType, type, alreadySeen, orig, factory, debugIndent+1)
}
}
// Treat generic types as "any type" to prevent fingerprint mismatch. This case we fall into when
// looking at A and B from Example<A, B> (remember we call this function recursively). When
// serialising a concrete example of the type we have A and B which are TypeVariables<*>'s but
// when deserializing we only have the wildcard placeholder ?, or AnyType
// Previously, we drew a distinction between TypeVariable, WildcardType, and AnyType, changing
// the signature of the fingerprinted object. This, however, doesn't work as it breaks bi-
// directional fingerprints. That is, fingerprinting a concrete instance of a generic
// type (Example<Int>), creates a different fingerprint from the generic type itself (Example<T>)
//
// Note, TypeVariable<*> used to be encoded as TYPE_VARIABLE_HASH but that again produces a
// differing fingerprint on serialisation and deserialization
// On serialization Example<Int> is treated as Example<T>, a TypeVariable
// On deserialisation it is seen as Example<?>, A WildcardType *and* a TypeVariable
// Note: AnyType is a special case of WildcardType used in other parts of the
// serializer so both cases need to be dealt with here
//
// If we treat these types as fundamentally different and alter the fingerprint we will
// end up breaking into the evolver when we shouldn't or, worse, evoking the carpenter.
is SerializerFactory.AnyType,
is WildcardType,
is TypeVariable<*> -> {
hasher.putUnencodedChars("?").putUnencodedChars(ANY_TYPE_HASH)
}
is Class<*> -> {
if (type.isArray) {
fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, offset+4)
fingerprintForType(type.componentType, contextType, alreadySeen, hasher, factory, debugIndent+1)
.putUnencodedChars(ARRAY_HASH)
} else if (SerializerFactory.isPrimitive(type)) {
hasher.putUnencodedChars(type.name)
@ -412,17 +420,15 @@ private fun fingerprintForType(type: Type, contextType: Type?, alreadySeen: Muta
// to the CorDapp but maybe reference to the JAR in the short term.
hasher.putUnencodedChars(type.name)
} else {
fingerprintForObject(type, type, alreadySeen, hasher, factory, offset+4)
fingerprintForObject(type, type, alreadySeen, hasher, factory, debugIndent+1)
}
}
}
}
// Hash the element type + some array hash
is GenericArrayType -> fingerprintForType(type.genericComponentType, contextType, alreadySeen,
hasher, factory, offset+4).putUnencodedChars(ARRAY_HASH)
// TODO: include bounds
is WildcardType -> {
hasher.putUnencodedChars(type.typeName).putUnencodedChars(WILDCARD_TYPE_HASH)
is GenericArrayType -> {
fingerprintForType(type.genericComponentType, contextType, alreadySeen,
hasher, factory, debugIndent+1).putUnencodedChars(ARRAY_HASH)
}
else -> throw NotSerializableException("Don't know how to hash")
}
@ -443,16 +449,17 @@ private fun fingerprintForObject(
alreadySeen: MutableSet<Type>,
hasher: Hasher,
factory: SerializerFactory,
offset: Int = 0): Hasher {
debugIndent: Int = 0): Hasher {
// Hash the class + properties + interfaces
val name = type.asClass()?.name ?: throw NotSerializableException("Expected only Class or ParameterizedType but found $type")
propertiesForSerialization(constructorForDeserialization(type), contextType ?: type, factory)
.serializationOrder
.fold(hasher.putUnencodedChars(name)) { orig, prop ->
fingerprintForType(prop.getter.resolvedType, type, alreadySeen, orig, factory, offset+4)
fingerprintForType(prop.getter.resolvedType, type, alreadySeen, orig, factory, debugIndent+1)
.putUnencodedChars(prop.getter.name)
.putUnencodedChars(if (prop.getter.mandatory) NOT_NULLABLE_HASH else NULLABLE_HASH)
}
interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, offset+4) }
interfacesForSerialization(type, factory).map { fingerprintForType(it, type, alreadySeen, hasher, factory, debugIndent+4) }
return hasher
}

30
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationHelper.kt
View File

@ -62,11 +62,12 @@ internal fun constructorForDeserialization(type: Type): KFunction<Any>? {
}
/**
* Identifies the properties to be used during serialization by attempting to find those that match the parameters to the
* deserialization constructor, if the class is concrete. If it is abstract, or an interface, then use all the properties.
* Identifies the properties to be used during serialization by attempting to find those that match the parameters
* to the deserialization constructor, if the class is concrete. If it is abstract, or an interface, then use all
* the properties.
*
* Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters have
* names accessible via reflection.
* Note, you will need any Java classes to be compiled with the `-parameters` option to ensure constructor parameters
* have names accessible via reflection.
*/
internal fun <T : Any> propertiesForSerialization(
kotlinConstructor: KFunction<T>?,
@ -113,8 +114,8 @@ internal fun <T : Any> propertiesForSerializationFromConstructor(
val returnType = resolveTypeVariables(getter.genericReturnType, type)
if (!constructorParamTakesReturnTypeOfGetter(returnType, getter.genericReturnType, param.value)) {
throw NotSerializableException(
"Property type $returnType for $name of $clazz differs from constructor parameter "
+ "type ${param.value.type.javaType}")
"Property type '$returnType' for '$name' of '$clazz' differs from constructor parameter "
+ "type '${param.value.type.javaType}'")
}
Pair(PublicPropertyReader(getter), returnType)
@ -165,9 +166,20 @@ private fun propertiesForSerializationFromSetters(
}
}
private fun constructorParamTakesReturnTypeOfGetter(getterReturnType: Type, rawGetterReturnType: Type, param: KParameter): Boolean {
val typeToken = TypeToken.of(param.type.javaType)
return typeToken.isSupertypeOf(getterReturnType) || typeToken.isSupertypeOf(rawGetterReturnType)
private fun constructorParamTakesReturnTypeOfGetter(
getterReturnType: Type,
rawGetterReturnType: Type,
param: KParameter): Boolean {
val paramToken = TypeToken.of(param.type.javaType)
val rawParamType = TypeToken.of(paramToken.rawType)
return paramToken.isSupertypeOf(getterReturnType)
|| paramToken.isSupertypeOf(rawGetterReturnType)
// cope with the case where the constructor parameter is a generic type (T etc) but we
// can discover it's raw type. When bounded this wil be the bounding type, unbounded
// generics this will be object
|| rawParamType.isSupertypeOf(getterReturnType)
|| rawParamType.isSupertypeOf(rawGetterReturnType)
}
private fun propertiesForSerializationFromAbstract(

12
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutput.kt
View File

@ -86,15 +86,15 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory
data.putObject(transformsSchema)
}
internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type) {
internal fun writeObjectOrNull(obj: Any?, data: Data, type: Type, debugIndent: Int) {
if (obj == null) {
data.putNull()
} else {
writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type)
writeObject(obj, data, if (type == SerializerFactory.AnyType) obj.javaClass else type, debugIndent)
}
}
internal fun writeObject(obj: Any, data: Data, type: Type) {
internal fun writeObject(obj: Any, data: Data, type: Type, debugIndent: Int = 0) {
val serializer = serializerFactory.get(obj.javaClass, type)
if (serializer !in serializerHistory) {
serializerHistory.add(serializer)
@ -103,11 +103,13 @@ open class SerializationOutput(internal val serializerFactory: SerializerFactory
val retrievedRefCount = objectHistory[obj]
if (retrievedRefCount == null) {
serializer.writeObject(obj, data, type, this)
serializer.writeObject(obj, data, type, this, debugIndent)
// Important to do it after serialization such that dependent object will have preceding reference numbers
// assigned to them first as they will be first read from the stream on receiving end.
// Skip for primitive types as they are too small and overhead of referencing them will be much higher than their content
if (suitableForObjectReference(obj.javaClass)) objectHistory.put(obj, objectHistory.size)
if (suitableForObjectReference(obj.javaClass)) {
objectHistory.put(obj, objectHistory.size)
}
} else {
data.writeReferencedObject(ReferencedObject(retrievedRefCount))
}

1
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializerFactory.kt
View File

@ -218,7 +218,6 @@ open class SerializerFactory(
for (typeNotation in schemaAndDescriptor.schemas.schema.types) {
try {
val serialiser = processSchemaEntry(typeNotation)
// if we just successfully built a serializer for the type but the type fingerprint
// doesn't match that of the serialised object then we are dealing with different
// instance of the class, as such we need to build an EvolutionSerializer

2
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/SingletonSerializer.kt
View File

@ -22,7 +22,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto
output.writeTypeNotations(typeNotation)
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, debugIndent: Int) {
data.withDescribed(typeNotation.descriptor) {
data.putBoolean(false)
}

7
node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/carpenter/AMQPSchemaExtensions.kt
View File

@ -124,7 +124,12 @@ fun AMQPField.getTypeAsClass(classloader: ClassLoader) = typeStrToType[Pair(type
"string" -> String::class.java
"binary" -> ByteArray::class.java
"*" -> if (requires.isEmpty()) Any::class.java else classloader.loadClass(requires[0])
else -> classloader.loadClass(type)
else -> {
classloader.loadClass(
if (type.endsWith("?>")) {
type.substring(0, type.indexOf('<'))
} else type)
}
}
fun AMQPField.validateType(classloader: ClassLoader) = when (type) {

1
node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/EvolvabilityTests.kt
View File

@ -6,6 +6,7 @@ import net.corda.testing.common.internal.ProjectStructure.projectRootDir
import org.junit.Test
import java.io.File
import java.io.NotSerializableException
import java.net.URI
import kotlin.test.assertEquals
// To regenerate any of the binary test files do the following

254
node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/GenericsTests.kt
View File

@ -1,21 +1,39 @@
package net.corda.nodeapi.internal.serialization.amqp
import net.corda.core.contracts.*
import net.corda.core.serialization.SerializedBytes
import net.corda.nodeapi.internal.serialization.AllWhitelist
import net.corda.testing.common.internal.ProjectStructure.projectRootDir
import org.junit.Test
import net.corda.core.identity.AbstractParty
import net.corda.core.identity.CordaX500Name
import net.corda.core.identity.Party
import net.corda.core.transactions.WireTransaction
import net.corda.testing.core.TestIdentity
import org.hibernate.Transaction
import java.io.File
import java.net.URI
import java.util.*
import java.util.concurrent.ConcurrentHashMap
import kotlin.test.assertEquals
data class TestContractState(
override val participants: List<AbstractParty>
) : ContractState
class TestAttachmentConstraint : AttachmentConstraint {
override fun isSatisfiedBy(attachment: Attachment) = true
}
class GenericsTests {
companion object {
val VERBOSE = false
val VERBOSE = true
@Suppress("UNUSED")
var localPath = projectRootDir.toUri().resolve(
"node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp")
val miniCorp = TestIdentity(CordaX500Name("MiniCorp", "London", "GB"))
}
private fun printSeparator() = if (VERBOSE) println("\n\n-------------------------------------------\n\n") else Unit
@ -252,29 +270,235 @@ class GenericsTests {
File(GenericsTests::class.java.getResource(resource).toURI()).readBytes())).t)
}
interface DifferentBounds {
fun go()
data class StateAndString(val state: TransactionState<*>, val ref: String)
data class GenericStateAndString<out T: ContractState>(val state: TransactionState<T>, val ref: String)
//
// If this doesn't blow up all is fine
private fun fingerprintingDiffersStrip(state: Any) {
class cl : ClassLoader()
val m = ClassLoader::class.java.getDeclaredMethod("findLoadedClass", *arrayOf<Class<*>>(String::class.java))
m.isAccessible = true
val factory1 = testDefaultFactory()
factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state)
// attempt at having a class loader without some of the derived non core types loaded and thus
// possibly altering how we serialise things
val altClassLoader = cl()
val factory2 = SerializerFactory(AllWhitelist, altClassLoader)
factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
val ser2 = TestSerializationOutput(VERBOSE, factory2).serializeAndReturnSchema(state)
// now deserialise those objects
val factory3 = testDefaultFactory()
factory3.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
val des1 = DeserializationInput(factory3).deserializeAndReturnEnvelope(ser1.obj)
val factory4 = SerializerFactory(AllWhitelist, cl())
factory4.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
val des2 = DeserializationInput(factory4).deserializeAndReturnEnvelope(ser2.obj)
}
@Test
fun differentBounds() {
data class A (val a: Int): DifferentBounds {
override fun go() {
println(a)
}
fun fingerprintingDiffers() {
val state = TransactionState<TestContractState> (
TestContractState(listOf(miniCorp.party)),
"wibble", miniCorp.party,
encumbrance = null,
constraint = TestAttachmentConstraint())
val sas = StateAndString(state, "wibble")
fingerprintingDiffersStrip(sas)
}
data class G<out T : DifferentBounds>(val b: T)
@Test
fun fingerprintingDiffersList() {
val state = TransactionState<TestContractState> (
TestContractState(listOf(miniCorp.party)),
"wibble", miniCorp.party,
encumbrance = null,
constraint = TestAttachmentConstraint())
val factorys = listOf(
SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()),
SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader()))
val sas = StateAndString(state, "wibble")
val ser = SerializationOutput(factorys[0])
fingerprintingDiffersStrip(Collections.singletonList(sas))
}
ser.serialize(G(A(10))).apply {
factorys.forEach {
//
// Force object to be serialised as Example<T> and deserialized as Example<?>
//
@Test
fun fingerprintingDiffersListLoaded() {
//
// using this wrapper class we force the object to be serialised as
// net.corda.core.contracts.TransactionState<T>
//
data class TransactionStateWrapper<out T : ContractState> (val o: List<GenericStateAndString<T>>)
val state = TransactionState<TestContractState> (
TestContractState(listOf(miniCorp.party)),
"wibble", miniCorp.party,
encumbrance = null,
constraint = TestAttachmentConstraint())
val sas = GenericStateAndString(state, "wibble")
val factory1 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
factory1.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(
TransactionStateWrapper(Collections.singletonList(sas)))
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(sas.ref, des1.obj.o.firstOrNull()?.ref ?: "WILL NOT MATCH")
}
@Test
fun nestedGenericsWithBound() {
open class BaseState(val a : Int)
class DState(a: Int) : BaseState(a)
data class LTransactionState<out T : BaseState> constructor(val data: T)
data class StateWrapper<out T : BaseState>(val state: LTransactionState<T>)
val factory1 = testDefaultFactoryNoEvolution()
val state = LTransactionState(DState(1020304))
val stateAndString = StateWrapper(state)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString)
//val factory2 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.data.a, des1.obj.state.data.a)
}
@Test
fun nestedMultiGenericsWithBound() {
open class BaseState(val a : Int)
class DState(a: Int) : BaseState(a)
class EState(a: Int, val msg: String) : BaseState(a)
data class LTransactionState<out T1 : BaseState, out T2: BaseState> (val data: T1, val context: T2)
data class StateWrapper<out T1 : BaseState, out T2: BaseState>(val state: LTransactionState<T1, T2>)
val factory1 = testDefaultFactoryNoEvolution()
val state = LTransactionState(DState(1020304), EState(5060708, msg = "thigns"))
val stateAndString = StateWrapper(state)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString)
//val factory2 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.data.a, des1.obj.state.data.a)
assertEquals(state.context.a, des1.obj.state.context.a)
}
@Test
fun nestedMultiGenericsNoBound() {
open class BaseState(val a : Int)
class DState(a: Int) : BaseState(a)
class EState(a: Int, val msg: String) : BaseState(a)
data class LTransactionState<out T1, out T2> (val data: T1, val context: T2)
data class StateWrapper<out T1, out T2>(val state: LTransactionState<T1, T2>)
val factory1 = testDefaultFactoryNoEvolution()
val state = LTransactionState(DState(1020304), EState(5060708, msg = "things"))
val stateAndString = StateWrapper(state)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString)
//val factory2 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.data.a, des1.obj.state.data.a)
assertEquals(state.context.a, des1.obj.state.context.a)
assertEquals(state.context.msg, des1.obj.state.context.msg)
}
@Test
fun baseClassInheritedButNotOverriden() {
val factory1 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
open class BaseState<T1, T2>(open val a : T1, open val b: T2)
class DState<T1, T2>(a: T1, b: T2) : BaseState<T1, T2>(a, b)
val state = DState(100, "hello")
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state)
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.a, des1.obj.a)
assertEquals(state.b, des1.obj.b)
class DState2<T1, T2, T3>(a: T1, b: T2, val c: T3) : BaseState<T1, T2>(a, b)
val state2 = DState2(100, "hello", 100L)
val ser2 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state2)
val des2 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser2.obj)
assertEquals(state2.a, des2.obj.a)
assertEquals(state2.b, des2.obj.b)
assertEquals(state2.c, des2.obj.c)
}
@Test
fun baseClassInheritedButNotOverridenBounded() {
val factory1 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
open class Bound(val a: Int)
open class BaseState<out T1 : Bound>(open val a: T1)
class DState<out T1: Bound>(a: T1) : BaseState<T1>(a)
val state = DState(Bound(100))
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(state)
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.a.a, des1.obj.a.a)
}
@Test
fun nestedMultiGenericsAtBottomWithBound() {
open class BaseState<T1, T2>(val a : T1, val b: T2)
class DState<T1, T2>(a: T1, b: T2) : BaseState<T1, T2>(a, b)
class EState<T1, T2>(a: T1, b: T2, val c: Long) : BaseState<T1, T2>(a, b)
data class LTransactionState<T1, T2, T3 : BaseState<T1, T2>, out T4: BaseState<T1, T2>> (val data: T3, val context: T4)
data class StateWrapper<T1, T2, T3 : BaseState<T1, T2>, out T4: BaseState<T1, T2>>(val state: LTransactionState<T1, T2, T3, T4>)
val factory1 = testDefaultFactoryNoEvolution()
val state = LTransactionState(DState(1020304, "Hello"), EState(5060708, "thins", 100L))
val stateAndString = StateWrapper(state)
val ser1 = TestSerializationOutput(VERBOSE, factory1).serializeAndReturnSchema(stateAndString)
//val factory2 = testDefaultFactoryNoEvolution()
val factory2 = testDefaultFactory()
val des1 = DeserializationInput(factory2).deserializeAndReturnEnvelope(ser1.obj)
assertEquals(state.data.a, des1.obj.state.data.a)
assertEquals(state.context.a, des1.obj.state.context.a)
}
}