Add better test, add support for constructor determination

This commit is contained in:
Katelyn Baker 2017-08-18 16:33:18 +01:00
parent 1d131eced5
commit 6bcfb2eddf
15 changed files with 330 additions and 60 deletions

View File

@ -0,0 +1,111 @@
package net.corda.nodeapi.internal.serialization.amqp
import net.corda.nodeapi.internal.serialization.carpenter.getTypeAsClass
import org.apache.qpid.proton.codec.Data
import java.lang.reflect.Type
import java.io.NotSerializableException
import kotlin.reflect.KFunction
import kotlin.reflect.full.primaryConstructor
import kotlin.reflect.full.findAnnotation
import kotlin.reflect.jvm.javaType
/**
*
*/
class EvolutionSerializer(
clazz: Type,
factory: SerializerFactory,
val oldParams : Map<String, oldParam>,
override val kotlinConstructor: KFunction<Any>?) : ObjectSerializer (clazz, factory) {
// explicitly null this out as we won't be using this list
override val propertySerializers: Collection<PropertySerializer> = listOf()
/**
* represents a paramter as would be passed to the constructor of the class as it was
* when it was serialised and NOT how that class appears now
*/
data class oldParam (val type: Type, val idx: Int, val property: PropertySerializer)
companion object {
/**
* Unlike the generic deserialisation case where we need to locate the primary constructor
* for the object (or our best guess) in the case of an object whose structure has changed
* since serialisation we need to attempt to locate a constructor that we can use. I.e.
* it's parameters match the serialised members and it will initialise any newly added
* elements
*
* TODO: Type evolution
* TODO: rename annotation
*/
internal fun getEvolverConstructor(type: Type, oldArgs: Map<String?, Type>): KFunction<Any>? {
val clazz: Class<*> = type.asClass()!!
if (!isConcrete(clazz)) return null
val oldArgumentSet = oldArgs.map { Pair (it.key, it.value) }
clazz.kotlin.constructors.forEach {
if (oldArgumentSet.containsAll(it.parameters.map { v -> Pair(v.name, v.type.javaType) })) {
return it
}
}
// if we didn't get an exact match revert to existing behaviour, if the new parameters
// are not mandatory (i.e. nullable) things are fine
return constructorForDeserialization(type)
}
/**
* Build a serialization object for deserialisation only of objects serislaised
* as different versions of a class
*
* @param old is an object holding the schema that represents the object
* as it was serialised and the type descriptor of that type
* @param new is the Serializer built for the Class as it exists now, not
* how it was serialised and persisted.
*/
fun make (old: schemaAndDescriptor, new: ObjectSerializer,
factory: SerializerFactory) : AMQPSerializer<Any> {
val oldFieldToType = (old.schema.types.first() as CompositeType).fields.map {
it.name as String? to it.getTypeAsClass(factory.classloader) as Type
}.toMap()
val constructor = getEvolverConstructor(new.type, oldFieldToType) ?:
throw NotSerializableException(
"Attempt to deserialize an interface: new.type. Serialized form is invalid.")
val oldArgs = mutableMapOf<String, oldParam>()
var idx = 0
(old.schema.types.first() as CompositeType).fields.forEach {
val returnType = it.getTypeAsClass(factory.classloader)
oldArgs[it.name] = oldParam(
returnType, idx++, PropertySerializer.make(it.name, null, returnType, factory))
}
return EvolutionSerializer(new.type, factory, oldArgs, constructor)
}
}
override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput) {
throw IllegalAccessException ("It should be impossible to write an evolution serializer")
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any {
if (obj !is List<*>) throw NotSerializableException("Body of described type is unexpected $obj")
val newArgs = kotlinConstructor?.parameters?.associateBy({ it.name!! }, {it.type.isMarkedNullable}) ?:
throw NotSerializableException ("Bad Constructor selected for object $obj")
return construct(newArgs.map {
val param = oldParams[it.key]
if (param == null && !it.value) {
throw NotSerializableException(
"New parameter ${it.key} is mandatory, should be nullable for evolution to worK")
}
param?.property?.readProperty(obj[param.idx], schema, input)
})
}
}

View File

@ -11,16 +11,12 @@ import kotlin.reflect.jvm.javaConstructor
/**
* Responsible for serializing and deserializing a regular object instance via a series of properties (matched with a constructor).
*/
class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPSerializer<Any> {
open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPSerializer<Any> {
override val type: Type get() = clazz
private val javaConstructor: Constructor<Any>?
internal val propertySerializers: Collection<PropertySerializer>
open internal val propertySerializers: Collection<PropertySerializer>
open val kotlinConstructor = constructorForDeserialization(clazz)
init {
println ("Object Serializer")
val kotlinConstructor = constructorForDeserialization(clazz)
javaConstructor = kotlinConstructor?.javaConstructor
javaConstructor?.isAccessible = true
propertySerializers = propertiesForSerialization(kotlinConstructor, clazz, factory)
}
@ -29,7 +25,7 @@ class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPSerial
override val typeDescriptor = "$DESCRIPTOR_DOMAIN:${fingerprintForType(type, factory)}"
private val interfaces = interfacesForSerialization(clazz, factory) // We restrict to only those annotated or whitelisted
internal val typeNotation: TypeNotation = CompositeType(typeName, null, generateProvides(), Descriptor(typeDescriptor, null), generateFields())
open internal val typeNotation : TypeNotation by lazy {CompositeType(typeName, null, generateProvides(), Descriptor(typeDescriptor, null), generateFields()) }
override fun writeClassInfo(output: SerializationOutput) {
if (output.writeTypeNotations(typeNotation)) {
@ -75,9 +71,9 @@ class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPSerial
fun construct(properties: List<Any?>): Any {
if (javaConstructor == null) {
val javaConstructor = kotlinConstructor?.javaConstructor ?:
throw NotSerializableException("Attempt to deserialize an interface: $clazz. Serialized form is invalid.")
}
return javaConstructor.newInstance(*properties.toTypedArray())
}
}
}

View File

@ -10,7 +10,7 @@ import kotlin.reflect.jvm.javaGetter
/**
* Base class for serialization of a property of an object.
*/
sealed class PropertySerializer(val name: String, val readMethod: Method, val resolvedType: Type) {
sealed class PropertySerializer(val name: String, val readMethod: Method?, val resolvedType: Type) {
abstract fun writeClassInfo(output: SerializationOutput)
abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput)
abstract fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any?
@ -44,7 +44,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
}
private fun generateMandatory(): Boolean {
return isJVMPrimitive || !readMethod.returnsNullable()
return isJVMPrimitive || !(readMethod?.returnsNullable() ?: true)
}
private fun Method.returnsNullable(): Boolean {
@ -69,7 +69,10 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
/**
* A property serializer for a complex type (another object).
*/
class DescribedTypePropertySerializer(name: String, readMethod: Method, resolvedType: Type, private val lazyTypeSerializer: () -> AMQPSerializer<*>) : PropertySerializer(name, readMethod, resolvedType) {
class DescribedTypePropertySerializer(
name: String, readMethod: Method?,
resolvedType: Type,
private val lazyTypeSerializer: () -> AMQPSerializer<*>) : PropertySerializer(name, readMethod, resolvedType) {
// This is lazy so we don't get an infinite loop when a method returns an instance of the class.
private val typeSerializer: AMQPSerializer<*> by lazy { lazyTypeSerializer() }
@ -84,14 +87,14 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
output.writeObjectOrNull(readMethod.invoke(obj), data, resolvedType)
output.writeObjectOrNull(readMethod!!.invoke(obj), data, resolvedType)
}
}
/**
* A property serializer for most AMQP primitive type (Int, String, etc).
*/
class AMQPPrimitivePropertySerializer(name: String, readMethod: Method, resolvedType: Type) : PropertySerializer(name, readMethod, resolvedType) {
class AMQPPrimitivePropertySerializer(name: String, readMethod: Method?, resolvedType: Type) : PropertySerializer(name, readMethod, resolvedType) {
override fun writeClassInfo(output: SerializationOutput) {}
override fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any? {
@ -99,7 +102,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
val value = readMethod.invoke(obj)
val value = readMethod!!.invoke(obj)
if (value is ByteArray) {
data.putObject(Binary(value))
} else {
@ -113,7 +116,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
* value of the character is stored in numeric UTF-16 form and on deserialisation requires explicit
* casting back to a char otherwise it's treated as an Integer and a TypeMismatch occurs
*/
class AMQPCharPropertySerializer(name: String, readMethod: Method) :
class AMQPCharPropertySerializer(name: String, readMethod: Method?) :
PropertySerializer(name, readMethod, Character::class.java) {
override fun writeClassInfo(output: SerializationOutput) {}
@ -122,7 +125,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method, val re
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) {
val input = readMethod.invoke(obj)
val input = readMethod!!.invoke(obj)
if (input != null) data.putShort((input as Char).toShort()) else data.putNull()
}
}

View File

@ -65,10 +65,9 @@ internal fun <T : Any> propertiesForSerialization(kotlinConstructor: KFunction<T
return if (kotlinConstructor != null) propertiesForSerializationFromConstructor(kotlinConstructor, type, factory) else propertiesForSerializationFromAbstract(clazz, type, factory)
}
private fun isConcrete(clazz: Class<*>): Boolean = !(clazz.isInterface || Modifier.isAbstract(clazz.modifiers))
fun isConcrete(clazz: Class<*>): Boolean = !(clazz.isInterface || Modifier.isAbstract(clazz.modifiers))
private fun <T : Any> propertiesForSerializationFromConstructor(kotlinConstructor: KFunction<T>, type: Type, factory: SerializerFactory): Collection<PropertySerializer> {
println ("propertiesForSerializationFromConstructor - $type")
val clazz = (kotlinConstructor.returnType.classifier as KClass<*>).javaObjectType
// Kotlin reflection doesn't work with Java getters the way you might expect, so we drop back to good ol' beans.
val properties = Introspector.getBeanInfo(clazz).propertyDescriptors.filter { it.name != "class" }.groupBy { it.name }.mapValues { it.value[0] }

View File

@ -16,6 +16,8 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import javax.annotation.concurrent.ThreadSafe
data class schemaAndDescriptor (val schema: Schema, val typeDescriptor: Any)
/**
* Factory of serializers designed to be shared across threads and invocations.
*/
@ -48,6 +50,12 @@ class SerializerFactory(val whitelist: ClassWhitelist, cl : ClassLoader) {
val classloader : ClassLoader
get() = classCarpenter.classloader
fun getEvolutionSerializer(schema: schemaAndDescriptor, newSerializer: ObjectSerializer) : AMQPSerializer<Any> {
return serializersByDescriptor.computeIfAbsent(schema.typeDescriptor) {
EvolutionSerializer.make(schema, newSerializer, this)
}
}
/**
* Look up, and manufacture if necessary, a serializer for the given type.
*
@ -56,14 +64,11 @@ class SerializerFactory(val whitelist: ClassWhitelist, cl : ClassLoader) {
*/
@Throws(NotSerializableException::class)
fun get(actualClass: Class<*>?, declaredType: Type): AMQPSerializer<Any> {
println ("get - $declaredType")
val declaredClass = declaredType.asClass() ?: throw NotSerializableException(
"Declared types of $declaredType are not supported.")
val actualType: Type = inferTypeVariables(actualClass, declaredClass, declaredType) ?: declaredType
println ("actual type - $actualType")
val serializer = if (Collection::class.java.isAssignableFrom(declaredClass)) {
serializersByType.computeIfAbsent(declaredType) {
CollectionSerializer(declaredType as? ParameterizedType ?: DeserializedParameterizedType(
@ -182,14 +187,19 @@ class SerializerFactory(val whitelist: ClassWhitelist, cl : ClassLoader) {
* Iterate over an AMQP schema, for each type ascertain weather it's on ClassPath of [classloader] amd
* if not use the [ClassCarpenter] to generate a class to use in it's place
*/
private fun processSchema(schema: Schema, sentinel: Boolean = false) {
private fun processSchema(schema: schemaAndDescriptor, sentinel: Boolean = false) {
val carpenterSchemas = CarpenterSchemas.newInstance()
for (typeNotation in schema.types) {
println("processSchema: ${typeNotation.descriptor} ${typeNotation.name}")
for (typeNotation in schema.schema.types) {
try {
processSchemaEntry(typeNotation)
val serialiser = processSchemaEntry(typeNotation)
// if we just successfully built a serialiser for the type but the type fingerprint
// doesn't match that of the serialised object then we are dealing with an older
// instance of the class, as such we need to build an evolverSerilaiser
if (serialiser.typeDescriptor != schema.typeDescriptor) {
getEvolutionSerializer(schema, serialiser as ObjectSerializer)
}
} catch (e: ClassNotFoundException) {
println("poop")
if (sentinel || (typeNotation !is CompositeType)) throw e
typeNotation.carpenterSchema(classloader, carpenterSchemas = carpenterSchemas)
}
@ -202,24 +212,21 @@ class SerializerFactory(val whitelist: ClassWhitelist, cl : ClassLoader) {
}
}
private fun processSchemaEntry(typeNotation: TypeNotation) {
when (typeNotation) {
private fun processSchemaEntry(typeNotation: TypeNotation) = when (typeNotation) {
is CompositeType -> processCompositeType(typeNotation) // java.lang.Class (whether a class or interface)
is RestrictedType -> processRestrictedType(typeNotation) // Collection / Map, possibly with generics
}
}
private fun processRestrictedType(typeNotation: RestrictedType) {
private fun processRestrictedType(typeNotation: RestrictedType): AMQPSerializer<Any> {
// TODO: class loader logic, and compare the schema.
val type = typeForName(typeNotation.name, classloader)
get(null, type)
return get(null, type)
}
private fun processCompositeType(typeNotation: CompositeType) {
private fun processCompositeType(typeNotation: CompositeType): AMQPSerializer<Any> {
// TODO: class loader logic, and compare the schema.
println("processCompositeType")
val type = typeForName(typeNotation.name, classloader)
get(type.asClass() ?: throw NotSerializableException("Unable to build composite type for $type"), type)
return get(type.asClass() ?: throw NotSerializableException("Unable to build composite type for $type"), type)
}
private fun makeClassSerializer(clazz: Class<*>, type: Type, declaredType: Type): AMQPSerializer<Any> = serializersByType.computeIfAbsent(type) {

View File

@ -19,7 +19,7 @@ class ThrowableSerializer(factory: SerializerFactory) : CustomSerializer.Proxy<T
val constructor = constructorForDeserialization(obj.javaClass)
val props = propertiesForSerialization(constructor, obj.javaClass, factory)
for (prop in props) {
extraProperties[prop.name] = prop.readMethod.invoke(obj)
extraProperties[prop.name] = prop.readMethod!!.invoke(obj)
}
} catch(e: NotSerializableException) {
}

View File

@ -1,37 +1,191 @@
package net.corda.nodeapi.internal.serialization.amqp
import net.corda.core.serialization.SerializedBytes
import net.corda.core.utilities.toHexString
import org.junit.Test
import java.io.File
import java.io.NotSerializableException
import kotlin.test.assertEquals
class EvolvabilityTests {
@Test
fun test1() {
val sf = SerializerFactory()
// Basis for the serialised version
// data class C (val a: Int)
// var sc = SerializationOutput(sf).serialize(C(1))
data class C (val a: Int, val b: Int)
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.test1")
println ("PATH = $path")
fun simpleOrderSwapSameType() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.simpleOrderSwapSameType")
val f = File(path.toURI())
println (sf)
// var sc = SerializationOutput(sf).serialize(C(1))
// f.writeBytes(sc.bytes)
val A = 1
val B = 2
// Original version of the class for the serialised version of this class
//
// data class C (val a: Int, val b: Int)
// val sc = SerializationOutput(sf).serialize(C(A, B))
// f.writeBytes(sc.bytes)
// new version of the class, in this case the order of the parameters has been swapped
data class C (val b: Int, val a: Int)
val sc2 = f.readBytes()
val deserializedC = DeserializationInput(sf).deserialize(SerializedBytes<C>(sc2))
assertEquals(A, deserializedC.a)
assertEquals(B, deserializedC.b)
}
@Test
fun simpleOrderSwapDifferentType() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.simpleOrderSwapDifferentType")
val f = File(path.toURI())
val A = 1
val B = "two"
// Original version of the class as it was serialised
//
// data class C (val a: Int, val b: String)
// val sc = SerializationOutput(sf).serialize(C(A, B))
// f.writeBytes(sc.bytes)
// new version of the class, in this case the order of the parameters has been swapped
data class C (val b: String, val a: Int)
val sc2 = f.readBytes()
val deserializedC = DeserializationInput(sf).deserialize(SerializedBytes<C>(sc2))
assertEquals(A, deserializedC.a)
assertEquals(B, deserializedC.b)
}
@Test
fun addAdditionalParamNotMandatory() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.addAdditionalParamNotMandatory")
val f = File(path.toURI())
val A = 1
// Original version of the class as it was serialised
//
// data class C(val a: Int)
// val sc = SerializationOutput(sf).serialize(C(A))
// f.writeBytes(sc.bytes)
// println ("Path = $path")
data class C (val a: Int, val b: Int?)
val sc2 = f.readBytes()
val deserializedC = DeserializationInput(sf).deserialize(SerializedBytes<C>(sc2))
assertEquals (A, deserializedC.a)
assertEquals (null, deserializedC.b)
}
@Test(expected = NotSerializableException::class)
fun addAdditionalParam() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.addAdditionalParam")
val f = File(path.toURI())
@Suppress("UNUSED_VARIABLE")
val A = 1
// Original version of the class as it was serialised
//
// data class C(val a: Int)
// val sc = SerializationOutput(sf).serialize(C(A))
// f.writeBytes(sc.bytes)
// println ("Path = $path")
// new version of the class, in this case a new parameter has been added (b)
data class C (val a: Int, val b: Int)
val sc2 = f.readBytes()
var deserializedC = DeserializationInput().deserialize(SerializedBytes<C>(sc2))
println (deserializedC.a)
// Expected to throw as we can't construct the new type as it contains a newly
// added parameter that isn't optional, i.e. not nullable and there isn't
// a compiler that takes the old parameters
DeserializationInput(sf).deserialize(SerializedBytes<C>(sc2))
}
@Suppress("UNUSED_VARIABLE")
@Test
fun removeParameters() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.removeParameters")
val f = File(path.toURI())
val A = 1
val B = "two"
val C = "three"
val D = 4
// Original version of the class as it was serialised
//
// data class CC(val a: Int, val b: String, val c: String, val d: Int)
// val scc = SerializationOutput(sf).serialize(CC(A, B, C, D))
// f.writeBytes(scc.bytes)
// println ("Path = $path")
data class CC (val b: String, val d: Int)
val sc2 = f.readBytes()
val deserializedCC = DeserializationInput(sf).deserialize(SerializedBytes<CC>(sc2))
assertEquals (B, deserializedCC.b)
assertEquals (D, deserializedCC.d)
}
@Suppress("UNUSED_VARIABLE")
@Test
fun addAndRemoveParameters() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.addAndRemoveParameters")
val f = File(path.toURI())
val A = 1
val B = "two"
val C = "three"
val D = 4
val E = null
// Original version of the class as it was serialised
//
// data class CC(val a: Int, val b: String, val c: String, val d: Int)
// val scc = SerializationOutput(sf).serialize(CC(A, B, C, D))
// f.writeBytes(scc.bytes)
// println ("Path = $path")
data class CC(val a: Int, val e: Boolean?, val d: Int)
val sc2 = f.readBytes()
val deserializedCC = DeserializationInput(sf).deserialize(SerializedBytes<CC>(sc2))
assertEquals(A, deserializedCC.a)
assertEquals(E, deserializedCC.e)
assertEquals(D, deserializedCC.d)
}
@Test
fun addMandatoryFieldWithAltConstructor() {
val sf = testDefaultFactory()
val path = EvolvabilityTests::class.java.getResource("EvolvabilityTests.addMandatoryFieldWithAltConstructor")
val f = File(path.toURI())
val A = 1
// Original version of the class as it was serialised
//
// data class CC(val a: Int)
// val scc = SerializationOutput(sf).serialize(CC(A))
// f.writeBytes(scc.bytes)
// println ("Path = $path")
@Suppress("UNUSED")
data class CC (val a: Int, val b: String) {
constructor (a: Int) : this (a, "hello")
}
val sc2 = f.readBytes()
val deserializedCC = DeserializationInput(sf).deserialize(SerializedBytes<CC>(sc2))
assertEquals (A, deserializedCC.a)
assertEquals ("hello", deserializedCC.b)
}
}