Merge pull request #2138 from corda/kat/feature/enumEvolution

CORDA-553 - Plumb the transform schema into the AMQP serialisation framework
This commit is contained in:
Katelyn Baker 2017-11-29 11:28:19 +00:00 committed by GitHub
commit a81352acbb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 113 additions and 53 deletions

View File

@ -26,5 +26,8 @@ class AMQPPrimitiveSerializer(clazz: Class<*>) : AMQPSerializer<Any> {
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any = (obj as? Binary)?.array ?: obj
override fun readObject(
obj: Any,
schemas: SerializationSchemas,
input: DeserializationInput): Any = (obj as? Binary)?.array ?: obj
}

View File

@ -35,5 +35,5 @@ interface AMQPSerializer<out T> {
/**
* Read the given object from the input. The envelope is provided in case the schema is required.
*/
fun readObject(obj: Any, schema: Schema, input: DeserializationInput): T
fun readObject(obj: Any, schema: SerializationSchemas, input: DeserializationInput): T
}

View File

@ -56,9 +56,9 @@ open class ArraySerializer(override val type: Type, factory: SerializerFactory)
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
if (obj is List<*>) {
return obj.map { input.readObjectOrNull(it, schema, elementType) }.toArrayOfType(elementType)
return obj.map { input.readObjectOrNull(it, schemas, elementType) }.toArrayOfType(elementType)
} else throw NotSerializableException("Expected a List but found $obj")
}

View File

@ -77,8 +77,8 @@ class CollectionSerializer(val declaredType: ParameterizedType, factory: Seriali
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
// TODO: Can we verify the entries in the list?
concreteBuilder((obj as List<*>).map { input.readObjectOrNull(it, schema, declaredType.actualTypeArguments[0]) })
concreteBuilder((obj as List<*>).map { input.readObjectOrNull(it, schemas, declaredType.actualTypeArguments[0]) })
}
}

View File

@ -67,8 +67,8 @@ abstract class CustomSerializer<T : Any> : AMQPSerializer<T> {
superClassSerializer.writeDescribedObject(obj, data, type, output)
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): T {
return superClassSerializer.readObject(obj, schema, input)
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): T {
return superClassSerializer.readObject(obj, schemas, input)
}
}
@ -133,8 +133,8 @@ abstract class CustomSerializer<T : Any> : AMQPSerializer<T> {
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): T {
val proxy: P = uncheckedCast(proxySerializer.readObject(obj, schema, input))
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): T {
val proxy: P = uncheckedCast(proxySerializer.readObject(obj, schemas, input))
return fromProxy(proxy)
}
}
@ -166,7 +166,7 @@ abstract class CustomSerializer<T : Any> : AMQPSerializer<T> {
data.putString(unmaker(obj))
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): T {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): T {
val proxy = obj as String
return maker(proxy)
}

View File

@ -97,21 +97,21 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
@Throws(NotSerializableException::class)
fun <T : Any> deserialize(bytes: ByteSequence, clazz: Class<T>): T = des {
val envelope = getEnvelope(bytes)
clazz.cast(readObjectOrNull(envelope.obj, envelope.schema, clazz))
clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz))
}
@Throws(NotSerializableException::class)
fun <T : Any> deserializeAndReturnEnvelope(bytes: SerializedBytes<T>, clazz: Class<T>): ObjectAndEnvelope<T> = des {
val envelope = getEnvelope(bytes)
// Now pick out the obj and schema from the envelope.
ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, envelope.schema, clazz)), envelope)
ObjectAndEnvelope(clazz.cast(readObjectOrNull(envelope.obj, SerializationSchemas(envelope.schema, envelope.transformsSchema), clazz)), envelope)
}
internal fun readObjectOrNull(obj: Any?, schema: Schema, type: Type): Any? {
internal fun readObjectOrNull(obj: Any?, schema: SerializationSchemas, type: Type): Any? {
return if (obj == null) null else readObject(obj, schema, type)
}
internal fun readObject(obj: Any, schema: Schema, type: Type): Any =
internal fun readObject(obj: Any, schemas: SerializationSchemas, type: Type): Any =
if (obj is DescribedType && ReferencedObject.DESCRIPTOR == obj.descriptor) {
// It must be a reference to an instance that has already been read, cheaply and quickly returning it by reference.
val objectIndex = (obj.described as UnsignedInteger).toInt()
@ -127,11 +127,11 @@ class DeserializationInput(internal val serializerFactory: SerializerFactory) {
val objectRead = when (obj) {
is DescribedType -> {
// Look up serializer in factory by descriptor
val serializer = serializerFactory.get(obj.descriptor, schema)
val serializer = serializerFactory.get(obj.descriptor, schemas)
if (SerializerFactory.AnyType != type && serializer.type != type && with(serializer.type) { !isSubClassOf(type) && !materiallyEquivalentTo(type) })
throw NotSerializableException("Described type with descriptor ${obj.descriptor} was " +
"expected to be of type $type but was ${serializer.type}")
serializer.readObject(obj.described, schema, this)
serializer.readObject(obj.described, schemas, this)
}
is Binary -> obj.array
else -> obj // this will be the case for primitive types like [boolean] et al.

View File

@ -27,7 +27,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Seria
output.writeTypeNotations(typeNotation)
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
val enumName = (obj as List<*>)[0] as String
val enumOrd = obj[1] as Int
val fromOrd = type.asClass()!!.enumConstants[enumOrd] as Enum<*>?

View File

@ -32,8 +32,8 @@ class EvolutionSerializer(
* @param property object to read the actual property value
*/
data class OldParam(val type: Type, val idx: Int, val property: PropertySerializer) {
fun readProperty(paramValues: List<*>, schema: Schema, input: DeserializationInput) =
property.readProperty(paramValues[idx], schema, input)
fun readProperty(paramValues: List<*>, schemas: SerializationSchemas, input: DeserializationInput) =
property.readProperty(paramValues[idx], schemas, input)
}
companion object {
@ -121,10 +121,10 @@ class EvolutionSerializer(
*
* TODO: Object references
*/
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
if (obj !is List<*>) throw NotSerializableException("Body of described type is unexpected $obj")
return construct(readers.map { it?.readProperty(obj, schema, input) })
return construct(readers.map { it?.readProperty(obj, schemas, input) })
}
}

View File

@ -88,15 +88,15 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: Serial
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any = ifThrowsAppend({ declaredType.typeName }) {
// TODO: General generics question. Do we need to validate that entries in Maps and Collections match the generic type? Is it a security hole?
val entries: Iterable<Pair<Any?, Any?>> = (obj as Map<*, *>).map { readEntry(schema, input, it) }
val entries: Iterable<Pair<Any?, Any?>> = (obj as Map<*, *>).map { readEntry(schemas, input, it) }
concreteBuilder(entries.toMap())
}
private fun readEntry(schema: Schema, input: DeserializationInput, entry: Map.Entry<Any?, Any?>) =
input.readObjectOrNull(entry.key, schema, declaredType.actualTypeArguments[0]) to
input.readObjectOrNull(entry.value, schema, declaredType.actualTypeArguments[1])
private fun readEntry(schemas: SerializationSchemas, input: DeserializationInput, entry: Map.Entry<Any?, Any?>) =
input.readObjectOrNull(entry.key, schemas, declaredType.actualTypeArguments[0]) to
input.readObjectOrNull(entry.value, schemas, declaredType.actualTypeArguments[1])
// Cannot use * as a bound for EnumMap and EnumSet since * is not an enum. So, we use a sample enum instead.
// We don't actually care about the type, we just need to make the compiler happier.

View File

@ -55,10 +55,15 @@ open class ObjectSerializer(val clazz: Type, factory: SerializerFactory) : AMQPS
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any = ifThrowsAppend({ clazz.typeName }) {
override fun readObject(
obj: Any,
schemas: SerializationSchemas,
input: DeserializationInput): Any = ifThrowsAppend({ clazz.typeName }) {
if (obj is List<*>) {
if (obj.size > propertySerializers.size) throw NotSerializableException("Too many properties in described type $typeName")
val params = obj.zip(propertySerializers).map { it.second.readProperty(it.first, schema, input) }
if (obj.size > propertySerializers.size) {
throw NotSerializableException("Too many properties in described type $typeName")
}
val params = obj.zip(propertySerializers).map { it.second.readProperty(it.first, schemas, input) }
construct(params)
} else throw NotSerializableException("Body of described type is unexpected $obj")
}

View File

@ -14,7 +14,7 @@ import kotlin.reflect.jvm.javaGetter
sealed class PropertySerializer(val name: String, val readMethod: Method?, val resolvedType: Type) {
abstract fun writeClassInfo(output: SerializationOutput)
abstract fun writeProperty(obj: Any?, data: Data, output: SerializationOutput)
abstract fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any?
abstract fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any?
val type: String = generateType()
val requires: List<String> = generateRequires()
@ -91,8 +91,8 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r
}
}
override fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any? = ifThrowsAppend({ nameForDebug }) {
input.readObjectOrNull(obj, schema, resolvedType)
override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? = ifThrowsAppend({ nameForDebug }) {
input.readObjectOrNull(obj, schemas, resolvedType)
}
override fun writeProperty(obj: Any?, data: Data, output: SerializationOutput) = ifThrowsAppend({ nameForDebug }) {
@ -108,7 +108,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r
class AMQPPrimitivePropertySerializer(name: String, readMethod: Method?, resolvedType: Type) : PropertySerializer(name, readMethod, resolvedType) {
override fun writeClassInfo(output: SerializationOutput) {}
override fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any? {
override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? {
return if (obj is Binary) obj.array else obj
}
@ -131,7 +131,7 @@ sealed class PropertySerializer(val name: String, val readMethod: Method?, val r
PropertySerializer(name, readMethod, Character::class.java) {
override fun writeClassInfo(output: SerializationOutput) {}
override fun readProperty(obj: Any?, schema: Schema, input: DeserializationInput): Any? {
override fun readProperty(obj: Any?, schemas: SerializationSchemas, input: DeserializationInput): Any? {
return if (obj == null) null else (obj as Short).toChar()
}

View File

@ -4,7 +4,9 @@ import com.google.common.primitives.Primitives
import com.google.common.reflect.TypeResolver
import net.corda.core.internal.uncheckedCast
import net.corda.core.serialization.ClassWhitelist
import net.corda.nodeapi.internal.serialization.carpenter.*
import net.corda.nodeapi.internal.serialization.carpenter.CarpenterMetaSchema
import net.corda.nodeapi.internal.serialization.carpenter.ClassCarpenter
import net.corda.nodeapi.internal.serialization.carpenter.MetaCarpenter
import org.apache.qpid.proton.amqp.*
import java.io.NotSerializableException
import java.lang.reflect.*
@ -13,7 +15,8 @@ import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import javax.annotation.concurrent.ThreadSafe
data class FactorySchemaAndDescriptor(val schema: Schema, val typeDescriptor: Any)
data class SerializationSchemas(val schema: Schema, val transforms: TransformsSchema)
data class FactorySchemaAndDescriptor(val schemas: SerializationSchemas, val typeDescriptor: Any)
/**
* Factory of serializers designed to be shared across threads and invocations.
@ -40,7 +43,10 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
val classloader: ClassLoader
get() = classCarpenter.classloader
private fun getEvolutionSerializer(typeNotation: TypeNotation, newSerializer: AMQPSerializer<Any>): AMQPSerializer<Any> {
private fun getEvolutionSerializer(
typeNotation: TypeNotation,
newSerializer: AMQPSerializer<Any>,
transforms: TransformsSchema): AMQPSerializer<Any> {
return serializersByDescriptor.computeIfAbsent(typeNotation.descriptor.name!!) {
when (typeNotation) {
is CompositeType -> EvolutionSerializer.make(typeNotation, newSerializer as ObjectSerializer, this)
@ -168,7 +174,7 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
* contained in the [Schema].
*/
@Throws(NotSerializableException::class)
fun get(typeDescriptor: Any, schema: Schema): AMQPSerializer<Any> {
fun get(typeDescriptor: Any, schema: SerializationSchemas): AMQPSerializer<Any> {
return serializersByDescriptor[typeDescriptor] ?: {
processSchema(FactorySchemaAndDescriptor(schema, typeDescriptor))
serializersByDescriptor[typeDescriptor] ?: throw NotSerializableException(
@ -194,9 +200,9 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
* Iterate over an AMQP schema, for each type ascertain weather it's on ClassPath of [classloader] amd
* if not use the [ClassCarpenter] to generate a class to use in it's place
*/
private fun processSchema(schema: FactorySchemaAndDescriptor, sentinel: Boolean = false) {
private fun processSchema(schemaAndDescriptor: FactorySchemaAndDescriptor, sentinel: Boolean = false) {
val metaSchema = CarpenterMetaSchema.newInstance()
for (typeNotation in schema.schema.types) {
for (typeNotation in schemaAndDescriptor.schemas.schema.types) {
try {
val serialiser = processSchemaEntry(typeNotation)
@ -204,7 +210,7 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
// doesn't match that of the serialised object then we are dealing with different
// instance of the class, as such we need to build an EvolutionSerialiser
if (serialiser.typeDescriptor != typeNotation.descriptor.name) {
getEvolutionSerializer(typeNotation, serialiser)
getEvolutionSerializer(typeNotation, serialiser, schemaAndDescriptor.schemas.transforms)
}
} catch (e: ClassNotFoundException) {
if (sentinel) throw e
@ -215,7 +221,7 @@ open class SerializerFactory(val whitelist: ClassWhitelist, cl: ClassLoader) {
if (metaSchema.isNotEmpty()) {
val mc = MetaCarpenter(metaSchema, classCarpenter)
mc.build()
processSchema(schema, true)
processSchema(schemaAndDescriptor, true)
}
}

View File

@ -28,7 +28,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): Any {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): Any {
return singleton
}
}

View File

@ -85,6 +85,9 @@ class UnknownTransform : Transform() {
override val name: String get() = typeName
}
/**
* Used by the unit testing framework
*/
class UnknownTestTransform(val a: Int, val b: Int, val c: Int) : Transform() {
companion object : DescribedTypeConstructor<UnknownTestTransform> {
val typeName = "UnknownTest"

View File

@ -34,8 +34,8 @@ object InputStreamSerializer : CustomSerializer.Implements<InputStream>(InputStr
}
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): InputStream {
val bits = input.readObject(obj, schema, ByteArray::class.java) as ByteArray
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): InputStream {
val bits = input.readObject(obj, schemas, ByteArray::class.java) as ByteArray
return ByteArrayInputStream(bits)
}
}

View File

@ -20,8 +20,8 @@ object PrivateKeySerializer : CustomSerializer.Implements<PrivateKey>(PrivateKey
output.writeObject(obj.encoded, data, clazz)
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): PrivateKey {
val bits = input.readObject(obj, schema, ByteArray::class.java) as ByteArray
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): PrivateKey {
val bits = input.readObject(obj, schemas, ByteArray::class.java) as ByteArray
return Crypto.decodePrivateKey(bits)
}
}

View File

@ -17,8 +17,8 @@ object PublicKeySerializer : CustomSerializer.Implements<PublicKey>(PublicKey::c
output.writeObject(obj.encoded, data, clazz)
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): PublicKey {
val bits = input.readObject(obj, schema, ByteArray::class.java) as ByteArray
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): PublicKey {
val bits = input.readObject(obj, schemas, ByteArray::class.java) as ByteArray
return Crypto.decodePublicKey(bits)
}
}

View File

@ -20,8 +20,8 @@ object X509CertificateSerializer : CustomSerializer.Implements<X509Certificate>(
output.writeObject(obj.encoded, data, clazz)
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): X509Certificate {
val bits = input.readObject(obj, schema, ByteArray::class.java) as ByteArray
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): X509Certificate {
val bits = input.readObject(obj, schemas, ByteArray::class.java) as ByteArray
return X509CertificateFactory().generateCertificate(bits.inputStream())
}
}

View File

@ -0,0 +1,43 @@
package net.corda.nodeapi.internal.serialization.amqp
import net.corda.core.serialization.CordaSerializationTransformEnumDefault
import net.corda.core.serialization.SerializedBytes
import net.corda.testing.common.internal.ProjectStructure.projectRootDir
import org.assertj.core.api.Assertions
import org.junit.Test
import java.io.File
import java.io.NotSerializableException
import java.net.URI
// NOTE: To recreate the test files used by these tests uncomment the original test classes and comment
// the new ones out, then change each test to write out the serialized bytes rather than read
// the file.
class EnumEvolveTests {
var localPath = projectRootDir.toUri().resolve(
"node-api/src/test/resources/net/corda/nodeapi/internal/serialization/amqp")
// Version of the class as it was serialised
//
// @CordaSerializationTransformEnumDefault("D", "C")
// enum class DeserializeNewerSetToUnknown { A, B, C, D }
//
// Version of the class as it's used in the test
enum class DeserializeNewerSetToUnknown { A, B, C }
@Test
fun deserialiseNewerSetToUnknown() {
val resource = "${this.javaClass.simpleName}.${testName()}"
val sf = testDefaultFactory()
data class C (val e : DeserializeNewerSetToUnknown)
// Uncomment to re-generate test files
// File(URI("$localPath/$resource")).writeBytes(
// SerializationOutput(sf).serialize(C(DeserializeNewerSetToUnknown.D)).bytes)
Assertions.assertThatThrownBy {
DeserializationInput(sf).deserialize(SerializedBytes<C>(
File(EvolvabilityTests::class.java.getResource(resource).toURI()).readBytes()))
}.isInstanceOf(NotSerializableException::class.java)
}
}

View File

@ -17,7 +17,7 @@ class OverridePKSerializerTest {
throw SerializerTestException("Custom write call")
}
override fun readObject(obj: Any, schema: Schema, input: DeserializationInput): PublicKey {
override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): PublicKey {
throw SerializerTestException("Custom read call")
}