diff --git a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt index 77289d8c8e..407cac3cf4 100644 --- a/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt +++ b/core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt @@ -172,15 +172,21 @@ interface SerializationContext { * The default is false. */ val preventDataLoss: Boolean + /** * The use case we are serializing or deserializing for. See [UseCase]. */ val useCase: UseCase + /** * Additional custom serializers that will be made available during (de)serialization. */ val customSerializers: Set> + // EXPERIMENTAL + val integerFingerprints: IntegerFingerprints? + val externalSchema: ExternalSchema? + /** * Helper method to return a new context based on this context with the property added. */ @@ -251,6 +257,10 @@ interface SerializationContext { */ fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist): SerializationContext + // EXPERIMENTAL + fun withIntegerFingerprint(): SerializationContext + fun withExternalSchema(externalSchema: ExternalSchema): SerializationContext + /** * The use case that we are serializing for, since it influences the implementations chosen. */ @@ -378,9 +388,18 @@ interface EncodingWhitelist { */ fun SerializationContext.withWhitelist(classes: List>): SerializationContext { var currentContext = this - classes.forEach { - clazz -> currentContext = currentContext.withWhitelisted(clazz) + classes.forEach { clazz -> + currentContext = currentContext.withWhitelisted(clazz) } return currentContext } + +// EXPERIMENTAL +@KeepForDJVM +class ExternalSchema() + +@KeepForDJVM +class IntegerFingerprints() { + val descriptorMappings: MutableMap = mutableMapOf() +} diff --git a/finance/contracts/src/test/kotlin/net/corda/finance/contracts/asset/CashTests.kt b/finance/contracts/src/test/kotlin/net/corda/finance/contracts/asset/CashTests.kt index bb9356a8c4..353bebc6a8 100644 --- a/finance/contracts/src/test/kotlin/net/corda/finance/contracts/asset/CashTests.kt +++ b/finance/contracts/src/test/kotlin/net/corda/finance/contracts/asset/CashTests.kt @@ -885,6 +885,7 @@ class CashTests { } val wtx = tx.toWireTransaction(ourServices) + fun out(i: Int) = wtx.getOutput(i) as Cash.State assertEquals(8, wtx.outputs.size) diff --git a/finance/workflows/src/test/kotlin/net/corda/finance/flows/CompatibilityTest.kt b/finance/workflows/src/test/kotlin/net/corda/finance/flows/CompatibilityTest.kt index 27a6f4324a..fbaf7594d4 100644 --- a/finance/workflows/src/test/kotlin/net/corda/finance/flows/CompatibilityTest.kt +++ b/finance/workflows/src/test/kotlin/net/corda/finance/flows/CompatibilityTest.kt @@ -1,14 +1,22 @@ package net.corda.finance.flows import net.corda.core.serialization.SerializationDefaults +import net.corda.core.serialization.SerializationFactory import net.corda.core.serialization.SerializedBytes +import net.corda.core.transactions.ComponentGroup import net.corda.core.transactions.SignedTransaction +import net.corda.core.transactions.WireTransaction +import net.corda.core.utilities.OpaqueBytes import net.corda.finance.contracts.asset.Cash import net.corda.serialization.internal.AllWhitelist +import net.corda.serialization.internal.CordaSerializationEncoding import net.corda.serialization.internal.amqp.DeserializationInput import net.corda.serialization.internal.amqp.Schema import net.corda.serialization.internal.amqp.SerializationOutput +import net.corda.serialization.internal.amqp.SerializerFactory import net.corda.serialization.internal.amqp.SerializerFactoryBuilder +import net.corda.serialization.internal.amqp.custom.BigDecimalSerializer +import net.corda.serialization.internal.amqp.custom.CurrencySerializer import net.corda.serialization.internal.amqp.custom.PublicKeySerializer import net.corda.testing.core.SerializationEnvironmentRule import org.junit.Rule @@ -26,8 +34,10 @@ class CompatibilityTest { @JvmField val testSerialization = SerializationEnvironmentRule() - val serializerFactory = SerializerFactoryBuilder.build(AllWhitelist, ClassLoader.getSystemClassLoader()).apply { + val serializerFactory: SerializerFactory = SerializerFactoryBuilder.build(AllWhitelist, ClassLoader.getSystemClassLoader()).apply { register(PublicKeySerializer) + register(BigDecimalSerializer) + register(CurrencySerializer) } @Test(timeout=300_000) @@ -36,6 +46,7 @@ class CompatibilityTest { assertNotNull(inputStream) val inByteArray: ByteArray = inputStream.readBytes() + println("Original size = ${inByteArray.size}") val input = DeserializationInput(serializerFactory) val (transaction, envelope) = input.deserializeAndReturnEnvelope( @@ -48,13 +59,31 @@ class CompatibilityTest { assertEquals(1, commands.size) assertTrue(commands.first().value is Cash.Commands.Issue) + val newWtx = SerializationFactory.defaultFactory.asCurrent { + withCurrentContext(SerializationDefaults.STORAGE_CONTEXT) { + WireTransaction(transaction.tx.componentGroups.map { cg: ComponentGroup -> + ComponentGroup(cg.groupIndex, cg.components.map { bytes -> + val componentInput = DeserializationInput(serializerFactory) + val component = componentInput.deserialize(SerializedBytes(bytes.bytes), SerializationDefaults.STORAGE_CONTEXT) + val componentOutput = SerializationOutput(serializerFactory) + val componentOutputBytes = componentOutput.serialize(component, SerializationDefaults.STORAGE_CONTEXT.withIntegerFingerprint()).bytes + OpaqueBytes(componentOutputBytes) + }) + }) + } + } + val newTransaction = SignedTransaction(newWtx, transaction.sigs) + // Serialize back and check that representation is byte-to-byte identical to what it was originally. val output = SerializationOutput(serializerFactory) - val (serializedBytes, schema) = output.serializeAndReturnSchema(transaction, SerializationDefaults.STORAGE_CONTEXT) + val outByteArray = output.serialize(newTransaction, SerializationDefaults.STORAGE_CONTEXT.withEncoding(CordaSerializationEncoding.SNAPPY) + .withIntegerFingerprint()).bytes + //val (serializedBytes, schema) = output.serializeAndReturnSchema(transaction, SerializationDefaults.STORAGE_CONTEXT) + println("Output size = ${outByteArray.size}") - assertSchemasMatch(envelope.schema, schema) + //assertSchemasMatch(envelope.schema, schema) - assertTrue(inByteArray.contentEquals(serializedBytes.bytes)) + //assertTrue(inByteArray.contentEquals(serializedBytes.bytes)) } private fun assertSchemasMatch(original: Schema, reserialized: Schema) { diff --git a/lib/quasar.jar b/lib/quasar.jar index 6f1e8c2fca..98ecabcf43 100644 Binary files a/lib/quasar.jar and b/lib/quasar.jar differ diff --git a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeEnumWithEvolutionTest.kt b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeEnumWithEvolutionTest.kt index 32fea60fd0..2a6da2f0bd 100644 --- a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeEnumWithEvolutionTest.kt +++ b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeEnumWithEvolutionTest.kt @@ -9,10 +9,10 @@ import net.corda.core.serialization.SerializationContext import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal._contextSerializationEnv import net.corda.core.serialization.serialize -import net.corda.serialization.djvm.EvolvedEnum.ONE -import net.corda.serialization.djvm.EvolvedEnum.TWO -import net.corda.serialization.djvm.EvolvedEnum.THREE import net.corda.serialization.djvm.EvolvedEnum.FOUR +import net.corda.serialization.djvm.EvolvedEnum.ONE +import net.corda.serialization.djvm.EvolvedEnum.THREE +import net.corda.serialization.djvm.EvolvedEnum.TWO import net.corda.serialization.djvm.OriginalEnum.One import net.corda.serialization.djvm.OriginalEnum.Two import net.corda.serialization.djvm.SandboxType.KOTLIN @@ -77,7 +77,7 @@ class DeserializeEnumWithEvolutionTest : TestBase(KOTLIN) { putAll(transforms) } } - return SerializedBytes(envelope.write()) + return SerializedBytes(envelope.write(context)) } @ParameterizedTest diff --git a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeRemoteCustomisedEnumTest.kt b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeRemoteCustomisedEnumTest.kt index 7c2f5ffee7..d1b0da91da 100644 --- a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeRemoteCustomisedEnumTest.kt +++ b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/DeserializeRemoteCustomisedEnumTest.kt @@ -5,6 +5,7 @@ import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal._contextSerializationEnv import net.corda.core.serialization.serialize import net.corda.serialization.djvm.SandboxType.KOTLIN +import net.corda.serialization.internal.AMQP_STORAGE_CONTEXT import net.corda.serialization.internal.amqp.CompositeType import net.corda.serialization.internal.amqp.DeserializationInput import net.corda.serialization.internal.amqp.RestrictedType @@ -52,7 +53,7 @@ class DeserializeRemoteCustomisedEnumTest : TestBase(KOTLIN) { name = toWorking(restrictedType.name) ) } - return SerializedBytes(envelope.write()) + return SerializedBytes(envelope.write(AMQP_STORAGE_CONTEXT)) } @ParameterizedTest @@ -102,7 +103,7 @@ class DeserializeRemoteCustomisedEnumTest : TestBase(KOTLIN) { name = toWorking(restrictedType.name) ) } - return SerializedBytes(envelope.write()) + return SerializedBytes(envelope.write(AMQP_STORAGE_CONTEXT)) } private fun toWorking(oldName: String): String = oldName.replace("Broken", "Working") diff --git a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/SafeDeserialisationTest.kt b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/SafeDeserialisationTest.kt index 1552279f45..6d851aabd7 100644 --- a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/SafeDeserialisationTest.kt +++ b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/SafeDeserialisationTest.kt @@ -30,10 +30,10 @@ class SafeDeserialisationTest : TestBase(KOTLIN) { val envelope = DeserializationInput.getEnvelope(innocentData, context.encodingWhitelist).apply { val innocentType = schema.types[0] as CompositeType (schema.types as MutableList)[0] = innocentType.copy( - name = innocentType.name.replace("Innocent", "VeryEvil") + name = innocentType.name.replace("Innocent", "VeryEvil") ) } - val evilData = SerializedBytes(envelope.write()) + val evilData = SerializedBytes(envelope.write(context)) sandbox { _contextSerializationEnv.set(createSandboxSerializationEnv(classLoader)) diff --git a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/TestHelpers.kt b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/TestHelpers.kt index 0d5a46d179..909a3a1235 100644 --- a/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/TestHelpers.kt +++ b/serialization-djvm/src/test/kotlin/net/corda/serialization/djvm/TestHelpers.kt @@ -1,6 +1,7 @@ @file:JvmName("TestHelpers") package net.corda.serialization.djvm +import net.corda.core.serialization.SerializationContext import net.corda.serialization.internal.SectionId import net.corda.serialization.internal.amqp.Envelope import net.corda.serialization.internal.amqp.alsoAsByteBuffer @@ -10,9 +11,9 @@ import net.corda.serialization.internal.amqp.withList import org.apache.qpid.proton.codec.Data import java.io.ByteArrayOutputStream -fun Envelope.write(): ByteArray { +fun Envelope.write(context: SerializationContext): ByteArray { val data = Data.Factory.create() - data.withDescribed(Envelope.DESCRIPTOR_OBJECT) { + data.withDescribed(Envelope.DESCRIPTOR_OBJECT, context) { withList { putObject(obj) putObject(schema) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt index 6b63a46655..49840417d5 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/SerializationScheme.kt @@ -5,13 +5,23 @@ import net.corda.core.KeepForDJVM import net.corda.core.crypto.SecureHash import net.corda.core.internal.VisibleForTesting import net.corda.core.internal.copyBytes -import net.corda.core.serialization.* +import net.corda.core.serialization.ClassWhitelist +import net.corda.core.serialization.EncodingWhitelist +import net.corda.core.serialization.ExternalSchema +import net.corda.core.serialization.IntegerFingerprints +import net.corda.core.serialization.ObjectWithCompatibleContext +import net.corda.core.serialization.SerializationContext +import net.corda.core.serialization.SerializationCustomSerializer +import net.corda.core.serialization.SerializationEncoding +import net.corda.core.serialization.SerializationFactory +import net.corda.core.serialization.SerializationMagic +import net.corda.core.serialization.SerializedBytes import net.corda.core.serialization.internal.CustomSerializationSchemeUtils.Companion.getSchemeIdIfCustomSerializationMagic import net.corda.core.utilities.ByteSequence import net.corda.serialization.internal.amqp.amqpMagic import org.slf4j.LoggerFactory import java.io.NotSerializableException -import java.util.* +import java.util.Collections import java.util.concurrent.ConcurrentHashMap internal object NullEncodingWhitelist : EncodingWhitelist { @@ -36,7 +46,9 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe override val lenientCarpenterEnabled: Boolean = false, override val carpenterDisabled: Boolean = false, override val preventDataLoss: Boolean = false, - override val customSerializers: Set> = emptySet()) : SerializationContext { + override val customSerializers: Set> = emptySet(), + override val integerFingerprints: IntegerFingerprints? = null, + override val externalSchema: ExternalSchema? = null) : SerializationContext { /** * {@inheritDoc} */ @@ -80,6 +92,8 @@ data class SerializationContextImpl @JvmOverloads constructor(override val prefe override fun withPreferredSerializationVersion(magic: SerializationMagic) = copy(preferredSerializationVersion = magic) override fun withEncoding(encoding: SerializationEncoding?) = copy(encoding = encoding) override fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist) = copy(encodingWhitelist = encodingWhitelist) + override fun withIntegerFingerprint() = copy(integerFingerprints = IntegerFingerprints()) + override fun withExternalSchema(externalSchema: ExternalSchema) = copy(externalSchema = externalSchema) } @KeepForDJVM diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPRemoteTypeModel.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPRemoteTypeModel.kt index ab4edf859f..afcc6c1316 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPRemoteTypeModel.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/AMQPRemoteTypeModel.kt @@ -1,9 +1,14 @@ package net.corda.serialization.internal.amqp import net.corda.serialization.internal.NotSerializableDetailedException -import net.corda.serialization.internal.model.* +import net.corda.serialization.internal.model.DefaultCacheProvider +import net.corda.serialization.internal.model.EnumTransforms +import net.corda.serialization.internal.model.InvalidEnumTransformsException +import net.corda.serialization.internal.model.RemotePropertyInformation +import net.corda.serialization.internal.model.RemoteTypeInformation +import net.corda.serialization.internal.model.TypeDescriptor +import net.corda.serialization.internal.model.TypeIdentifier import java.io.NotSerializableException -import kotlin.collections.LinkedHashMap /** * Interprets AMQP [Schema] information to obtain [RemoteTypeInformation], caching by [TypeDescriptor]. @@ -192,8 +197,9 @@ class AMQPRemoteTypeModel { } } -private val TypeNotation.typeDescriptor: String get() = descriptor.name?.toString() ?: -throw NotSerializableException("Type notation has no type descriptor: $this") +private val TypeNotation.typeDescriptor: String + get() = descriptor.code?.toString() ?: descriptor.name?.toString() + ?: throw NotSerializableException("Type notation has no type descriptor: $this") private val String.typeIdentifier get(): TypeIdentifier = AMQPTypeIdentifierParser.parse(this) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt index b6a1b8294e..0a130f4bcd 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ArraySerializer.kt @@ -77,7 +77,7 @@ open class ArraySerializer(override val type: Type, factory: LocalSerializerFact context: SerializationContext, debugIndent: Int ) { // Write described - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { withList { for (entry in obj as Array<*>) { output.writeObjectOrNull(entry, this, elementType, context, debugIndent) @@ -136,8 +136,8 @@ abstract class PrimArraySerializer(type: Type, factory: LocalSerializerFactory) fun make(type: Type, factory: LocalSerializerFactory) = primTypes[type]!!(factory) } - fun localWriteObject(data: Data, func: () -> Unit) { - data.withDescribed(typeNotation.descriptor) { withList { func() } } + fun localWriteObject(data: Data, context: SerializationContext, func: () -> Unit) { + data.withDescribed(typeNotation.descriptor, context) { withList { func() } } } } @@ -145,7 +145,7 @@ class PrimIntArraySerializer(factory: LocalSerializerFactory) : PrimArraySeriali override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as IntArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } @@ -155,7 +155,7 @@ class PrimCharArraySerializer(factory: LocalSerializerFactory) : PrimArraySerial override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as CharArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } @@ -176,7 +176,7 @@ class PrimBooleanArraySerializer(factory: LocalSerializerFactory) : PrimArraySer override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as BooleanArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } @@ -187,7 +187,7 @@ class PrimDoubleArraySerializer(factory: LocalSerializerFactory) : override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as DoubleArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } @@ -197,7 +197,7 @@ class PrimFloatArraySerializer(factory: LocalSerializerFactory) : PrimArraySerializer(FloatArray::class.java, factory) { override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as FloatArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } @@ -208,7 +208,7 @@ class PrimShortArraySerializer(factory: LocalSerializerFactory) : override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as ShortArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } @@ -219,7 +219,7 @@ class PrimLongArraySerializer(factory: LocalSerializerFactory) : override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - localWriteObject(data) { + localWriteObject(data, context) { (obj as LongArray).forEach { output.writeObjectOrNull(it, data, elementType, context, debugIndent + 1) } } } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CollectionSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CollectionSerializer.kt index 29953e840a..ad48a72749 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CollectionSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CollectionSerializer.kt @@ -10,8 +10,10 @@ import org.apache.qpid.proton.codec.Data import java.io.NotSerializableException import java.lang.reflect.ParameterizedType import java.lang.reflect.Type -import java.util.* -import kotlin.collections.LinkedHashSet +import java.util.Collections +import java.util.NavigableSet +import java.util.SortedSet +import java.util.TreeSet /** * Serialization / deserialization of predefined set of supported [Collection] types covering mostly [List]s and [Set]s. @@ -108,7 +110,7 @@ class CollectionSerializer(private val declaredType: ParameterizedType, factory: context: SerializationContext, debugIndent: Int) = ifThrowsAppend({ declaredType.typeName }) { // Write described - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { withList { for (entry in obj as Collection<*>) { output.writeObjectOrNull(entry, this, outboundType, context, debugIndent) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CorDappCustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CorDappCustomSerializer.kt index a55d334b40..5cffc1b1c4 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CorDappCustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CorDappCustomSerializer.kt @@ -76,7 +76,7 @@ class CorDappCustomSerializer( val proxy = uncheckedCast, SerializationCustomSerializer>(serializer).toProxy(obj) - data.withDescribed(descriptor) { + data.withDescribed(descriptor, context) { data.withList { proxySerializer.propertySerializers.forEach { (_, serializer) -> serializer.writeProperty(proxy, this, output, context, debugIndent) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt index ee28ca00de..3b99e64086 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/CustomSerializer.kt @@ -50,7 +50,7 @@ abstract class CustomSerializer : AMQPSerializer, SerializerFor { override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - data.withDescribed(descriptor) { + data.withDescribed(descriptor, context) { @Suppress("unchecked_cast") writeDescribedObject(obj as T, data, type, output, context) } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EnumSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EnumSerializer.kt index f4e9647486..ede6a00e28 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EnumSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EnumSerializer.kt @@ -47,7 +47,7 @@ class EnumSerializer(declaredType: Type, declaredClass: Class<*>, factory: Local ) { if (obj !is Enum<*>) throw AMQPNotSerializableException(type, "Serializing $obj as enum when it isn't") - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { withList { data.putString(obj.name) data.putInt(obj.ordinal) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Envelope.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Envelope.kt index 70c4ac55bd..7e9ac86068 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Envelope.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Envelope.kt @@ -21,10 +21,12 @@ data class Envelope(val obj: Any?, val schema: Schema, val transformsSchema: Tra // described list should either be two or three elements long private const val ENVELOPE_WITHOUT_TRANSFORMS = 2 private const val ENVELOPE_WITH_TRANSFORMS = 3 + private const val ENVELOPE_WITH_TRANSFORMS_AND_EXTERNAL_SCHEMA = 3 private const val BLOB_IDX = 0 private const val SCHEMA_IDX = 1 private const val TRANSFORMS_SCHEMA_IDX = 2 + private const val EXTERNAL_SCHEMA_IDX = 3 fun get(data: Data): Envelope { val describedType = data.`object` as DescribedType @@ -39,8 +41,19 @@ data class Envelope(val obj: Any?, val schema: Schema, val transformsSchema: Tra val transformSchema: Any? = when (list.size) { ENVELOPE_WITHOUT_TRANSFORMS -> null ENVELOPE_WITH_TRANSFORMS -> list[TRANSFORMS_SCHEMA_IDX] + ENVELOPE_WITH_TRANSFORMS_AND_EXTERNAL_SCHEMA -> list[TRANSFORMS_SCHEMA_IDX] else -> throw AMQPNoTypeNotSerializableException( - "Malformed list, bad length of ${list.size} (should be 2 or 3)") + "Malformed list, bad length of ${list.size} (should be 2, 3 or 4)") + } + + // We need to cope with objects serialised without the external schema header element in the + // envelope + val externalSchema: Any? = when (list.size) { + ENVELOPE_WITHOUT_TRANSFORMS -> null + ENVELOPE_WITH_TRANSFORMS -> null + ENVELOPE_WITH_TRANSFORMS_AND_EXTERNAL_SCHEMA -> list[EXTERNAL_SCHEMA_IDX] + else -> throw AMQPNoTypeNotSerializableException( + "Malformed list, bad length of ${list.size} (should be 2, 3 or 4)") } return newInstance(listOf(list[BLOB_IDX], Schema.get(list[SCHEMA_IDX]!!), diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/MapSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/MapSerializer.kt index 2e00e8d206..90db74e936 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/MapSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/MapSerializer.kt @@ -11,8 +11,14 @@ import org.apache.qpid.proton.codec.Data import java.io.NotSerializableException import java.lang.reflect.ParameterizedType import java.lang.reflect.Type -import java.util.* -import kotlin.collections.LinkedHashMap +import java.util.Collections +import java.util.Dictionary +import java.util.EnumMap +import java.util.HashMap +import java.util.NavigableMap +import java.util.SortedMap +import java.util.TreeMap +import java.util.WeakHashMap private typealias MapCreationFunction = (Map<*, *>) -> Map<*, *> @@ -108,7 +114,7 @@ class MapSerializer(private val declaredType: ParameterizedType, factory: LocalS ) = ifThrowsAppend({ declaredType.typeName }) { obj.javaClass.checkSupportedMapType() // Write described - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { // Write map data.putMap() data.enter() diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ObjectSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ObjectSerializer.kt index 216c1ec5d7..3ead63df50 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ObjectSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/ObjectSerializer.kt @@ -141,7 +141,7 @@ class ComposableObjectWriter( context: SerializationContext, debugIndent: Int ) { - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { withList { propertySerializers.values.forEach { propertySerializer -> propertySerializer.writeProperty(obj, this, output, context, debugIndent + 1) diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt index 36ac18bfe6..883f5c6d72 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/Schema.kt @@ -2,12 +2,17 @@ package net.corda.serialization.internal.amqp import net.corda.core.KeepForDJVM import net.corda.core.internal.uncheckedCast +import net.corda.core.serialization.SerializationContext import net.corda.serialization.internal.CordaSerializationMagic import net.corda.serialization.internal.amqp.AMQPTypeIdentifiers.isPrimitive import net.corda.serialization.internal.model.TypeIdentifier -import net.corda.serialization.internal.model.TypeIdentifier.TopType import net.corda.serialization.internal.model.TypeIdentifier.Companion.forGenericType -import org.apache.qpid.proton.amqp.* +import net.corda.serialization.internal.model.TypeIdentifier.TopType +import org.apache.qpid.proton.amqp.Binary +import org.apache.qpid.proton.amqp.DescribedType +import org.apache.qpid.proton.amqp.Symbol +import org.apache.qpid.proton.amqp.UnsignedInteger +import org.apache.qpid.proton.amqp.UnsignedLong import org.apache.qpid.proton.codec.DescribedTypeConstructor import java.io.NotSerializableException import java.lang.reflect.Type @@ -183,6 +188,8 @@ sealed class TypeNotation : DescribedType { abstract val label: String? abstract val provides: List abstract val descriptor: Descriptor + + abstract fun maybeConvertDescriptorToInteger(context: SerializationContext): TypeNotation } @KeepForDJVM @@ -215,6 +222,7 @@ data class CompositeType( override fun getDescriptor(): Any = DESCRIPTOR override fun getDescribed(): Any = listOf(name, label, provides, descriptor, fields) + override fun maybeConvertDescriptorToInteger(context: SerializationContext): TypeNotation = copy(descriptor = descriptor.maybeConvertToInteger(context)) override fun toString(): String { val sb = StringBuilder(" Unit) { +fun Data.withDescribed(descriptor: Descriptor, context: SerializationContext, block: Data.() -> Unit) { // Write described putDescribed() enter() // Write descriptor - putObject(descriptor.code ?: descriptor.name) + with(descriptor.maybeConvertToInteger(context)) { + putObject(this.code ?: this.name) + } block() exit() // exit described } +// EXPERIMENTAL +fun Descriptor.maybeConvertToInteger(context: SerializationContext): Descriptor { + if (this.code != null) return this + if (!this.name.toString().endsWith("==")) return this + val descriptorMappings = context.integerFingerprints?.descriptorMappings + if (descriptorMappings == null) return this + return context.integerFingerprints!!.descriptorMappings.computeIfAbsent(this) { + Descriptor(null, UnsignedLong(32L + descriptorMappings.size or DESCRIPTOR_TOP_32BITS)) + } as Descriptor +} + /** * Extension helper for writing lists. */ diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializationOutput.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializationOutput.kt index 73b7eacae0..0fa90915b6 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializationOutput.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializationOutput.kt @@ -13,8 +13,7 @@ import java.io.NotSerializableException import java.io.OutputStream import java.lang.reflect.Type import java.lang.reflect.WildcardType -import java.util.* -import kotlin.collections.LinkedHashSet +import java.util.IdentityHashMap @KeepForDJVM data class BytesAndSchemas( @@ -78,10 +77,10 @@ open class SerializationOutput constructor( internal fun _serialize(obj: T, context: SerializationContext): SerializedBytes { val data = Data.Factory.create() - data.withDescribed(Envelope.DESCRIPTOR_OBJECT) { + data.withDescribed(Envelope.DESCRIPTOR_OBJECT, context) { withList { writeObject(obj, this, context) - val schema = Schema(schemaHistory.toList()) + val schema = Schema(schemaHistory.map { it.maybeConvertDescriptorToInteger(context) }.toList()) writeSchema(schema, this) writeTransformSchema(TransformsSchema.build(schema, serializerFactory), this) } diff --git a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SingletonSerializer.kt b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SingletonSerializer.kt index 6c268d5d3f..2eb7630031 100644 --- a/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SingletonSerializer.kt +++ b/serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SingletonSerializer.kt @@ -26,7 +26,7 @@ class SingletonSerializer(override val type: Class<*>, val singleton: Any, facto override fun writeObject(obj: Any, data: Data, type: Type, output: SerializationOutput, context: SerializationContext, debugIndent: Int ) { - data.withDescribed(typeNotation.descriptor) { + data.withDescribed(typeNotation.descriptor, context) { data.putBoolean(false) } } diff --git a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EnvelopeHelpers.kt b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EnvelopeHelpers.kt index 4b46ad2183..7f201083d8 100644 --- a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EnvelopeHelpers.kt +++ b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EnvelopeHelpers.kt @@ -8,7 +8,9 @@ import java.io.ByteArrayOutputStream fun Envelope.write(): ByteArray { val data = Data.Factory.create() - data.withDescribed(DESCRIPTOR_OBJECT) { + data.withDescribed( + DESCRIPTOR_OBJECT, + ) { withList { putObject(obj) putObject(schema)