mirror of
https://github.com/corda/corda.git
synced 2025-06-16 22:28:15 +00:00
Corda-847 - Remove Kryo for RPC
It's no longer used as we've switched over to AMQP for RPC calls so remove it from everywhere and only use it for checkpointing * Wire up demo bench post Kryo removal * Test Fixes * rebase and fix tests * Test Fix * wip * revert changes to api now we don't need to add annotations
This commit is contained in:
Kat Baker
@ -114,7 +114,8 @@ open class SerializationFactoryImpl(
|
||||
val lookupKey = magic to target
|
||||
return schemes.computeIfAbsent(lookupKey) {
|
||||
registeredSchemes.filter { it.canDeserializeVersion(magic, target) }.forEach { return@computeIfAbsent it } // XXX: Not single?
|
||||
logger.warn("Cannot find serialization scheme for: $lookupKey, registeredSchemes are: $registeredSchemes")
|
||||
logger.warn("Cannot find serialization scheme for: [$lookupKey, " +
|
||||
"${if (magic == amqpMagic) "AMQP" else if (magic == kryoMagic) "Kryo" else "UNKNOWN MAGIC"}] registeredSchemes are: $registeredSchemes")
|
||||
throw UnsupportedOperationException("Serialization scheme $lookupKey not supported.")
|
||||
} to magic
|
||||
}
|
||||
|
||||
@ -285,7 +285,11 @@ open class SerializerFactory(
|
||||
}
|
||||
|
||||
private fun makeClassSerializer(clazz: Class<*>, type: Type, declaredType: Type): AMQPSerializer<Any> = serializersByType.computeIfAbsent(type) {
|
||||
if (isPrimitive(clazz)) {
|
||||
if (clazz.isSynthetic) {
|
||||
// Explicitly ban synthetic classes, we have no way of recreating them when deserializing. This also
|
||||
// captures Lambda expressions and other anonymous functions
|
||||
throw NotSerializableException(type.typeName)
|
||||
} else if (isPrimitive(clazz)) {
|
||||
AMQPPrimitiveSerializer(clazz)
|
||||
} else {
|
||||
findCustomSerializer(clazz, declaredType) ?: run {
|
||||
|
||||
@ -7,7 +7,6 @@ import com.esotericsoftware.kryo.io.Output
|
||||
import com.esotericsoftware.kryo.serializers.CompatibleFieldSerializer
|
||||
import com.esotericsoftware.kryo.serializers.FieldSerializer
|
||||
import com.esotericsoftware.kryo.util.MapReferenceResolver
|
||||
import net.corda.core.concurrent.CordaFuture
|
||||
import net.corda.core.contracts.PrivacySalt
|
||||
import net.corda.core.crypto.Crypto
|
||||
import net.corda.core.crypto.SecureHash
|
||||
@ -18,8 +17,6 @@ import net.corda.core.serialization.SerializationContext.UseCase.Checkpoint
|
||||
import net.corda.core.serialization.SerializationContext.UseCase.Storage
|
||||
import net.corda.core.serialization.SerializeAsTokenContext
|
||||
import net.corda.core.serialization.SerializedBytes
|
||||
import net.corda.core.toFuture
|
||||
import net.corda.core.toObservable
|
||||
import net.corda.core.transactions.*
|
||||
import net.corda.core.utilities.OpaqueBytes
|
||||
import net.corda.nodeapi.internal.crypto.X509CertificateFactory
|
||||
@ -27,7 +24,6 @@ import net.corda.nodeapi.internal.serialization.CordaClassResolver
|
||||
import net.corda.nodeapi.internal.serialization.serializationContextKey
|
||||
import org.slf4j.Logger
|
||||
import org.slf4j.LoggerFactory
|
||||
import rx.Observable
|
||||
import java.io.InputStream
|
||||
import java.lang.reflect.InvocationTargetException
|
||||
import java.security.PrivateKey
|
||||
@ -46,39 +42,16 @@ import kotlin.reflect.jvm.isAccessible
|
||||
import kotlin.reflect.jvm.javaType
|
||||
|
||||
/**
|
||||
* Serialization utilities, using the Kryo framework with a custom serialiser for immutable data classes and a dead
|
||||
* simple, totally non-extensible binary (sub)format.
|
||||
*
|
||||
* This is NOT what should be used in any final platform product, rather, the final state should be a precisely
|
||||
* specified and standardised binary format with attention paid to anti-malleability, versioning and performance.
|
||||
* FIX SBE is a potential candidate: it prioritises performance over convenience and was designed for HFT. Google
|
||||
* Protocol Buffers with a minor tightening to make field reordering illegal is another possibility.
|
||||
*
|
||||
* FIX SBE:
|
||||
* https://real-logic.github.io/simple-binary-encoding/
|
||||
* http://mechanical-sympathy.blogspot.co.at/2014/05/simple-binary-encoding.html
|
||||
* Protocol buffers:
|
||||
* https://developers.google.com/protocol-buffers/
|
||||
*
|
||||
* But for now we use Kryo to maximise prototyping speed.
|
||||
*
|
||||
* Note that this code ignores *ALL* concerns beyond convenience, in particular it ignores:
|
||||
*
|
||||
* - Performance
|
||||
* - Security
|
||||
*
|
||||
* This code will happily deserialise literally anything, including malicious streams that would reconstruct classes
|
||||
* in invalid states, thus violating system invariants. It isn't designed to handle malicious streams and therefore,
|
||||
* isn't usable beyond the prototyping stage. But that's fine: we can revisit serialisation technologies later after
|
||||
* a formal evaluation process.
|
||||
*
|
||||
* We now distinguish between internal, storage related Kryo and external, network facing Kryo. We presently use
|
||||
* some non-whitelisted classes as part of internal storage.
|
||||
* TODO: eliminate internal, storage related whitelist issues, such as private keys in blob storage.
|
||||
* Serialization utilities, using the Kryo framework with a custom serializer for immutable data classes and a dead
|
||||
* simple, totally non-extensible binary (sub)format. Used exclusively within Corda for checkpointing flows as
|
||||
* it will happily deserialise literally anything, including malicious streams that would reconstruct classes
|
||||
* in invalid states and thus violating system invariants. In the context of checkpointing a Java stack, this is
|
||||
* absolutely the functionality we desire, for a stable binary wire format and persistence technology, we have
|
||||
* the AMQP implementation.
|
||||
*/
|
||||
|
||||
/**
|
||||
* A serialiser that avoids writing the wrapper class to the byte stream, thus ensuring [SerializedBytes] is a pure
|
||||
* A serializer that avoids writing the wrapper class to the byte stream, thus ensuring [SerializedBytes] is a pure
|
||||
* type safety hack.
|
||||
*/
|
||||
object SerializedBytesSerializer : Serializer<SerializedBytes<Any>>() {
|
||||
@ -391,44 +364,6 @@ open class CordaKryo(classResolver: ClassResolver) : Kryo(classResolver, MapRefe
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Kryo used for the RPC wire protocol.
|
||||
*/
|
||||
// Every type in the wire protocol is listed here explicitly.
|
||||
// This is annoying to write out, but will make it easier to formalise the wire protocol when the time comes,
|
||||
// because we can see everything we're using in one place.
|
||||
class RPCKryo(observableSerializer: Serializer<Observable<*>>, serializationContext: SerializationContext) : CordaKryo(CordaClassResolver(serializationContext)) {
|
||||
init {
|
||||
DefaultKryoCustomizer.customize(this)
|
||||
|
||||
// RPC specific classes
|
||||
register(InputStream::class.java, InputStreamSerializer)
|
||||
register(Observable::class.java, observableSerializer)
|
||||
register(CordaFuture::class,
|
||||
read = { kryo, input -> observableSerializer.read(kryo, input, Observable::class.java).toFuture() },
|
||||
write = { kryo, output, obj -> observableSerializer.write(kryo, output, obj.toObservable()) }
|
||||
)
|
||||
}
|
||||
|
||||
override fun getRegistration(type: Class<*>): Registration {
|
||||
if (Observable::class.java != type && Observable::class.java.isAssignableFrom(type)) {
|
||||
return super.getRegistration(Observable::class.java)
|
||||
}
|
||||
if (InputStream::class.java != type && InputStream::class.java.isAssignableFrom(type)) {
|
||||
return super.getRegistration(InputStream::class.java)
|
||||
}
|
||||
if (CordaFuture::class.java != type && CordaFuture::class.java.isAssignableFrom(type)) {
|
||||
return super.getRegistration(CordaFuture::class.java)
|
||||
}
|
||||
type.requireExternal("RPC not allowed to deserialise internal classes")
|
||||
return super.getRegistration(type)
|
||||
}
|
||||
|
||||
private fun Class<*>.requireExternal(msg: String) {
|
||||
require(!name.startsWith("net.corda.node.") && ".internal" !in name) { "$msg: $name" }
|
||||
}
|
||||
}
|
||||
|
||||
inline fun <T : Any> Kryo.register(
|
||||
type: KClass<T>,
|
||||
crossinline read: (Kryo, Input) -> T,
|
||||
|
||||
@ -4,13 +4,13 @@ import com.google.common.collect.Maps;
|
||||
import net.corda.core.serialization.SerializationContext;
|
||||
import net.corda.core.serialization.SerializationFactory;
|
||||
import net.corda.core.serialization.SerializedBytes;
|
||||
import net.corda.nodeapi.internal.serialization.kryo.CordaClosureBlacklistSerializer;
|
||||
import net.corda.nodeapi.internal.serialization.kryo.KryoSerializationSchemeKt;
|
||||
import net.corda.nodeapi.internal.serialization.amqp.SchemaKt;
|
||||
import net.corda.testing.core.SerializationEnvironmentRule;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.NotSerializableException;
|
||||
import java.io.Serializable;
|
||||
import java.util.EnumSet;
|
||||
import java.util.concurrent.Callable;
|
||||
@ -33,20 +33,17 @@ public final class ForbiddenLambdaSerializationTests {
|
||||
@Test
|
||||
public final void serialization_fails_for_serializable_java_lambdas() {
|
||||
contexts.forEach(ctx -> {
|
||||
SerializationContext context = new SerializationContextImpl(KryoSerializationSchemeKt.getKryoMagic(),
|
||||
SerializationContext context = new SerializationContextImpl(SchemaKt.getAmqpMagic(),
|
||||
this.getClass().getClassLoader(), AllWhitelist.INSTANCE, Maps.newHashMap(), true, ctx, null);
|
||||
String value = "Hey";
|
||||
Callable<String> target = (Callable<String> & Serializable) () -> value;
|
||||
|
||||
Throwable throwable = catchThrowable(() -> serialize(target, context));
|
||||
|
||||
assertThat(throwable).isNotNull();
|
||||
assertThat(throwable).isInstanceOf(IllegalArgumentException.class);
|
||||
if (ctx != SerializationContext.UseCase.RPCServer && ctx != SerializationContext.UseCase.Storage) {
|
||||
assertThat(throwable).hasMessage(CordaClosureBlacklistSerializer.ERROR_MESSAGE);
|
||||
} else {
|
||||
assertThat(throwable).hasMessageContaining("RPC not allowed to deserialise internal classes");
|
||||
}
|
||||
assertThat(throwable)
|
||||
.isNotNull()
|
||||
.isInstanceOf(NotSerializableException.class)
|
||||
.hasMessageContaining(getClass().getName());
|
||||
});
|
||||
}
|
||||
|
||||
@ -54,21 +51,17 @@ public final class ForbiddenLambdaSerializationTests {
|
||||
@SuppressWarnings("unchecked")
|
||||
public final void serialization_fails_for_not_serializable_java_lambdas() {
|
||||
contexts.forEach(ctx -> {
|
||||
SerializationContext context = new SerializationContextImpl(KryoSerializationSchemeKt.getKryoMagic(),
|
||||
SerializationContext context = new SerializationContextImpl(SchemaKt.getAmqpMagic(),
|
||||
this.getClass().getClassLoader(), AllWhitelist.INSTANCE, Maps.newHashMap(), true, ctx, null);
|
||||
String value = "Hey";
|
||||
Callable<String> target = () -> value;
|
||||
|
||||
Throwable throwable = catchThrowable(() -> serialize(target, context));
|
||||
|
||||
assertThat(throwable).isNotNull();
|
||||
assertThat(throwable).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThat(throwable).isInstanceOf(IllegalArgumentException.class);
|
||||
if (ctx != SerializationContext.UseCase.RPCServer && ctx != SerializationContext.UseCase.Storage) {
|
||||
assertThat(throwable).hasMessage(CordaClosureBlacklistSerializer.ERROR_MESSAGE);
|
||||
} else {
|
||||
assertThat(throwable).hasMessageContaining("RPC not allowed to deserialise internal classes");
|
||||
}
|
||||
assertThat(throwable)
|
||||
.isNotNull()
|
||||
.isInstanceOf(NotSerializableException.class)
|
||||
.hasMessageContaining(getClass().getName());
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -9,14 +9,12 @@ import net.corda.core.serialization.SerializationContext
|
||||
import net.corda.core.serialization.deserialize
|
||||
import net.corda.core.serialization.serialize
|
||||
import net.corda.node.serialization.amqp.AMQPServerSerializationScheme
|
||||
import net.corda.node.serialization.kryo.KryoServerSerializationScheme
|
||||
import net.corda.nodeapi.internal.config.SSLConfiguration
|
||||
import net.corda.nodeapi.internal.createDevKeyStores
|
||||
import net.corda.nodeapi.internal.serialization.AllWhitelist
|
||||
import net.corda.nodeapi.internal.serialization.SerializationContextImpl
|
||||
import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
|
||||
import net.corda.nodeapi.internal.serialization.amqp.amqpMagic
|
||||
import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
|
||||
import net.corda.testing.core.ALICE_NAME
|
||||
import net.corda.testing.core.BOB_NAME
|
||||
import net.corda.testing.core.TestIdentity
|
||||
|
||||
@ -5,6 +5,7 @@ import com.esotericsoftware.kryo.KryoException
|
||||
import com.esotericsoftware.kryo.KryoSerializable
|
||||
import com.esotericsoftware.kryo.io.Input
|
||||
import com.esotericsoftware.kryo.io.Output
|
||||
import com.esotericsoftware.kryo.pool.KryoPool
|
||||
import com.google.common.primitives.Ints
|
||||
import com.nhaarman.mockito_kotlin.doReturn
|
||||
import com.nhaarman.mockito_kotlin.whenever
|
||||
@ -15,7 +16,6 @@ import net.corda.core.serialization.*
|
||||
import net.corda.core.utilities.ByteSequence
|
||||
import net.corda.core.utilities.ProgressTracker
|
||||
import net.corda.core.utilities.sequence
|
||||
import net.corda.node.serialization.kryo.KryoServerSerializationScheme
|
||||
import net.corda.node.services.persistence.NodeAttachmentService
|
||||
import net.corda.nodeapi.internal.serialization.*
|
||||
import net.corda.testing.core.ALICE_NAME
|
||||
@ -35,6 +35,17 @@ import java.time.Instant
|
||||
import java.util.*
|
||||
import kotlin.test.*
|
||||
|
||||
class TestScheme : AbstractKryoSerializationScheme() {
|
||||
override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean {
|
||||
return magic == kryoMagic && target != SerializationContext.UseCase.RPCClient
|
||||
}
|
||||
|
||||
override fun rpcClientKryoPool(context: SerializationContext): KryoPool = throw UnsupportedOperationException()
|
||||
|
||||
override fun rpcServerKryoPool(context: SerializationContext): KryoPool = throw UnsupportedOperationException()
|
||||
|
||||
}
|
||||
|
||||
@RunWith(Parameterized::class)
|
||||
class KryoTests(private val compression: CordaSerializationEncoding?) {
|
||||
companion object {
|
||||
@ -49,7 +60,7 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
|
||||
|
||||
@Before
|
||||
fun setup() {
|
||||
factory = SerializationFactoryImpl().apply { registerScheme(KryoServerSerializationScheme()) }
|
||||
factory = SerializationFactoryImpl().apply { registerScheme(TestScheme()) }
|
||||
context = SerializationContextImpl(kryoMagic,
|
||||
javaClass.classLoader,
|
||||
AllWhitelist,
|
||||
@ -270,7 +281,7 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
|
||||
}
|
||||
}
|
||||
Tmp()
|
||||
val factory = SerializationFactoryImpl().apply { registerScheme(KryoServerSerializationScheme()) }
|
||||
val factory = SerializationFactoryImpl().apply { registerScheme(TestScheme()) }
|
||||
val context = SerializationContextImpl(kryoMagic,
|
||||
javaClass.classLoader,
|
||||
AllWhitelist,
|
||||
|
||||
Reference in New Issue
Block a user