CORDA-1511: Kryo only used for node checkpoints and so moved out of serialization module and into node (#3228)

This commit is contained in:
Shams Asari
2018-05-24 12:18:07 +01:00
committed by GitHub
parent 76918de656
commit 3136e973a7
37 changed files with 150 additions and 153 deletions

View File

@ -1,17 +1,17 @@
package net.corda.serialization.internal;
import com.google.common.collect.Maps;
import net.corda.core.serialization.SerializationContext;
import net.corda.core.serialization.SerializationFactory;
import net.corda.core.serialization.SerializedBytes;
import net.corda.node.serialization.kryo.CordaClosureSerializer;
import net.corda.node.serialization.kryo.KryoSerializationSchemeKt;
import net.corda.testing.core.SerializationEnvironmentRule;
import net.corda.serialization.internal.kryo.CordaClosureSerializer;
import net.corda.serialization.internal.kryo.KryoSerializationSchemeKt;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import java.io.Serializable;
import java.util.Collections;
import java.util.concurrent.Callable;
import static org.assertj.core.api.Assertions.assertThat;
@ -26,7 +26,15 @@ public final class LambdaCheckpointSerializationTest {
@Before
public void setup() {
factory = testSerialization.getSerializationFactory();
context = new SerializationContextImpl(KryoSerializationSchemeKt.getKryoMagic(), this.getClass().getClassLoader(), AllWhitelist.INSTANCE, Maps.newHashMap(), true, SerializationContext.UseCase.Checkpoint, null);
context = new SerializationContextImpl(
KryoSerializationSchemeKt.getKryoMagic(),
getClass().getClassLoader(),
AllWhitelist.INSTANCE,
Collections.emptyMap(),
true,
SerializationContext.UseCase.Checkpoint,
null
);
}
@Test

View File

@ -14,9 +14,9 @@ import net.corda.core.node.services.AttachmentStorage
import net.corda.core.serialization.ClassWhitelist
import net.corda.core.serialization.CordaSerializable
import net.corda.core.serialization.SerializationContext
import net.corda.serialization.internal.kryo.CordaClassResolver
import net.corda.serialization.internal.kryo.CordaKryo
import net.corda.serialization.internal.kryo.kryoMagic
import net.corda.node.serialization.kryo.CordaClassResolver
import net.corda.node.serialization.kryo.CordaKryo
import net.corda.node.serialization.kryo.kryoMagic
import net.corda.testing.internal.rigorousMock
import net.corda.testing.services.MockAttachmentStorage
import org.junit.Rule

View File

@ -3,11 +3,11 @@ package net.corda.serialization.internal
import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.util.DefaultClassResolver
import net.corda.core.serialization.*
import net.corda.node.serialization.kryo.kryoMagic
import net.corda.node.services.statemachine.DataSessionMessage
import net.corda.serialization.internal.amqp.DeserializationInput
import net.corda.serialization.internal.amqp.Envelope
import net.corda.serialization.internal.amqp.SerializerFactory
import net.corda.serialization.internal.kryo.kryoMagic
import net.corda.testing.internal.amqpSpecific
import net.corda.testing.internal.kryoSpecific
import net.corda.testing.core.SerializationEnvironmentRule

View File

@ -6,8 +6,8 @@ import net.corda.core.identity.CordaX500Name
import net.corda.core.serialization.CordaSerializable
import net.corda.core.serialization.deserialize
import net.corda.core.serialization.serialize
import net.corda.node.serialization.kryo.kryoMagic
import net.corda.node.services.statemachine.DataSessionMessage
import net.corda.serialization.internal.kryo.kryoMagic
import net.corda.testing.core.SerializationEnvironmentRule
import net.corda.testing.internal.amqpSpecific
import net.corda.testing.internal.kryoSpecific

View File

@ -5,10 +5,10 @@ import com.esotericsoftware.kryo.KryoException
import com.esotericsoftware.kryo.io.Output
import net.corda.core.serialization.*
import net.corda.core.utilities.OpaqueBytes
import net.corda.serialization.internal.kryo.CordaClassResolver
import net.corda.serialization.internal.kryo.CordaKryo
import net.corda.serialization.internal.kryo.DefaultKryoCustomizer
import net.corda.serialization.internal.kryo.kryoMagic
import net.corda.node.serialization.kryo.CordaClassResolver
import net.corda.node.serialization.kryo.CordaKryo
import net.corda.node.serialization.kryo.DefaultKryoCustomizer
import net.corda.node.serialization.kryo.kryoMagic
import net.corda.testing.internal.rigorousMock
import net.corda.testing.core.SerializationEnvironmentRule
import org.assertj.core.api.Assertions.assertThat

View File

@ -4,8 +4,8 @@ import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.util.DefaultClassResolver
import net.corda.core.serialization.deserialize
import net.corda.core.serialization.serialize
import net.corda.node.serialization.kryo.kryoMagic
import net.corda.node.services.statemachine.DataSessionMessage
import net.corda.serialization.internal.kryo.kryoMagic
import net.corda.testing.core.SerializationEnvironmentRule
import net.corda.testing.internal.kryoSpecific
import org.junit.Assert.assertArrayEquals

View File

@ -3,8 +3,8 @@ package net.corda.serialization.internal.amqp
import net.corda.core.serialization.*
import net.corda.core.utilities.ByteSequence
import net.corda.serialization.internal.*
import net.corda.serialization.internal.kryo.BuiltInExceptionsWhitelist
import net.corda.serialization.internal.kryo.GlobalTransientClassWhiteList
import net.corda.serialization.internal.BuiltInExceptionsWhitelist
import net.corda.serialization.internal.GlobalTransientClassWhiteList
import org.junit.Test
import java.util.concurrent.ConcurrentHashMap
import kotlin.test.assertEquals

View File

@ -1,98 +0,0 @@
package net.corda.serialization.internal.kryo
import net.corda.core.internal.declaredField
import net.corda.serialization.internal.ByteBufferOutputStream
import org.assertj.core.api.Assertions.catchThrowable
import org.junit.Assert.assertArrayEquals
import org.junit.Test
import java.io.*
import java.nio.BufferOverflowException
import java.util.*
import java.util.zip.DeflaterOutputStream
import java.util.zip.InflaterInputStream
import kotlin.test.assertEquals
import kotlin.test.assertSame
class KryoStreamsTest {
class NegOutputStream(private val stream: OutputStream) : OutputStream() {
override fun write(b: Int) = stream.write(-b)
}
class NegInputStream(private val stream: InputStream) : InputStream() {
override fun read() = stream.read().let {
if (it != -1) 0xff and -it else -1
}
}
@Test
fun `substitute output works`() {
assertArrayEquals(byteArrayOf(100, -101), kryoOutput {
write(100)
substitute(::NegOutputStream)
write(101)
})
}
@Test
fun `substitute input works`() {
kryoInput(byteArrayOf(100, 101).inputStream()) {
assertEquals(100, read())
substitute(::NegInputStream)
assertEquals(-101, read().toByte())
assertEquals(-1, read())
}
}
@Test
fun `zip round-trip`() {
val data = ByteArray(12345).also { Random(0).nextBytes(it) }
val encoded = kryoOutput {
write(data)
substitute(::DeflaterOutputStream)
write(data)
substitute(::DeflaterOutputStream) // Potentially useful if a different codec.
write(data)
}
kryoInput(encoded.inputStream()) {
assertArrayEquals(data, readBytes(data.size))
substitute(::InflaterInputStream)
assertArrayEquals(data, readBytes(data.size))
substitute(::InflaterInputStream)
assertArrayEquals(data, readBytes(data.size))
assertEquals(-1, read())
}
}
@Test
fun `ByteBufferOutputStream works`() {
val stream = ByteBufferOutputStream(3)
stream.write("abc".toByteArray())
val getBuf = stream.declaredField<ByteArray>(ByteArrayOutputStream::class, "buf")::value
assertEquals(3, getBuf().size)
repeat(2) {
assertSame<Any>(BufferOverflowException::class.java, catchThrowable {
stream.alsoAsByteBuffer(9) {
it.put("0123456789".toByteArray())
}
}.javaClass)
assertEquals(3 + 9, getBuf().size)
}
// This time make too much space:
stream.alsoAsByteBuffer(11) {
it.put("0123456789".toByteArray())
}
stream.write("def".toByteArray())
assertArrayEquals("abc0123456789def".toByteArray(), stream.toByteArray())
}
@Test
fun `ByteBufferOutputStream discards data after final position`() {
val stream = ByteBufferOutputStream(0)
stream.alsoAsByteBuffer(10) {
it.put("0123456789".toByteArray())
it.position(5)
}
stream.write("def".toByteArray())
assertArrayEquals("01234def".toByteArray(), stream.toByteArray())
}
}

View File

@ -1,352 +0,0 @@
package net.corda.serialization.internal.kryo
import com.esotericsoftware.kryo.Kryo
import com.esotericsoftware.kryo.KryoException
import com.esotericsoftware.kryo.KryoSerializable
import com.esotericsoftware.kryo.io.Input
import com.esotericsoftware.kryo.io.Output
import com.esotericsoftware.kryo.pool.KryoPool
import com.google.common.primitives.Ints
import com.nhaarman.mockito_kotlin.doReturn
import com.nhaarman.mockito_kotlin.whenever
import net.corda.core.contracts.PrivacySalt
import net.corda.core.crypto.*
import net.corda.core.internal.FetchDataFlow
import net.corda.core.serialization.*
import net.corda.core.utilities.ByteSequence
import net.corda.core.utilities.ProgressTracker
import net.corda.core.utilities.sequence
import net.corda.node.services.persistence.NodeAttachmentService
import net.corda.serialization.internal.*
import net.corda.testing.core.ALICE_NAME
import net.corda.testing.core.TestIdentity
import net.corda.testing.internal.rigorousMock
import org.assertj.core.api.Assertions.*
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertEquals
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.junit.runners.Parameterized
import org.junit.runners.Parameterized.Parameters
import org.slf4j.LoggerFactory
import java.io.InputStream
import java.time.Instant
import java.util.*
import kotlin.test.*
class TestScheme : AbstractKryoSerializationScheme() {
override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean {
return magic == kryoMagic && target != SerializationContext.UseCase.RPCClient
}
override fun rpcClientKryoPool(context: SerializationContext): KryoPool = throw UnsupportedOperationException()
override fun rpcServerKryoPool(context: SerializationContext): KryoPool = throw UnsupportedOperationException()
}
@RunWith(Parameterized::class)
class KryoTests(private val compression: CordaSerializationEncoding?) {
companion object {
private val ALICE_PUBKEY = TestIdentity(ALICE_NAME, 70).publicKey
@Parameters(name = "{0}")
@JvmStatic
fun compression() = arrayOf<CordaSerializationEncoding?>(null) + CordaSerializationEncoding.values()
}
private lateinit var factory: SerializationFactory
private lateinit var context: SerializationContext
@Before
fun setup() {
factory = SerializationFactoryImpl().apply { registerScheme(TestScheme()) }
context = SerializationContextImpl(kryoMagic,
javaClass.classLoader,
AllWhitelist,
emptyMap(),
true,
SerializationContext.UseCase.Storage,
compression,
rigorousMock<EncodingWhitelist>().also {
if (compression != null) doReturn(true).whenever(it).acceptEncoding(compression)
})
}
@Test
fun `simple data class`() {
val birthday = Instant.parse("1984-04-17T00:30:00.00Z")
val mike = Person("mike", birthday)
val bits = mike.serialize(factory, context)
assertThat(bits.deserialize(factory, context)).isEqualTo(Person("mike", birthday))
}
@Test
fun `null values`() {
val bob = Person("bob", null)
val bits = bob.serialize(factory, context)
assertThat(bits.deserialize(factory, context)).isEqualTo(Person("bob", null))
}
@Test
fun `serialised form is stable when the same object instance is added to the deserialised object graph`() {
val noReferencesContext = context.withoutReferences()
val obj : ByteSequence = Ints.toByteArray(0x01234567).sequence()
val originalList : ArrayList<ByteSequence> = arrayListOf(obj)
val deserialisedList = originalList.serialize(factory, noReferencesContext).deserialize(factory, noReferencesContext)
originalList += obj
deserialisedList += obj
assertThat(deserialisedList.serialize(factory, noReferencesContext)).isEqualTo(originalList.serialize(factory, noReferencesContext))
}
@Test
fun `serialised form is stable when the same object instance occurs more than once, and using java serialisation`() {
val noReferencesContext = context.withoutReferences()
val instant = Instant.ofEpochMilli(123)
val instantCopy = Instant.ofEpochMilli(123)
assertThat(instant).isNotSameAs(instantCopy)
val listWithCopies = arrayListOf(instant, instantCopy)
val listWithSameInstances = arrayListOf(instant, instant)
assertThat(listWithSameInstances.serialize(factory, noReferencesContext)).isEqualTo(listWithCopies.serialize(factory, noReferencesContext))
}
@Test
fun `cyclic object graph`() {
val cyclic = Cyclic(3)
val bits = cyclic.serialize(factory, context)
assertThat(bits.deserialize(factory, context)).isEqualTo(cyclic)
}
@Test
fun `deserialised key pair functions the same as serialised one`() {
val keyPair = generateKeyPair()
val bitsToSign: ByteArray = Ints.toByteArray(0x01234567)
val wrongBits: ByteArray = Ints.toByteArray(0x76543210)
val signature = keyPair.sign(bitsToSign)
signature.verify(bitsToSign)
assertThatThrownBy { signature.verify(wrongBits) }
val deserialisedKeyPair = keyPair.serialize(factory, context).deserialize(factory, context)
val deserialisedSignature = deserialisedKeyPair.sign(bitsToSign)
deserialisedSignature.verify(bitsToSign)
assertThatThrownBy { deserialisedSignature.verify(wrongBits) }
}
@Test
fun `write and read Kotlin object singleton`() {
val serialised = TestSingleton.serialize(factory, context)
val deserialised = serialised.deserialize(factory, context)
assertThat(deserialised).isSameAs(TestSingleton)
}
@Test
fun `check Kotlin EmptyList can be serialised`() {
val deserialisedList: List<Int> = emptyList<Int>().serialize(factory, context).deserialize(factory, context)
assertEquals(0, deserialisedList.size)
assertEquals<Any>(Collections.emptyList<Int>().javaClass, deserialisedList.javaClass)
}
@Test
fun `check Kotlin EmptySet can be serialised`() {
val deserialisedSet: Set<Int> = emptySet<Int>().serialize(factory, context).deserialize(factory, context)
assertEquals(0, deserialisedSet.size)
assertEquals<Any>(Collections.emptySet<Int>().javaClass, deserialisedSet.javaClass)
}
@Test
fun `check Kotlin EmptyMap can be serialised`() {
val deserialisedMap: Map<Int, Int> = emptyMap<Int, Int>().serialize(factory, context).deserialize(factory, context)
assertEquals(0, deserialisedMap.size)
assertEquals<Any>(Collections.emptyMap<Int, Int>().javaClass, deserialisedMap.javaClass)
}
@Test
fun `InputStream serialisation`() {
val rubbish = ByteArray(12345, { (it * it * 0.12345).toByte() })
val readRubbishStream: InputStream = rubbish.inputStream().serialize(factory, context).deserialize(factory, context)
for (i in 0..12344) {
assertEquals(rubbish[i], readRubbishStream.read().toByte())
}
assertEquals(-1, readRubbishStream.read())
}
@Test
fun `InputStream serialisation does not write trailing garbage`() {
val byteArrays = listOf("123", "456").map { it.toByteArray() }
val streams = byteArrays.map { it.inputStream() }.serialize(factory, context).deserialize(factory, context).iterator()
byteArrays.forEach { assertArrayEquals(it, streams.next().readBytes()) }
assertFalse(streams.hasNext())
}
@Test
fun `serialize - deserialize SignableData`() {
val testString = "Hello World"
val testBytes = testString.toByteArray()
val meta = SignableData(testBytes.sha256(), SignatureMetadata(1, Crypto.findSignatureScheme(ALICE_PUBKEY).schemeNumberID))
val serializedMetaData = meta.serialize(factory, context).bytes
val meta2 = serializedMetaData.deserialize<SignableData>(factory, context)
assertEquals(meta2, meta)
}
@Test
fun `serialize - deserialize Logger`() {
val storageContext: SerializationContext = context // TODO: make it storage context
val logger = LoggerFactory.getLogger("aName")
val logger2 = logger.serialize(factory, storageContext).deserialize(factory, storageContext)
assertEquals(logger.name, logger2.name)
assertTrue(logger === logger2)
}
@Test
fun `HashCheckingStream (de)serialize`() {
val rubbish = ByteArray(12345, { (it * it * 0.12345).toByte() })
val readRubbishStream: InputStream = NodeAttachmentService.HashCheckingStream(
SecureHash.sha256(rubbish),
rubbish.size,
rubbish.inputStream()
).serialize(factory, context).deserialize(factory, context)
for (i in 0..12344) {
assertEquals(rubbish[i], readRubbishStream.read().toByte())
}
assertEquals(-1, readRubbishStream.read())
}
@CordaSerializable
private data class Person(val name: String, val birthday: Instant?)
@Suppress("unused")
@CordaSerializable
private class Cyclic(val value: Int) {
val thisInstance = this
override fun equals(other: Any?): Boolean = (this === other) || (other is Cyclic && this.value == other.value)
override fun hashCode(): Int = value.hashCode()
override fun toString(): String = "Cyclic($value)"
}
@Test
fun `serialize - deserialize PrivacySalt`() {
val expected = PrivacySalt(byteArrayOf(
1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
31, 32
))
val serializedBytes = expected.serialize(factory, context)
val actual = serializedBytes.deserialize(factory, context)
assertEquals(expected, actual)
}
@CordaSerializable
private object TestSingleton
object SimpleSteps {
object ONE : ProgressTracker.Step("one")
object TWO : ProgressTracker.Step("two")
object THREE : ProgressTracker.Step("three")
object FOUR : ProgressTracker.Step("four")
fun tracker() = ProgressTracker(ONE, TWO, THREE, FOUR)
}
object ChildSteps {
object AYY : ProgressTracker.Step("ayy")
object BEE : ProgressTracker.Step("bee")
object SEA : ProgressTracker.Step("sea")
fun tracker() = ProgressTracker(AYY, BEE, SEA)
}
@Test
fun rxSubscriptionsAreNotSerialized() {
val pt: ProgressTracker = SimpleSteps.tracker()
val pt2: ProgressTracker = ChildSteps.tracker()
class Unserializable : KryoSerializable {
override fun write(kryo: Kryo?, output: Output?) = throw AssertionError("not called")
override fun read(kryo: Kryo?, input: Input?) = throw AssertionError("not called")
fun foo() {
println("bar")
}
}
pt.setChildProgressTracker(SimpleSteps.TWO, pt2)
class Tmp {
val unserializable = Unserializable()
init {
pt2.changes.subscribe { unserializable.foo() }
}
}
Tmp()
val factory = SerializationFactoryImpl().apply { registerScheme(TestScheme()) }
val context = SerializationContextImpl(kryoMagic,
javaClass.classLoader,
AllWhitelist,
emptyMap(),
true,
SerializationContext.UseCase.P2P,
null)
pt.serialize(factory, context)
}
@Test
fun `serialize - deserialize Exception with suppressed`() {
val exception = IllegalArgumentException("fooBar")
val toBeSuppressedOnSenderSide = IllegalStateException("bazz1")
exception.addSuppressed(toBeSuppressedOnSenderSide)
val exception2 = exception.serialize(factory, context).deserialize(factory, context)
assertEquals(exception.message, exception2.message)
assertEquals(1, exception2.suppressed.size)
assertNotNull({ exception2.suppressed.find { it.message == toBeSuppressedOnSenderSide.message } })
val toBeSuppressedOnReceiverSide = IllegalStateException("bazz2")
exception2.addSuppressed(toBeSuppressedOnReceiverSide)
assertTrue { exception2.suppressed.contains(toBeSuppressedOnReceiverSide) }
assertEquals(2, exception2.suppressed.size)
}
@Test
fun `serialize - deserialize Exception no suppressed`() {
val exception = IllegalArgumentException("fooBar")
val exception2 = exception.serialize(factory, context).deserialize(factory, context)
assertEquals(exception.message, exception2.message)
assertEquals(0, exception2.suppressed.size)
val toBeSuppressedOnReceiverSide = IllegalStateException("bazz2")
exception2.addSuppressed(toBeSuppressedOnReceiverSide)
assertEquals(1, exception2.suppressed.size)
assertTrue { exception2.suppressed.contains(toBeSuppressedOnReceiverSide) }
}
@Test
fun `serialize - deserialize HashNotFound`() {
val randomHash = SecureHash.randomSHA256()
val exception = FetchDataFlow.HashNotFound(randomHash)
val exception2 = exception.serialize(factory, context).deserialize(factory, context)
assertEquals(randomHash, exception2.requested)
}
@Test
fun `compression has the desired effect`() {
compression ?: return
val data = ByteArray(12345).also { Random(0).nextBytes(it) }.let { it + it }
val compressed = data.serialize(factory, context)
assertEquals(.5, compressed.size.toDouble() / data.size, .03)
assertArrayEquals(data, compressed.deserialize(factory, context))
}
@Test
fun `a particular encoding can be banned for deserialization`() {
compression ?: return
doReturn(false).whenever(context.encodingWhitelist).acceptEncoding(compression)
val compressed = "whatever".serialize(factory, context)
catchThrowable { compressed.deserialize(factory, context) }.run {
assertSame<Any>(KryoException::class.java, javaClass)
assertEquals(encodingNotPermittedFormat.format(compression), message)
}
}
}