CORDA-847 - AMQP RPC

* Client and server support for amqp
 * Observable (and supporting) serialisers
   Unit Tests
 * Fixing tests
 * Test fixes
 * CORDA-847 - Update api doc with additon of @CordaSerializable annotation
 * TestFixes
 * review comments
 * TestFixes
 * Test Fix
 * Test Fix
 * Test Fix
 * Test Fix
 * Test Fix
 * Test Fix
 * TestFix
 * Test Fix
 * Review Comments

node-api/src/main/kotlin/net/corda/nodeapi/exceptions/InternalNodeException.kt

@@ -28,7 +28,9 @@ class InternalNodeException(message: String) : CordaRuntimeException(message) {
             (wrapped as? CordaRuntimeException)?.setCause(null)
             return when {
                 whitelisted.any { it.isInstance(wrapped) } -> wrapped
-                else -> InternalNodeException(DEFAULT_MESSAGE)
+                else -> InternalNodeException(DEFAULT_MESSAGE).apply {
+                    stackTrace = emptyArray()
+                }
             }
         }
     }

node-api/src/main/kotlin/net/corda/nodeapi/internal/network/NetworkBootstrapper.kt

@@ -23,10 +23,11 @@ import net.corda.nodeapi.internal.ContractsJarFile
 import net.corda.nodeapi.internal.DEV_ROOT_CA
 import net.corda.nodeapi.internal.SignedNodeInfo
 import net.corda.nodeapi.internal.network.NodeInfoFilesCopier.Companion.NODE_INFO_FILE_NAME_PREFIX
-import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
 import net.corda.nodeapi.internal.serialization.CordaSerializationMagic
+import net.corda.nodeapi.internal.serialization.AMQP_P2P_CONTEXT
 import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
-import net.corda.nodeapi.internal.serialization.amqp.AMQPServerSerializationScheme
+import net.corda.nodeapi.internal.serialization.amqp.AbstractAMQPSerializationScheme
+import net.corda.nodeapi.internal.serialization.amqp.amqpMagic
 import net.corda.nodeapi.internal.serialization.kryo.AbstractKryoSerializationScheme
 import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
 import java.nio.file.Path
@@ -278,7 +279,7 @@ class NetworkBootstrapper {
         _contextSerializationEnv.set(SerializationEnvironmentImpl(
                 SerializationFactoryImpl().apply {
                     registerScheme(KryoParametersSerializationScheme)
-                    registerScheme(AMQPServerSerializationScheme())
+                    registerScheme(AMQPParametersSerializationScheme)
                 },
                 AMQP_P2P_CONTEXT)
         )
@@ -292,4 +293,13 @@ class NetworkBootstrapper {
         override fun rpcClientKryoPool(context: SerializationContext) = throw UnsupportedOperationException()
         override fun rpcServerKryoPool(context: SerializationContext) = throw UnsupportedOperationException()
     }
+
+    private object AMQPParametersSerializationScheme : AbstractAMQPSerializationScheme(emptyList()) {
+        override fun rpcClientSerializerFactory(context: SerializationContext) = throw UnsupportedOperationException()
+        override fun rpcServerSerializerFactory(context: SerializationContext) = throw UnsupportedOperationException()
+
+        override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean {
+            return magic == amqpMagic && target == SerializationContext.UseCase.P2P
+        }
+    }
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/protonwrapper/engine/ConnectionStateMachine.kt

@@ -352,7 +352,7 @@ internal class ConnectionStateMachine(serverMode: Boolean,
                 val connection = event.connection
                 val channel = connection?.context as? Channel
                 if (channel != null) {
-                    val appProperties = HashMap(amqpMessage.applicationProperties.value as Map<String, Any?>)
+                    val appProperties = HashMap(amqpMessage.applicationProperties.value)
                     appProperties["_AMQ_VALIDATED_USER"] = remoteLegalName
                     val localAddress = channel.localAddress() as InetSocketAddress
                     val remoteAddress = channel.remoteAddress() as InetSocketAddress

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/ClientContexts.kt

@@ -13,13 +13,7 @@ import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
  * servers from trying to instantiate any of them.
  */
 
-val KRYO_RPC_CLIENT_CONTEXT = SerializationContextImpl(kryoMagic,
-        SerializationDefaults.javaClass.classLoader,
-        GlobalTransientClassWhiteList(BuiltInExceptionsWhitelist()),
-        emptyMap(),
-        true,
-        SerializationContext.UseCase.RPCClient,
-        null)
+
 val AMQP_RPC_CLIENT_CONTEXT = SerializationContextImpl(amqpMagic,
         SerializationDefaults.javaClass.classLoader,
         GlobalTransientClassWhiteList(BuiltInExceptionsWhitelist()),

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SerializationScheme.kt

@@ -115,7 +115,7 @@ open class SerializationFactoryImpl(
         return schemes.computeIfAbsent(lookupKey) {
             registeredSchemes.filter { it.canDeserializeVersion(magic, target) }.forEach { return@computeIfAbsent it } // XXX: Not single?
             logger.warn("Cannot find serialization scheme for: $lookupKey, registeredSchemes are: $registeredSchemes")
-            throw UnsupportedOperationException("Serialization scheme not supported.")
+            throw UnsupportedOperationException("Serialization scheme $lookupKey not supported.")
         } to magic
     }
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/ServerContexts.kt

@@ -17,13 +17,6 @@ import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
  * MUST be kept separate!
  */
 
-val KRYO_RPC_SERVER_CONTEXT = SerializationContextImpl(kryoMagic,
-        SerializationDefaults.javaClass.classLoader,
-        GlobalTransientClassWhiteList(BuiltInExceptionsWhitelist()),
-        emptyMap(),
-        true,
-        SerializationContext.UseCase.RPCServer,
-        null)
 
 val AMQP_STORAGE_CONTEXT = SerializationContextImpl(amqpMagic,
         SerializationDefaults.javaClass.classLoader,

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt

@@ -5,6 +5,7 @@ package net.corda.nodeapi.internal.serialization.amqp
 import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner
 import net.corda.core.cordapp.Cordapp
 import net.corda.core.internal.objectOrNewInstance
+import net.corda.core.internal.uncheckedCast
 import net.corda.core.serialization.*
 import net.corda.core.utilities.ByteSequence
 import net.corda.nodeapi.internal.serialization.CordaSerializationMagic
@@ -12,7 +13,6 @@ import net.corda.nodeapi.internal.serialization.DefaultWhitelist
 import net.corda.nodeapi.internal.serialization.MutableClassWhitelist
 import net.corda.nodeapi.internal.serialization.SerializationScheme
 import java.lang.reflect.Modifier
-import java.security.PublicKey
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 
@@ -118,6 +118,12 @@ abstract class AbstractAMQPSerializationScheme(
                 factory.registerExternal(CorDappCustomSerializer(customSerializer, factory))
             }
         }
+
+        context.properties[ContextPropertyKeys.SERIALIZERS]?.apply {
+            uncheckedCast<Any, List<CustomSerializer<out Any>>>(this).forEach {
+                factory.register(it)
+            }
+        }
     }
 
     /*
@@ -131,7 +137,9 @@ abstract class AbstractAMQPSerializationScheme(
 
     protected abstract fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory
     protected abstract fun rpcServerSerializerFactory(context: SerializationContext): SerializerFactory
-    protected open val publicKeySerializer: CustomSerializer.Implements<PublicKey> = net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer
+
+    // Not used as a simple direct import to facilitate testing
+    open val publicKeySerializer : CustomSerializer<*> = net.corda.nodeapi.internal.serialization.amqp.custom.PublicKeySerializer
 
     private fun getSerializerFactory(context: SerializationContext): SerializerFactory {
         return serializerFactoriesForContexts.computeIfAbsent(Pair(context.whitelist, context.deserializationClassLoader)) {
@@ -162,52 +170,3 @@ abstract class AbstractAMQPSerializationScheme(
 
     protected fun canDeserializeVersion(magic: CordaSerializationMagic) = magic == amqpMagic
 }
-
-// TODO: This will eventually cover server RPC as well and move to node module, but for now this is not implemented
-class AMQPServerSerializationScheme(
-    cordappCustomSerializers: Set<SerializationCustomSerializer<*, *>>,
-    serializerFactoriesForContexts: MutableMap<Pair<ClassWhitelist, ClassLoader>, SerializerFactory>
-) : AbstractAMQPSerializationScheme(cordappCustomSerializers, serializerFactoriesForContexts) {
-    constructor(cordapps: List<Cordapp>) : this(cordapps.customSerializers, ConcurrentHashMap())
-
-    constructor() : this(emptySet(), ConcurrentHashMap())
-
-    override fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory {
-        throw UnsupportedOperationException()
-    }
-
-    override fun rpcServerSerializerFactory(context: SerializationContext): SerializerFactory {
-        throw UnsupportedOperationException()
-    }
-
-    override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean {
-        return canDeserializeVersion(magic) &&
-                (target == SerializationContext.UseCase.P2P || target == SerializationContext.UseCase.Storage)
-    }
-
-}
-
-// TODO: This will eventually cover client RPC as well and move to client module, but for now this is not implemented
-class AMQPClientSerializationScheme(
-    cordappCustomSerializers: Set<SerializationCustomSerializer<*,*>>,
-    serializerFactoriesForContexts: MutableMap<Pair<ClassWhitelist, ClassLoader>, SerializerFactory>
-) : AbstractAMQPSerializationScheme(cordappCustomSerializers, serializerFactoriesForContexts) {
-    constructor(cordapps: List<Cordapp>) : this(cordapps.customSerializers, ConcurrentHashMap())
-
-    constructor() : this(emptySet(), ConcurrentHashMap())
-
-    override fun rpcClientSerializerFactory(context: SerializationContext): SerializerFactory {
-        throw UnsupportedOperationException()
-    }
-
-    override fun rpcServerSerializerFactory(context: SerializationContext): SerializerFactory {
-        throw UnsupportedOperationException()
-    }
-
-    override fun canDeserializeVersion(magic: CordaSerializationMagic, target: SerializationContext.UseCase): Boolean {
-        return canDeserializeVersion(magic) &&
-                (target == SerializationContext.UseCase.P2P || target == SerializationContext.UseCase.Storage)
-    }
-
-}
-

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/ClassSerializer.kt

@@ -2,7 +2,6 @@ package net.corda.nodeapi.internal.serialization.amqp.custom
 
 import net.corda.nodeapi.internal.serialization.amqp.CustomSerializer
 import net.corda.nodeapi.internal.serialization.amqp.SerializerFactory
-import net.corda.nodeapi.internal.serialization.amqp.custom.ClassSerializer.ClassProxy
 
 /**
  * A serializer for [Class] that uses [ClassProxy] proxy object to write out

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/InputStreamSerializer.kt

@@ -39,7 +39,7 @@ object InputStreamSerializer : CustomSerializer.Implements<InputStream>(InputStr
 
     override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput,
                             context: SerializationContext
-    ): InputStream {
+    ) : InputStream {
         val bits = input.readObject(obj, schemas, ByteArray::class.java, context) as ByteArray
         return bits.inputStream()
     }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/RxNotificationSerializer.kt

@@ -0,0 +1,28 @@
+package net.corda.nodeapi.internal.serialization.amqp.custom
+
+import net.corda.nodeapi.internal.serialization.amqp.CustomSerializer
+import net.corda.nodeapi.internal.serialization.amqp.SerializerFactory
+import rx.Notification
+
+class RxNotificationSerializer(
+        factory: SerializerFactory
+) : CustomSerializer.Proxy<rx.Notification<*>, RxNotificationSerializer.Proxy>(
+        Notification::class.java,
+        Proxy::class.java,
+        factory
+) {
+    data class Proxy(
+            val kind: Notification.Kind,
+            val t: Throwable?,
+            val value: Any?)
+
+    override fun toProxy(obj: Notification<*>) = Proxy(obj.kind, obj.throwable, obj.value)
+
+    override fun fromProxy(proxy: Proxy): Notification<*> {
+        return when (proxy.kind) {
+            Notification.Kind.OnCompleted -> Notification.createOnCompleted<Any>()
+            Notification.Kind.OnError -> Notification.createOnError<Any>(proxy.t)
+            Notification.Kind.OnNext -> Notification.createOnNext(proxy.value)
+        }
+    }
+}

node-api/src/test/kotlin/net/corda/nodeapi/internal/crypto/X509UtilitiesTest.kt

@@ -8,12 +8,14 @@ import net.corda.core.internal.div
 import net.corda.core.serialization.SerializationContext
 import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
+import net.corda.node.serialization.amqp.AMQPServerSerializationScheme
 import net.corda.node.serialization.kryo.KryoServerSerializationScheme
 import net.corda.nodeapi.internal.config.SSLConfiguration
 import net.corda.nodeapi.internal.createDevKeyStores
 import net.corda.nodeapi.internal.serialization.AllWhitelist
 import net.corda.nodeapi.internal.serialization.SerializationContextImpl
 import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
+import net.corda.nodeapi.internal.serialization.amqp.amqpMagic
 import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
 import net.corda.testing.core.ALICE_NAME
 import net.corda.testing.core.BOB_NAME
@@ -335,8 +337,8 @@ class X509UtilitiesTest {
 
     @Test
     fun `serialize - deserialize X509Certififcate`() {
-        val factory = SerializationFactoryImpl().apply { registerScheme(KryoServerSerializationScheme()) }
-        val context = SerializationContextImpl(kryoMagic,
+        val factory = SerializationFactoryImpl().apply { registerScheme(AMQPServerSerializationScheme()) }
+        val context = SerializationContextImpl(amqpMagic,
                 javaClass.classLoader,
                 AllWhitelist,
                 emptyMap(),
@@ -351,8 +353,8 @@ class X509UtilitiesTest {
 
     @Test
     fun `serialize - deserialize X509CertPath`() {
-        val factory = SerializationFactoryImpl().apply { registerScheme(KryoServerSerializationScheme()) }
-        val context = SerializationContextImpl(kryoMagic,
+        val factory = SerializationFactoryImpl().apply { registerScheme(AMQPServerSerializationScheme()) }
+        val context = SerializationContextImpl(amqpMagic,
                 javaClass.classLoader,
                 AllWhitelist,
                 emptyMap(),

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt

@@ -30,6 +30,7 @@ import net.corda.core.serialization.SerializationContext
 import net.corda.core.serialization.SerializationFactory
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.utilities.OpaqueBytes
+import net.corda.node.serialization.amqp.AMQPServerSerializationScheme
 import net.corda.nodeapi.internal.DEV_INTERMEDIATE_CA
 import net.corda.nodeapi.internal.crypto.ContentSignerBuilder
 import net.corda.nodeapi.internal.serialization.AllWhitelist

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoTests.kt

@@ -12,6 +12,7 @@ import net.corda.core.contracts.PrivacySalt
 import net.corda.core.crypto.*
 import net.corda.core.internal.FetchDataFlow
 import net.corda.core.serialization.*
+import net.corda.core.utilities.ByteSequence
 import net.corda.core.utilities.ProgressTracker
 import net.corda.core.utilities.sequence
 import net.corda.node.serialization.kryo.KryoServerSerializationScheme
@@ -76,11 +77,12 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
         assertThat(bits.deserialize(factory, context)).isEqualTo(Person("bob", null))
     }
 
+
     @Test
     fun `serialised form is stable when the same object instance is added to the deserialised object graph`() {
         val noReferencesContext = context.withoutReferences()
-        val obj = Ints.toByteArray(0x01234567).sequence()
-        val originalList = arrayListOf(obj)
+        val obj : ByteSequence = Ints.toByteArray(0x01234567).sequence()
+        val originalList : ArrayList<ByteSequence> = arrayListOf(obj)
         val deserialisedList = originalList.serialize(factory, noReferencesContext).deserialize(factory, noReferencesContext)
         originalList += obj
         deserialisedList += obj