CORDA-535: Allow notary implementations to specify a serialization filter (#4054)

Only allow custom serialization filters in dev mode.

docs/source/changelog.rst

@@ -11,6 +11,9 @@ Unreleased
 
 * Introduce minimum and target platform version for CorDapps.
 
+* BFT-Smart and Raft notary implementations have been extracted out of node into ``experimental`` CorDapps to emphasise
+  their experimental nature. Moreover, the BFT-Smart notary will only work in dev mode due to its use of Java serialization.
+
 * Vault storage of contract state constraints metadata and associated vault query functions to retrieve and sort by constraint type.
 
 * New overload for ``CordaRPCClient.start()`` method allowing to specify target legal identity to use for RPC call.

docs/source/running-a-notary.rst

@@ -43,6 +43,8 @@ Byzantine fault-tolerant (experimental)
 
 A prototype BFT notary implementation based on `BFT-Smart <https://github.com/bft-smart/library>`_ is available. You can
 try it out on our `notary demo <https://github.com/corda/corda/tree/release-V3.1/samples/notary-demo>`_ page. Note that it
-is still experimental and there is active work ongoing for a production ready solution.
+is still experimental and there is active work ongoing for a production ready solution. Additionally, BFT-Smart requires Java
+serialization which is disabled by default in Corda due to security risks, and it will only work in dev mode where this can
+be customised.
 
 We do not recommend using it in any long-running test or production deployments.

experimental/notary-bft-smart/src/main/kotlin/net/corda/notary/bftsmart/BFTSMaRtConfig.kt

@@ -92,10 +92,3 @@ fun maxFaultyReplicas(clusterSize: Int) = (clusterSize - 1) / 3
 fun minCorrectReplicas(clusterSize: Int) = (2 * clusterSize + 3) / 3
 fun minClusterSize(maxFaultyReplicas: Int) = maxFaultyReplicas * 3 + 1
 
-fun bftSMaRtSerialFilter(clazz: Class<*>): Boolean = clazz.name.let {
-    it.startsWith("bftsmart.")
-            || it.startsWith("java.security.")
-            || it.startsWith("java.util.")
-            || it.startsWith("java.lang.")
-            || it.startsWith("java.net.")
-}

experimental/notary-bft-smart/src/main/kotlin/net/corda/notary/bftsmart/BftSmartNotaryService.kt

@@ -10,7 +10,6 @@ import net.corda.core.identity.Party
 import net.corda.core.internal.notary.NotaryInternalException
 import net.corda.core.internal.notary.NotaryService
 import net.corda.core.internal.notary.verifySignature
-import net.corda.core.schemas.MappedSchema
 import net.corda.core.schemas.PersistentStateRef
 import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
@@ -41,6 +40,17 @@ class BftSmartNotaryService(
 ) : NotaryService() {
     companion object {
         private val log = contextLogger()
+        @JvmStatic
+        val serializationFilter
+            get() = { clazz: Class<*> ->
+                clazz.name.let {
+                    it.startsWith("bftsmart.")
+                            || it.startsWith("java.security.")
+                            || it.startsWith("java.util.")
+                            || it.startsWith("java.lang.")
+                            || it.startsWith("java.net.")
+                }
+            }
     }
 
     private val notaryConfig = services.configuration.notary

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -32,6 +32,7 @@ import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.utilities.*
 import net.corda.node.CordaClock
+import net.corda.node.SerialFilter
 import net.corda.node.VersionInfo
 import net.corda.node.cordapp.CordappLoader
 import net.corda.node.internal.classloading.requireAnnotation
@@ -86,6 +87,7 @@ import org.slf4j.Logger
 import rx.Observable
 import rx.Scheduler
 import java.io.IOException
+import java.lang.UnsupportedOperationException
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.Paths
 import java.security.KeyPair
@@ -153,6 +155,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             schemaService,
             configuration.dataSourceProperties,
             cacheFactory)
+
     init {
         // TODO Break cyclic dependency
         identityService.database = database
@@ -766,6 +769,10 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
 
             val notaryKey = myNotaryIdentity?.owningKey
                     ?: throw IllegalArgumentException("Unable to start notary service $serviceClass: notary identity not found")
+
+            /** Some notary implementations only work with Java serialization. */
+            maybeInstallSerializationFilter(serviceClass)
+
             val constructor = serviceClass.getDeclaredConstructor(ServiceHubInternal::class.java, PublicKey::class.java).apply { isAccessible = true }
             val service = constructor.newInstance(services, notaryKey) as NotaryService
 
@@ -779,6 +786,23 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         }
     }
 
+    /** Installs a custom serialization filter defined by a notary service implementation. Only supported in dev mode. */
+    private fun maybeInstallSerializationFilter(serviceClass: Class<out NotaryService>) {
+        try {
+            @Suppress("UNCHECKED_CAST")
+            val filter = serviceClass.getDeclaredMethod("getSerializationFilter").invoke(null) as ((Class<*>) -> Boolean)
+            if (configuration.devMode) {
+                log.warn("Installing a custom Java serialization filter, required by ${serviceClass.name}. " +
+                        "Note this is only supported in dev mode – a production node will fail to start if serialization filters are used.")
+                SerialFilter.install(filter)
+            } else {
+                throw UnsupportedOperationException("Unable to install a custom Java serialization filter, not in dev mode.")
+            }
+        } catch (e: NoSuchMethodException) {
+            // No custom serialization filter declared
+        }
+    }
+
     private fun getNotaryServiceClass(className: String): Class<out NotaryService> {
         val loadedImplementations = cordappLoader.cordapps.mapNotNull { it.notaryService }
         log.debug("Notary service implementations found: ${loadedImplementations.joinToString(", ")}")

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -414,17 +414,8 @@ open class NodeStartup : CordaCliWrapper("corda", "Runs a Corda Node") {
     protected open fun loadConfigFile(): Pair<Config, Try<NodeConfiguration>> = cmdLineOptions.loadConfig()
 
     protected open fun banJavaSerialisation(conf: NodeConfiguration) {
-        SerialFilter.install(if (conf.notary?.bftSMaRt != null) ::bftSMaRtSerialFilter else ::defaultSerialFilter)
+        // Note that in dev mode this filter can be overridden by a notary service implementation.
-    }
+        SerialFilter.install(::defaultSerialFilter)
-
-    /** This filter is required for BFT-Smart to work as it only supports Java serialization. */
-    // TODO: move this filter out of the node, allow Cordapps to specify filters.
-    private fun bftSMaRtSerialFilter(clazz: Class<*>): Boolean = clazz.name.let {
-        it.startsWith("bftsmart.")
-                || it.startsWith("java.security.")
-                || it.startsWith("java.util.")
-                || it.startsWith("java.lang.")
-                || it.startsWith("java.net.")
     }
 
     protected open fun getVersionInfo(): VersionInfo {