CORDA-535: Allow notary implementations to specify a serialization filter (#4054)

Only allow custom serialization filters in dev mode.

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -32,6 +32,7 @@ import net.corda.core.serialization.SerializeAsToken
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.utilities.*
 import net.corda.node.CordaClock
+import net.corda.node.SerialFilter
 import net.corda.node.VersionInfo
 import net.corda.node.cordapp.CordappLoader
 import net.corda.node.internal.classloading.requireAnnotation
@@ -86,6 +87,7 @@ import org.slf4j.Logger
 import rx.Observable
 import rx.Scheduler
 import java.io.IOException
+import java.lang.UnsupportedOperationException
 import java.lang.reflect.InvocationTargetException
 import java.nio.file.Paths
 import java.security.KeyPair
@@ -153,6 +155,7 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
             schemaService,
             configuration.dataSourceProperties,
             cacheFactory)
+
     init {
         // TODO Break cyclic dependency
         identityService.database = database
@@ -766,6 +769,10 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
 
             val notaryKey = myNotaryIdentity?.owningKey
                     ?: throw IllegalArgumentException("Unable to start notary service $serviceClass: notary identity not found")
+
+            /** Some notary implementations only work with Java serialization. */
+            maybeInstallSerializationFilter(serviceClass)
+
             val constructor = serviceClass.getDeclaredConstructor(ServiceHubInternal::class.java, PublicKey::class.java).apply { isAccessible = true }
             val service = constructor.newInstance(services, notaryKey) as NotaryService
 
@@ -779,6 +786,23 @@ abstract class AbstractNode<S>(val configuration: NodeConfiguration,
         }
     }
 
+    /** Installs a custom serialization filter defined by a notary service implementation. Only supported in dev mode. */
+    private fun maybeInstallSerializationFilter(serviceClass: Class<out NotaryService>) {
+        try {
+            @Suppress("UNCHECKED_CAST")
+            val filter = serviceClass.getDeclaredMethod("getSerializationFilter").invoke(null) as ((Class<*>) -> Boolean)
+            if (configuration.devMode) {
+                log.warn("Installing a custom Java serialization filter, required by ${serviceClass.name}. " +
+                        "Note this is only supported in dev mode – a production node will fail to start if serialization filters are used.")
+                SerialFilter.install(filter)
+            } else {
+                throw UnsupportedOperationException("Unable to install a custom Java serialization filter, not in dev mode.")
+            }
+        } catch (e: NoSuchMethodException) {
+            // No custom serialization filter declared
+        }
+    }
+
     private fun getNotaryServiceClass(className: String): Class<out NotaryService> {
         val loadedImplementations = cordappLoader.cordapps.mapNotNull { it.notaryService }
         log.debug("Notary service implementations found: ${loadedImplementations.joinToString(", ")}")

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -414,17 +414,8 @@ open class NodeStartup : CordaCliWrapper("corda", "Runs a Corda Node") {
     protected open fun loadConfigFile(): Pair<Config, Try<NodeConfiguration>> = cmdLineOptions.loadConfig()
 
     protected open fun banJavaSerialisation(conf: NodeConfiguration) {
-        SerialFilter.install(if (conf.notary?.bftSMaRt != null) ::bftSMaRtSerialFilter else ::defaultSerialFilter)
-    }
-
-    /** This filter is required for BFT-Smart to work as it only supports Java serialization. */
-    // TODO: move this filter out of the node, allow Cordapps to specify filters.
-    private fun bftSMaRtSerialFilter(clazz: Class<*>): Boolean = clazz.name.let {
-        it.startsWith("bftsmart.")
-                || it.startsWith("java.security.")
-                || it.startsWith("java.util.")
-                || it.startsWith("java.lang.")
-                || it.startsWith("java.net.")
+        // Note that in dev mode this filter can be overridden by a notary service implementation.
+        SerialFilter.install(::defaultSerialFilter)
     }
 
     protected open fun getVersionInfo(): VersionInfo {