CORDA-3717: Apply custom serializers to checkpoints (#6392)

* CORDA-3717: Apply custom serializers to checkpoints

* Remove try/catch to fix TooGenericExceptionCaught detekt rule

* Rename exception

* Extract method

* Put calls to the userSerializer on their own lines to improve readability

* Remove unused constructors from exception

* Remove unused proxyType field

* Give field a descriptive name

* Explain why we are looking for two type parameters when we only use one

* Tidy up the fetching of types

* Use 0 seconds when forcing a flow checkpoint inside test

* Add test to check references are restored correctly

* Add CheckpointCustomSerializer interface

* Wire up the new CheckpointCustomSerializer interface

* Use kryo default for abstract classes

* Remove unused imports

* Remove need for external library in tests

* Make file match original to remove from diff

* Remove maySkipCheckpoint from calls to sleep

* Add newline to end of file

* Test custom serializers mapped to interfaces

* Test serializer configured with abstract class

* Move test into its own package

* Rename test

* Move flows and serializers into their own source file

* Move broken map into its own source file

* Delete comment now source file is simpler

* Rename class to have a shorter name

* Add tests that run the checkpoint serializer directly

* Check serialization of final classes

* Register as default unless the target class is final

* Test PublicKey serializer has not been overridden

* Add a broken serializer for EdDSAPublicKey to make test more robust

* Split serializer registration into default and non-default registrations. Run registrations at the right time to preserve Cordas own custom serializers.

* Check for duplicate custom checkpoint serializers

* Add doc comments

* Add doc comments to CustomSerializerCheckpointAdaptor

* Add test to check duplicate serializers are logged

* Do not log the duplicate serializer warning when the duplicate is the same class

* Update doc comment for CheckpointCustomSerializer

* Sort serializers by classname so we are not registering in an unknown or random order

* Add test to serialize a class that references itself

* Store custom serializer type in the Kryo stream so we can spot when a different serializer is being used to deserialize

* Testing has shown that registering custom serializers as default is more robust when adding new cordapps

* Remove new line character

* Remove unused imports

* Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt

* Remove comment

* Update comment on exception

* Make CustomSerializerCheckpointAdaptor internal

* Revert "Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt"

This reverts commit b835de79bd.

* Restore "Add interface net.corda.core.serialization.CheckpointCustomSerializer to api-current.txt""

This reverts commit 718873a4e9.

* Pass the class loader instead of the context

* Do less work in test setup

* Make the serialization context unique for CustomCheckpointSerializerTest so we get a new Kryo pool for the test

* Rebuild the Kryo pool for the given context when we change custom serializers

* Rebuild all Kryo pools on serializer change to keep serializer list consistent

* Move the custom serializer list into CheckpointSerializationContext to reduce scope from global to a serialization context

* Remove unused imports

* Make the new checkpointCustomSerializers property default to the empty list

* Delegate implementation using kotlin language feature

core/src/main/kotlin/net/corda/core/cordapp/Cordapp.kt

@@ -7,6 +7,7 @@ import net.corda.core.crypto.SecureHash
 import net.corda.core.flows.FlowLogic
 import net.corda.core.internal.cordapp.CordappImpl.Companion.UNKNOWN_VALUE
 import net.corda.core.schemas.MappedSchema
+import net.corda.core.serialization.CheckpointCustomSerializer
 import net.corda.core.serialization.SerializationCustomSerializer
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
@@ -29,6 +30,7 @@ import java.net.URL
  * @property services List of RPC services
  * @property serializationWhitelists List of Corda plugin registries
  * @property serializationCustomSerializers List of serializers
+ * @property checkpointCustomSerializers List of serializers for checkpoints
  * @property customSchemas List of custom schemas
  * @property allFlows List of all flow classes
  * @property jarPath The path to the JAR for this CorDapp
@@ -49,6 +51,7 @@ interface Cordapp {
     val services: List<Class<out SerializeAsToken>>
     val serializationWhitelists: List<SerializationWhitelist>
     val serializationCustomSerializers: List<SerializationCustomSerializer<*, *>>
+    val checkpointCustomSerializers: List<CheckpointCustomSerializer<*, *>>
     val customSchemas: Set<MappedSchema>
     val allFlows: List<Class<out FlowLogic<*>>>
     val jarPath: URL

core/src/main/kotlin/net/corda/core/internal/cordapp/CordappImpl.kt

@@ -9,6 +9,7 @@ import net.corda.core.internal.VisibleForTesting
 import net.corda.core.internal.notary.NotaryService
 import net.corda.core.internal.toPath
 import net.corda.core.schemas.MappedSchema
+import net.corda.core.serialization.CheckpointCustomSerializer
 import net.corda.core.serialization.SerializationCustomSerializer
 import net.corda.core.serialization.SerializationWhitelist
 import net.corda.core.serialization.SerializeAsToken
@@ -25,6 +26,7 @@ data class CordappImpl(
         override val services: List<Class<out SerializeAsToken>>,
         override val serializationWhitelists: List<SerializationWhitelist>,
         override val serializationCustomSerializers: List<SerializationCustomSerializer<*, *>>,
+        override val checkpointCustomSerializers: List<CheckpointCustomSerializer<*, *>>,
         override val customSchemas: Set<MappedSchema>,
         override val allFlows: List<Class<out FlowLogic<*>>>,
         override val jarPath: URL,
@@ -79,6 +81,7 @@ data class CordappImpl(
                 services = emptyList(),
                 serializationWhitelists = emptyList(),
                 serializationCustomSerializers = emptyList(),
+                checkpointCustomSerializers = emptyList(),
                 customSchemas = emptySet(),
                 jarPath = Paths.get("").toUri().toURL(),
                 info = UNKNOWN_INFO,

core/src/main/kotlin/net/corda/core/serialization/SerializationCustomSerializer.kt

@@ -25,3 +25,26 @@ interface SerializationCustomSerializer<OBJ, PROXY> {
      */
     fun fromProxy(proxy: PROXY): OBJ
 }
+
+/**
+ * Allows CorDapps to provide custom serializers for classes that do not serialize successfully during a checkpoint.
+ * In this case, a proxy serializer can be written that implements this interface whose purpose is to move between
+ * unserializable types and an intermediate representation.
+ *
+ * NOTE: Only implement this interface if you have a class that triggers an error during normal checkpoint
+ * serialization/deserialization.
+ */
+@KeepForDJVM
+interface CheckpointCustomSerializer<OBJ, PROXY> {
+    /**
+     * Should facilitate the conversion of the third party object into the serializable
+     * local class specified by [PROXY]
+     */
+    fun toProxy(obj: OBJ): PROXY
+
+    /**
+     * Should facilitate the conversion of the proxy object into a new instance of the
+     * unserializable type
+     */
+    fun fromProxy(proxy: PROXY): OBJ
+}

core/src/main/kotlin/net/corda/core/serialization/internal/CheckpointSerializationAPI.kt

@@ -56,6 +56,10 @@ interface CheckpointSerializationContext {
      * otherwise they appear as new copies of the object.
      */
     val objectReferencesEnabled: Boolean
+    /**
+     * User defined custom serializers for use in checkpoint serialization.
+     */
+    val checkpointCustomSerializers: Iterable<CheckpointCustomSerializer<*,*>>
 
     /**
      * Helper method to return a new context based on this context with the property added.
@@ -86,6 +90,11 @@ interface CheckpointSerializationContext {
      * A shallow copy of this context but with the given encoding whitelist.
      */
     fun withEncodingWhitelist(encodingWhitelist: EncodingWhitelist): CheckpointSerializationContext
+
+    /**
+     * A shallow copy of this context but with the given custom serializers.
+     */
+    fun withCheckpointCustomSerializers(checkpointCustomSerializers: Iterable<CheckpointCustomSerializer<*, *>>): CheckpointSerializationContext
 }
 
 /*