ENT-6947 Intern common types to reduce heap footprint (#7239)

ENT-6947: Implement interning for SecureHash, CordaX500Name, PublicKey, AsbtractParty and SignatureAttachmentConstraint, including automatic detection of internable types off companion objects in AMQP & Kyro deserialization.  In some cases, add new factory methods to companion objects, and make main code base use them.

Performance tested in performance cluster with no negative impact visible (so default concurrency setting seems okay).

Testing suggests 5-6x memory saving for tokens in TokensSDK in memory selector.  Should see approx. 1 million tokens per GB or better (1.5 million for the tokens we tested with).

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/CordaClassResolver.kt

@@ -1,17 +1,23 @@
 package net.corda.nodeapi.internal.serialization.kryo
 
-import com.esotericsoftware.kryo.*
+import com.esotericsoftware.kryo.DefaultSerializer
+import com.esotericsoftware.kryo.Kryo
+import com.esotericsoftware.kryo.KryoException
+import com.esotericsoftware.kryo.KryoSerializable
+import com.esotericsoftware.kryo.Registration
+import com.esotericsoftware.kryo.Serializer
 import com.esotericsoftware.kryo.io.Input
 import com.esotericsoftware.kryo.io.Output
 import com.esotericsoftware.kryo.serializers.FieldSerializer
 import com.esotericsoftware.kryo.util.DefaultClassResolver
 import com.esotericsoftware.kryo.util.Util
 import net.corda.core.internal.kotlinObjectInstance
+import net.corda.core.internal.utilities.PrivateInterner
 import net.corda.core.internal.writer
-import net.corda.core.serialization.internal.CheckpointSerializationContext
 import net.corda.core.serialization.ClassWhitelist
-import net.corda.core.utilities.contextLogger
 import net.corda.core.serialization.internal.AttachmentsClassLoader
+import net.corda.core.serialization.internal.CheckpointSerializationContext
+import net.corda.core.utilities.contextLogger
 import net.corda.serialization.internal.MutableClassWhitelist
 import net.corda.serialization.internal.TransientClassWhiteList
 import net.corda.serialization.internal.amqp.hasCordaSerializable
@@ -19,8 +25,11 @@ import java.io.PrintWriter
 import java.lang.reflect.Modifier.isAbstract
 import java.nio.charset.StandardCharsets.UTF_8
 import java.nio.file.Paths
-import java.nio.file.StandardOpenOption.*
-import java.util.*
+import java.nio.file.StandardOpenOption.APPEND
+import java.nio.file.StandardOpenOption.CREATE
+import java.nio.file.StandardOpenOption.WRITE
+import java.util.ArrayList
+import java.util.Collections
 
 /**
  * Corda specific class resolver which enables extra customisation for the purposes of serialization using Kryo
@@ -86,7 +95,7 @@ class CordaClassResolver(serializationContext: CheckpointSerializationContext) :
                 kotlin.jvm.internal.Lambda::class.java.isAssignableFrom(targetType) -> // Kotlin lambdas extend this class and any captured variables are stored in synthetic fields
                     FieldSerializer<Any>(kryo, targetType).apply { setIgnoreSyntheticFields(false) }
                 Throwable::class.java.isAssignableFrom(targetType) -> ThrowableSerializer(kryo, targetType)
-                else -> kryo.getDefaultSerializer(targetType)
+                else -> maybeWrapForInterning(kryo.getDefaultSerializer(targetType), targetType)
             }
             return register(Registration(targetType, serializer, NAME.toInt()))
         } finally {
@@ -94,6 +103,11 @@ class CordaClassResolver(serializationContext: CheckpointSerializationContext) :
         }
     }
 
+    private fun maybeWrapForInterning(serializer: Serializer<Any>, targetType: Class<*>): Serializer<Any> {
+        val interner = PrivateInterner.findFor(targetType)
+        return if (interner != null) InterningSerializer(serializer, interner) else serializer
+    }
+
     override fun writeName(output: Output, type: Class<*>, registration: Registration) {
         super.writeName(output, registration.type ?: type, registration)
     }
@@ -104,6 +118,11 @@ class CordaClassResolver(serializationContext: CheckpointSerializationContext) :
         override fun write(kryo: Kryo, output: Output, obj: Any) = Unit
     }
 
+    private class InterningSerializer(private val delegate: Serializer<Any>, private val interner: PrivateInterner<Any>) : Serializer<Any>() {
+        override fun read(kryo: Kryo, input: Input, type: Class<Any>): Any = interner.intern(delegate.read(kryo, input, type))
+        override fun write(kryo: Kryo, output: Output, obj: Any) = delegate.write(kryo, output, obj)
+    }
+
     // We don't allow the annotation for classes in attachments for now.  The class will be on the main classpath if we have the CorDapp installed.
     // We also do not allow extension of KryoSerializable for annotated classes, or combination with @DefaultSerializer for custom serialisation.
     // TODO: Later we can support annotations on attachment classes and spin up a proxy via bytecode that we know is harmless.

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/DefaultKryoCustomizer.kt

@@ -59,9 +59,8 @@ import java.security.PrivateKey
 import java.security.PublicKey
 import java.security.cert.CertPath
 import java.security.cert.X509Certificate
-import java.util.Arrays
-import java.util.BitSet
-import java.util.ServiceLoader
+import java.util.*
+import kotlin.collections.ArrayList
 
 object DefaultKryoCustomizer {
     private val serializationWhitelists: List<SerializationWhitelist> by lazy {
@@ -233,7 +232,7 @@ object DefaultKryoCustomizer {
         @Suppress("UNCHECKED_CAST")
         override fun read(kryo: Kryo, input: Input, type: Class<ContractAttachment>): ContractAttachment {
             if (kryo.serializationContext() != null) {
-                val attachmentHash = SecureHash.SHA256(input.readBytes(32))
+                val attachmentHash = SecureHash.createSHA256(input.readBytes(32))
                 val contract = input.readString()
                 val additionalContracts = kryo.readClassAndObject(input) as Set<ContractClassName>
                 val uploader = input.readString()

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/kryo/KryoTests.kt

@@ -9,22 +9,38 @@ import com.google.common.primitives.Ints
 import com.nhaarman.mockito_kotlin.doReturn
 import com.nhaarman.mockito_kotlin.whenever
 import net.corda.core.contracts.PrivacySalt
-import net.corda.core.crypto.*
+import net.corda.core.contracts.SignatureAttachmentConstraint
+import net.corda.core.crypto.Crypto
+import net.corda.core.crypto.SecureHash
+import net.corda.core.crypto.SignableData
+import net.corda.core.crypto.SignatureMetadata
+import net.corda.core.crypto.generateKeyPair
+import net.corda.core.crypto.sha256
+import net.corda.core.crypto.sign
+import net.corda.core.identity.AnonymousParty
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.identity.Party
 import net.corda.core.internal.FetchDataFlow
-import net.corda.core.serialization.*
+import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.EncodingWhitelist
 import net.corda.core.serialization.internal.CheckpointSerializationContext
 import net.corda.core.serialization.internal.checkpointDeserialize
 import net.corda.core.serialization.internal.checkpointSerialize
 import net.corda.core.utilities.ByteSequence
 import net.corda.core.utilities.ProgressTracker
 import net.corda.core.utilities.sequence
-import net.corda.serialization.internal.*
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.serialization.internal.AllWhitelist
+import net.corda.serialization.internal.CheckpointSerializationContextImpl
+import net.corda.serialization.internal.CordaSerializationEncoding
+import net.corda.serialization.internal.encodingNotPermittedFormat
 import net.corda.testing.core.ALICE_NAME
 import net.corda.testing.core.TestIdentity
 import net.corda.testing.core.internal.CheckpointSerializationEnvironmentRule
-import net.corda.coretesting.internal.rigorousMock
 import org.apache.commons.lang3.SystemUtils
-import org.assertj.core.api.Assertions.*
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.assertj.core.api.Assertions.catchThrowable
 import org.junit.Assert.assertArrayEquals
 import org.junit.Assert.assertEquals
 import org.junit.Before
@@ -36,9 +52,13 @@ import org.junit.runners.Parameterized.Parameters
 import org.slf4j.LoggerFactory
 import java.io.InputStream
 import java.time.Instant
-import java.util.*
-import kotlin.collections.ArrayList
-import kotlin.test.*
+import java.util.Collections
+import java.util.Random
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertNotNull
+import kotlin.test.assertSame
+import kotlin.test.assertTrue
 
 @RunWith(Parameterized::class)
 class KryoTests(private val compression: CordaSerializationEncoding?) {
@@ -129,6 +149,7 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
         val deserialisedSignature = deserialisedKeyPair.sign(bitsToSign)
         deserialisedSignature.verify(bitsToSign)
         assertThatThrownBy { deserialisedSignature.verify(wrongBits) }
+        assertSame(keyPair.public, deserialisedKeyPair.public)
     }
 
     @Test(timeout=300_000)
@@ -178,7 +199,7 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
     }
 
     @Test(timeout=300_000)
-	fun `serialize - deserialize SignableData`() {
+    fun `serialize - deserialize SignableData`() {
         val testString = "Hello World"
         val testBytes = testString.toByteArray()
 
@@ -186,15 +207,33 @@ class KryoTests(private val compression: CordaSerializationEncoding?) {
         val serializedMetaData = meta.checkpointSerialize(context).bytes
         val meta2 = serializedMetaData.checkpointDeserialize<SignableData>(context)
         assertEquals(meta2, meta)
+        assertSame(meta.txId, meta2.txId)
     }
 
-    @Test(timeout=300_000)
-	fun `serialize - deserialize Logger`() {
+    @Test(timeout = 300_000)
+    fun `serialize - deserialize internables`() {
+        val list: List<Any> = listOf(
+                SecureHash.randomSHA256(),
+                CordaX500Name.parse("O=bank A, L=New York, C=DE, OU=Org Unit, CN=Service Name"),
+                Party.create(CordaX500Name.parse("O=bank A, L=New York, C=DE, OU=Org Unit, CN=Service Name"), Crypto.generateKeyPair().public),
+                AnonymousParty.create(Crypto.generateKeyPair().public),
+                SignatureAttachmentConstraint.create(Crypto.generateKeyPair().public)
+        )
+
+        val serializedList = list.checkpointSerialize(context).bytes
+        val list2 = serializedList.checkpointDeserialize<List<Any>>(context)
+        list.zip(list2).forEach { (original, deserialized) ->
+            assertSame(original, deserialized, "${original.javaClass} not interned")
+        }
+    }
+
+    @Test(timeout = 300_000)
+    fun `serialize - deserialize Logger`() {
         val storageContext: CheckpointSerializationContext = context
         val logger = LoggerFactory.getLogger("aName")
         val logger2 = logger.checkpointSerialize(storageContext).checkpointDeserialize(storageContext)
         assertEquals(logger.name, logger2.name)
-        assertTrue(logger === logger2)
+        assertSame(logger, logger2)
     }
 
     @CordaSerializable