Identity service refactor for confidential-identities and accounts (#5434)

* Removed IdentityServiceInternal as it is no longer used.

* Removed externalIdForPublicKey API from KMS and added it to IdentityService.
Added a registerKeyToExternalId API on IdentityService.

* Fix remaining compile errors.

* Removed "registerKeyToParty" and in its place added a new registerKey method which takes a PublicKey, Party and optionally a UUID.
Added a cache to the "PersistentIdentityService" to store other node's public keys.
Added the cache and new hibernate entity to all teh places where one needs to add them.
New keys created by teh node now automatically get associated entries in the KEY -> PARTY map and optionally the KEy -> EXT ID map.
Added a test.

* Removed old comments and TODOs.

* Fixed broken test.
Added comments/explanations for what's going on in IdentityService.
Updated kdocs.

* First try at Implementing publicKeysForExternalId.

* Fixed broken test.

* Added migration.
Amended existing persistent identity service migration to handle new migration.
Addressed some review comments.

* Fixed broken test - whoops!

* Implemented mock identity service methods.

* Added back exception when remapping a key to a different party.

* Fixed compile errors.
Fixed broken tests.

* Use set instead of first entry in ourNames.

testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt

@@ -79,8 +79,7 @@ open class MockServices private constructor(
         private val moreKeys: Array<out KeyPair>,
         override val keyManagementService: KeyManagementService = MockKeyManagementService(
                 identityService,
-                *arrayOf(initialIdentity.keyPair) + moreKeys,
-                pkToIdCache = MockPublicKeyToOwningIdentityCache()
+                *arrayOf(initialIdentity.keyPair) + moreKeys
         )
 ) : ServiceHub {
 
@@ -128,8 +127,7 @@ open class MockServices private constructor(
             val database = configureDatabase(dataSourceProps, DatabaseConfig(), identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, schemaService, schemaService.internalSchemas())
             val keyManagementService = MockKeyManagementService(
                     identityService,
-                    *arrayOf(initialIdentity.keyPair) + moreKeys,
-                    pkToIdCache = MockPublicKeyToOwningIdentityCache()
+                    *arrayOf(initialIdentity.keyPair) + moreKeys
             )
             val mockService = database.transaction {
                 makeMockMockServices(cordappLoader, identityService, networkParameters, initialIdentity, moreKeys.toSet(), keyManagementService, schemaService, database)
@@ -160,20 +158,28 @@ open class MockServices private constructor(
             val dataSourceProps = makeTestDataSourceProperties()
             val schemaService = NodeSchemaService(cordappLoader.cordappSchemas)
             val identityService = PersistentIdentityService(TestingNamedCacheFactory())
-            val persistence = configureDatabase(dataSourceProps, DatabaseConfig(), identityService::wellKnownPartyFromX500Name, identityService::wellKnownPartyFromAnonymous, schemaService, schemaService.internalSchemas())
+            val persistence = configureDatabase(
+                    hikariProperties = dataSourceProps,
+                    databaseConfig = DatabaseConfig(),
+                    wellKnownPartyFromX500Name = identityService::wellKnownPartyFromX500Name,
+                    wellKnownPartyFromAnonymous = identityService::wellKnownPartyFromAnonymous,
+                    schemaService = schemaService,
+                    internalSchemas = schemaService.internalSchemas()
+            )
+
+            val pkToIdCache = PublicKeyToOwningIdentityCacheImpl(persistence, TestingNamedCacheFactory())
 
             // Create a persistent identity service and add all the supplied identities.
             identityService.apply {
                 ourNames = setOf(initialIdentity.name)
                 database = persistence
-                start(DEV_ROOT_CA.certificate)
+                start(DEV_ROOT_CA.certificate, pkToIdCache = pkToIdCache)
                 persistence.transaction { identityService.loadIdentities(moreIdentities + initialIdentity.identity) }
             }
 
             // Create a persistent key management service and add the key pair which was created for the TestIdentity.
             // We only add the keypair for the initial identity and any other keys which this node may control. Note: We don't add the keys
             // for the other identities.
-            val pkToIdCache = PublicKeyToOwningIdentityCacheImpl(persistence, TestingNamedCacheFactory())
             val aliasKeyMap = mutableMapOf<String, KeyPair>()
             val aliasedMoreKeys = moreKeys.mapIndexed { index, keyPair ->
                 val alias = "Extra key $index"
@@ -187,8 +193,7 @@ open class MockServices private constructor(
                     TestingNamedCacheFactory(),
                     identityService,
                     persistence,
-                    MockCryptoService(aliasKeyMap),
-                    pkToIdCache
+                    MockCryptoService(aliasKeyMap)
             )
             persistence.transaction { keyManagementService.start(aliasedMoreKeys + aliasedIdentityKey) }
 

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt

@@ -373,7 +373,7 @@ open class InternalMockNetwork(cordappPackages: List<String> = emptyList(),
         }
 
         override fun makeKeyManagementService(identityService: PersistentIdentityService): KeyManagementServiceInternal {
-            return BasicHSMKeyManagementService(cacheFactory, identityService, database, cryptoService, pkToIdCache)
+            return BasicHSMKeyManagementService(cacheFactory, identityService, database, cryptoService)
         }
 
         override fun startShell() {

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/MockKeyManagementService.kt

@@ -4,6 +4,8 @@ import net.corda.core.crypto.*
 import net.corda.core.node.services.IdentityService
 import net.corda.core.node.services.KeyManagementService
 import net.corda.core.serialization.SingletonSerializeAsToken
+import net.corda.node.services.identity.InMemoryIdentityService
+import net.corda.node.services.identity.PersistentIdentityService
 import net.corda.node.services.keys.KeyManagementServiceInternal
 import net.corda.node.services.persistence.WritablePublicKeyToOwningIdentityCache
 import net.corda.nodeapi.internal.KeyOwningIdentity
@@ -18,10 +20,10 @@ import java.util.*
  *
  * @property identityService The [IdentityService] which contains the given identities.
  */
-class MockKeyManagementService(override val identityService: IdentityService,
-                               vararg initialKeys: KeyPair,
-                               private val pkToIdCache: WritablePublicKeyToOwningIdentityCache) : SingletonSerializeAsToken(), KeyManagementServiceInternal {
-
+class MockKeyManagementService(
+        override val identityService: IdentityService,
+        vararg initialKeys: KeyPair
+) : SingletonSerializeAsToken(), KeyManagementServiceInternal {
 
     private val keyStore: MutableMap<PublicKey, PrivateKey> = initialKeys.associateByTo(HashMap(), { it.public }, { it.private })
 
@@ -32,7 +34,9 @@ class MockKeyManagementService(override val identityService: IdentityService,
     override fun freshKeyInternal(externalId: UUID?): PublicKey {
         val k = nextKeys.poll() ?: generateKeyPair()
         keyStore[k.public] = k.private
-        pkToIdCache[k.public] = KeyOwningIdentity.fromUUID(externalId)
+        if (externalId != null) {
+            (identityService as InMemoryIdentityService).registerKeyToExternalId(k.public, externalId)
+        }
         return k.public
     }
 
@@ -59,8 +63,4 @@ class MockKeyManagementService(override val identityService: IdentityService,
         val keyPair = getSigningKeyPair(publicKey)
         return keyPair.sign(signableData)
     }
-
-    override fun externalIdForPublicKey(publicKey: PublicKey): UUID? {
-        return pkToIdCache[publicKey]?.uuid
-    }
 }