CORDA-2232: external id to pubkey mapping (#4210)

* First pass
Update test.
Address review comments.
Added docs and kdocs.
Clean-up.

* Addressed review comments.

Changes to docsite.

* First pass at account service.
Added new hibernate schemas and liquibase scripts.
Added indexes to new tables.
Removed mock network.
Removed fresh key for external id from key management service.
Removed some redundant changes.
Rebase to master.

* Clean up.

* Added try/catch block as recommended by Andras.

* Removed accounts test to another branch.
Removed element collections from fungible states and linear states table.
Added a new state_parties table which stores x500 names and public key hashes.
Added a view which can be used to query by external ID.

* Removed try catch block. It's not required as the checkpoint serialiser deals with this.
Re-used existing DB session instead of creating a new session.
Entity manager auto flushes.

* Added java friendly api.

* 
 This is a combination of 10 commits.

 This is the 1st commit message:

Shortened table name.


 This is the commit message #2:

Minor changes.


 This is the commit message #3:

Common criteria parser now returns a predicate set which is concatenated to the predicate sets of sub-class criteria.


 This is the commit message #4:

Fixed api compatibility issue.
Reverted some changes to reduce size of PR.


 This is the commit message #5:

Multiple states can now be mapped to the same externalId.
Multiple externalIds can now be mapped to the same state.


 This is the commit message #6:

Relaxed upper bound type constraint in some of the vault types.


 This is the commit message #7:

Added comment to test.


 This is the commit message #8:

Changed name of external id to public key join table.
Removed some comments/TODOs.


 This is the commit message #9:

Added docs.
General clean up.


 This is the commit message #10:

Fixed participants query bug and updated unit test.

* Removed unused code.

.ci/api-current.txt

@@ -4041,7 +4041,7 @@ public static final class net.corda.core.node.services.vault.QueryCriteria$Fungi
   @Nullable
   public final java.util.List<net.corda.core.identity.AbstractParty> getOwner()
   @Nullable
-  public final java.util.List<net.corda.core.identity.AbstractParty> getParticipants()
+  public java.util.List<net.corda.core.identity.AbstractParty> getParticipants()
   @Nullable
   public final net.corda.core.node.services.vault.ColumnPredicate<Long> getQuantity()
   @NotNull
@@ -4078,7 +4078,7 @@ public static final class net.corda.core.node.services.vault.QueryCriteria$Linea
   @Nullable
   public final java.util.List<String> getExternalId()
   @Nullable
-  public final java.util.List<net.corda.core.identity.AbstractParty> getParticipants()
+  public java.util.List<net.corda.core.identity.AbstractParty> getParticipants()
   @NotNull
   public net.corda.core.node.services.Vault$StateStatus getStatus()
   @Nullable
@@ -4305,16 +4305,16 @@ public abstract class net.corda.core.node.services.vault.SortAttribute extends j
 ##
 @CordaSerializable
 public static final class net.corda.core.node.services.vault.SortAttribute$Custom extends net.corda.core.node.services.vault.SortAttribute
-  public <init>(Class<? extends net.corda.core.schemas.PersistentState>, String)
+  public <init>(Class<? extends net.corda.core.schemas.StatePersistable>, String)
   @NotNull
-  public final Class<? extends net.corda.core.schemas.PersistentState> component1()
+  public final Class<? extends net.corda.core.schemas.StatePersistable> component1()
   @NotNull
   public final String component2()
   @NotNull
-  public final net.corda.core.node.services.vault.SortAttribute$Custom copy(Class<? extends net.corda.core.schemas.PersistentState>, String)
+  public final net.corda.core.node.services.vault.SortAttribute$Custom copy(Class<? extends net.corda.core.schemas.StatePersistable>, String)
   public boolean equals(Object)
   @NotNull
-  public final Class<? extends net.corda.core.schemas.PersistentState> getEntityStateClass()
+  public final Class<? extends net.corda.core.schemas.StatePersistable> getEntityStateClass()
   @NotNull
   public final String getEntityStateColumnName()
   public int hashCode()

core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt

@@ -8,7 +8,7 @@ import net.corda.core.contracts.StateRef
 import net.corda.core.contracts.UniqueIdentifier
 import net.corda.core.identity.AbstractParty
 import net.corda.core.node.services.Vault
-import net.corda.core.schemas.PersistentState
+import net.corda.core.schemas.StatePersistable
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.utilities.OpaqueBytes
 import java.time.Instant
@@ -76,6 +76,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
         open val relevancyStatus: Vault.RelevancyStatus = Vault.RelevancyStatus.ALL
         open val constraintTypes: Set<Vault.ConstraintInfo.Type> = emptySet()
         open val constraints: Set<Vault.ConstraintInfo> = emptySet()
+        open val participants: List<AbstractParty>? = null
         abstract val contractStateTypes: Set<Class<out ContractState>>?
         override fun visit(parser: IQueryCriteriaParser): Collection<Predicate> {
             return parser.parseCriteria(this)
@@ -94,7 +95,8 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
             val timeCondition: TimeCondition? = null,
             override val relevancyStatus: Vault.RelevancyStatus = Vault.RelevancyStatus.ALL,
             override val constraintTypes: Set<Vault.ConstraintInfo.Type> = emptySet(),
-            override val constraints: Set<Vault.ConstraintInfo> = emptySet()
+            override val constraints: Set<Vault.ConstraintInfo> = emptySet(),
+            override val participants: List<AbstractParty>? = null
     ) : CommonQueryCriteria() {
         override fun visit(parser: IQueryCriteriaParser): Collection<Predicate> {
             super.visit(parser)
@@ -124,7 +126,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
      * LinearStateQueryCriteria: provides query by attributes defined in [VaultSchema.VaultLinearState]
      */
     data class LinearStateQueryCriteria @JvmOverloads constructor(
-            val participants: List<AbstractParty>? = null,
+            override val participants: List<AbstractParty>? = null,
             val uuid: List<UUID>? = null,
             val externalId: List<String>? = null,
             override val status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED,
@@ -172,7 +174,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
      * FungibleStateQueryCriteria: provides query by attributes defined in [VaultSchema.VaultFungibleStates]
      */
     data class FungibleStateQueryCriteria(
-            val participants: List<AbstractParty>? = null,
+            override val participants: List<AbstractParty>? = null,
             val quantity: ColumnPredicate<Long>? = null,
             override val status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED,
             override val contractStateTypes: Set<Class<out ContractState>>? = null,
@@ -188,7 +190,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
      * FungibleStateQueryCriteria: provides query by attributes defined in [VaultSchema.VaultFungibleStates]
      */
     data class FungibleAssetQueryCriteria @JvmOverloads constructor(
-            val participants: List<AbstractParty>? = null,
+            override val participants: List<AbstractParty>? = null,
             val owner: List<AbstractParty>? = null,
             val quantity: ColumnPredicate<Long>? = null,
             val issuer: List<AbstractParty>? = null,
@@ -231,7 +233,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
      * Params
      *  [expression] refers to a (composable) type safe [CriteriaExpression]
      */
-    data class VaultCustomQueryCriteria<L : PersistentState> @JvmOverloads constructor(
+    data class VaultCustomQueryCriteria<L : StatePersistable> @JvmOverloads constructor(
             val expression: CriteriaExpression<L, Boolean>,
             override val status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED,
             override val contractStateTypes: Set<Class<out ContractState>>? = null,
@@ -299,7 +301,7 @@ interface IQueryCriteriaParser : BaseQueryCriteriaParser<QueryCriteria, IQueryCr
     fun parseCriteria(criteria: QueryCriteria.CommonQueryCriteria): Collection<Predicate>
     fun parseCriteria(criteria: QueryCriteria.FungibleAssetQueryCriteria): Collection<Predicate>
     fun parseCriteria(criteria: QueryCriteria.LinearStateQueryCriteria): Collection<Predicate>
-    fun <L : PersistentState> parseCriteria(criteria: QueryCriteria.VaultCustomQueryCriteria<L>): Collection<Predicate>
+    fun <L : StatePersistable> parseCriteria(criteria: QueryCriteria.VaultCustomQueryCriteria<L>): Collection<Predicate>
     fun parseCriteria(criteria: QueryCriteria.VaultQueryCriteria): Collection<Predicate>
 }
 

core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteriaUtils.kt

@@ -10,6 +10,7 @@ import net.corda.core.node.services.vault.ColumnPredicate.*
 import net.corda.core.node.services.vault.EqualityComparisonOperator.*
 import net.corda.core.node.services.vault.LikenessOperator.*
 import net.corda.core.schemas.PersistentState
+import net.corda.core.schemas.StatePersistable
 import net.corda.core.serialization.CordaSerializable
 import java.lang.reflect.Field
 import kotlin.jvm.internal.CallableReference
@@ -234,7 +235,7 @@ sealed class SortAttribute {
      * [entityStateColumnName] should reference an entity attribute name as defined by the associated mapped schema
      * (for example, [CashSchemaV1.PersistentCashState::currency.name])
      */
-    data class Custom(val entityStateClass: Class<out PersistentState>,
+    data class Custom(val entityStateClass: Class<out StatePersistable>,
                       val entityStateColumnName: String) : SortAttribute()
 }
 

docs/source/api-persistence.rst

@@ -165,7 +165,7 @@ useful if off-ledger data must be maintained in conjunction with on-ledger state
               as a custom schema. See Samples below.
 
 The code snippet below defines a ``PersistentFoo`` type inside ``FooSchemaV1``. Note that ``PersistentFoo`` is added to
-a list of mapped types which is passed to ``MappedSChema``. This is exactly how state schemas are defined, except that
+a list of mapped types which is passed to ``MappedSchema``. This is exactly how state schemas are defined, except that
 the entity in this case should not subclass ``PersistentState`` (as it is not a state object). See examples:
 
 .. container:: codeset
@@ -174,7 +174,6 @@ the entity in this case should not subclass ``PersistentState`` (as it is not a
 
         public class FooSchema {}
 
-        @CordaSerializable
         public class FooSchemaV1 extends MappedSchema {
             FooSchemaV1() {
                 super(FooSchema.class, 1, ImmutableList.of(PersistentFoo.class));
@@ -209,9 +208,8 @@ Instances of ``PersistentFoo`` can be persisted inside a flow as follows:
     .. sourcecode:: java
 
         PersistentFoo foo = new PersistentFoo(new UniqueIdentifier().getId().toString(), "Bar");
-        node.getServices().withEntityManager(entityManager -> {
+        serviceHub.withEntityManager(entityManager -> {
             entityManager.persist(foo);
-            entityManager.flush();
             return null;
         });
 

docs/source/api-vault-query.rst

@@ -1,3 +1,9 @@
+.. highlight:: kotlin
+.. raw:: html
+
+   <script type="text/javascript" src="_static/jquery.js"></script>
+   <script type="text/javascript" src="_static/codesets.js"></script>
+
 API: Vault Query
 ================
 
@@ -569,4 +575,78 @@ The Corda Tutorials provide examples satisfying these additional Use Cases:
  .. _JPQL: http://docs.jboss.org/hibernate/orm/current/userguide/html_single/Hibernate_User_Guide.html#hql
  .. _JPA: https://docs.spring.io/spring-data/jpa/docs/current/reference/html
 
+Mapping owning keys to external IDs
+-----------------------------------
 
+When creating new public keys via the ``KeyManagementService``, it is possible to create an association between the newly created public
+key and an external ID. This, in effect, allows CorDapp developers to group state ownership/participation keys by an account ID.
+
+.. note:: This only works with freshly generated public keys and *not* the node's legal identity key. If you require that the freshly
+          generated keys be for the node's identity then use ``PersistentKeyManagementService.freshKeyAndCert`` instead of ``freshKey``.
+          Currently, the generation of keys for other identities is not supported.
+
+The code snippet below show how keys can be associated with an external ID by using the exposed JPA functionality:
+
+.. container:: codeset
+
+    .. sourcecode:: java
+
+        public AnonymousParty freshKeyForExternalId(UUID externalId, ServiceHub services) {
+            // Create a fresh key pair and return the public key.
+            AnonymousParty anonymousParty = freshKey();
+            // Associate the fresh key to an external ID.
+            services.withEntityManager(entityManager -> {
+                PersistentKeyManagementService.PublicKeyHashToExternalId mapping = PersistentKeyManagementService.PublicKeyHashToExternalId(externalId, anonymousParty.owningKey);
+                entityManager.persist(mapping);
+                return null;
+            });
+            return anonymousParty;
+        }
+
+    .. sourcecode:: kotlin
+
+        fun freshKeyForExternalId(externalId: UUID, services: ServiceHub): AnonymousParty {
+            // Create a fresh key pair and return the public key.
+            val anonymousParty = freshKey()
+            // Associate the fresh key to an external ID.
+            services.withEntityManager {
+                val mapping = PersistentKeyManagementService.PublicKeyHashToExternalId(externalId, anonymousParty.owningKey)
+                persist(mapping)
+            }
+            return anonymousParty
+        }
+
+As can be seen in the code snippet above, the ``PublicKeyHashToExternalId`` entity has been added to ``PersistentKeyManagementService``,
+which allows you to associate your public keys with external IDs. So far, so good.
+
+.. note:: Here, it is worth noting that we must map **owning keys** to external IDs, as opposed to **state objects**. This is because it
+          might be the case that a ``LinearState`` is owned by two public keys generated by the same node.
+
+The intuition here is that when these public keys are used to own or participate in a state object, it is trivial to then associate those
+states with a particular external ID. Behind the scenes, when states are persisted to the vault, the owning keys for each state are
+persisted to a ``PersistentParty`` table. The ``PersistentParty`` table can be joined with the ``PublicKeyHashToExternalId`` table to create
+a view which maps each state to one or more external IDs. The entity relationship diagram below helps to explain how this works.
+
+.. image:: resources/state-to-external-id.png
+
+When performing a vault query, it is now possible to query for states by external ID using a custom query criteria.
+
+.. container:: codeset
+
+   .. sourcecode:: java
+
+        UUID id = someExternalId;
+        FieldInfo externalIdField = getField("externalId", VaultSchemaV1.StateToExternalId.class);
+        CriteriaExpression externalId = Builder.equal(externalIdField, id);
+        QueryCriteria query = new VaultCustomQueryCriteria(externalId);
+        Vault.Page<StateType> results = vaultService.queryBy(StateType.class, query);
+
+   .. sourcecode:: kotlin
+
+        val id: UUID = someExternalId
+        val externalId = builder { VaultSchemaV1.StateToExternalId::externalId.equal(id) }
+        val queryCriteria = QueryCriteria.VaultCustomQueryCriteria(externalId)
+        val results = vaultService.queryBy<StateType>(queryCriteria).states
+
+The ``VaultCustomQueryCriteria`` can also be combined with other query criteria, like custom schemas, for instance. See the vault query API
+examples above for how to combine ``QueryCriteria``.

node/src/main/kotlin/net/corda/node/services/keys/PersistentKeyManagementService.kt

@@ -14,10 +14,8 @@ import org.bouncycastle.operator.ContentSigner
 import java.security.KeyPair
 import java.security.PrivateKey
 import java.security.PublicKey
-import javax.persistence.Column
+import java.util.*
-import javax.persistence.Entity
+import javax.persistence.*
-import javax.persistence.Id
-import javax.persistence.Lob
 
 /**
  * A persistent re-implementation of [E2ETestKeyManagementService] to support node re-start.
@@ -29,7 +27,7 @@ import javax.persistence.Lob
 class PersistentKeyManagementService(cacheFactory: NamedCacheFactory, val identityService: PersistentIdentityService,
                                      private val database: CordaPersistence) : SingletonSerializeAsToken(), KeyManagementServiceInternal {
     @Entity
-    @javax.persistence.Table(name = "${NODE_DATABASE_PREFIX}our_key_pairs")
+    @Table(name = "${NODE_DATABASE_PREFIX}our_key_pairs")
     class PersistentKey(
             @Id
             @Column(name = "public_key_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
@@ -46,6 +44,24 @@ class PersistentKeyManagementService(cacheFactory: NamedCacheFactory, val identi
             : this(publicKey.toStringShort(), publicKey.encoded, privateKey.encoded)
     }
 
+    @Entity
+    @Table(name = "pk_hash_to_ext_id_map", indexes = [Index(name = "pk_hash_to_xid_idx", columnList = "public_key_hash")])
+    class PublicKeyHashToExternalId(
+            @Id
+            @GeneratedValue
+            @Column(name = "id", unique = true, nullable = false)
+            var key: Long? = null,
+
+            @Column(name = "external_id", nullable = false)
+            var externalId: UUID,
+
+            @Column(name = "public_key_hash", nullable = false)
+            var publicKeyHash: String
+    ) {
+        constructor(accountId: UUID, publicKey: PublicKey)
+                : this(null, accountId, publicKey.toStringShort())
+    }
+
     private companion object {
         fun createKeyMap(cacheFactory: NamedCacheFactory): AppendOnlyPersistentMap<PublicKey, PrivateKey, PersistentKey, String> {
             return AppendOnlyPersistentMap(

node/src/main/kotlin/net/corda/node/services/schema/NodeSchemaService.kt

@@ -42,7 +42,8 @@ class NodeSchemaService(private val extraSchemas: Set<MappedSchema> = emptySet()
                     P2PMessageDeduplicator.ProcessedMessage::class.java,
                     PersistentIdentityService.PersistentIdentity::class.java,
                     PersistentIdentityService.PersistentIdentityNames::class.java,
-                    ContractUpgradeServiceImpl.DBContractUpgrade::class.java
+                    ContractUpgradeServiceImpl.DBContractUpgrade::class.java,
+                    PersistentKeyManagementService.PublicKeyHashToExternalId::class.java
             )) {
         override val migrationResource = "node-core.changelog-master"
     }
@@ -77,17 +78,11 @@ class NodeSchemaService(private val extraSchemas: Set<MappedSchema> = emptySet()
     // Because schema is always one supported by the state, just delegate.
     override fun generateMappedObject(state: ContractState, schema: MappedSchema): PersistentState {
         if ((schema === VaultSchemaV1) && (state is LinearState))
-            return VaultSchemaV1.VaultLinearStates(state.linearId, state.participants)
+            return VaultSchemaV1.VaultLinearStates(state.linearId)
         if ((schema === VaultSchemaV1) && (state is FungibleAsset<*>))
-            return VaultSchemaV1.VaultFungibleStates(state.owner, state.amount.quantity, state.amount.token.issuer.party, state.amount.token.issuer.reference, state.participants)
+            return VaultSchemaV1.VaultFungibleStates(state.owner, state.amount.quantity, state.amount.token.issuer.party, state.amount.token.issuer.reference)
         if ((schema === VaultSchemaV1) && (state is FungibleState<*>))
-            return VaultSchemaV1.VaultFungibleStates(
+            return VaultSchemaV1.VaultFungibleStates(owner = null, quantity = state.amount.quantity, issuer = null, issuerRef = null)
-                    participants = state.participants.toMutableSet(),
-                    owner = null,
-                    quantity = state.amount.quantity,
-                    issuer = null,
-                    issuerRef = null
-            )
         return (state as QueryableState).generateMappedObject(schema)
     }
 

node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt

@@ -17,6 +17,7 @@ import net.corda.core.node.services.vault.NullOperator.NOT_NULL
 import net.corda.core.node.services.vault.QueryCriteria.CommonQueryCriteria
 import net.corda.core.schemas.PersistentState
 import net.corda.core.schemas.PersistentStateRef
+import net.corda.core.schemas.StatePersistable
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.contextLogger
 import net.corda.core.utilities.trace
@@ -222,7 +223,7 @@ class HibernateQueryCriteriaParser(val contractStateType: Class<out ContractStat
     }
 
     // incrementally build list of root entities (for later use in Sort parsing)
-    private val rootEntities = mutableMapOf<Class<out PersistentState>, Root<*>>(Pair(VaultSchemaV1.VaultStates::class.java, vaultStates))
+    private val rootEntities = mutableMapOf<Class<out StatePersistable>, Root<*>>(Pair(VaultSchemaV1.VaultStates::class.java, vaultStates))
     private val aggregateExpressions = mutableListOf<Expression<*>>()
     private val commonPredicates = mutableMapOf<Pair<String, Operator>, Predicate>()   // schema attribute Name, operator -> predicate
     private val constraintPredicates = mutableSetOf<Predicate>()
@@ -412,13 +413,25 @@ class HibernateQueryCriteriaParser(val contractStateType: Class<out ContractStat
             predicateSet.add(criteriaBuilder.and(vaultFungibleStates.get<ByteArray>("issuerRef").`in`(issuerRefs)))
         }
 
-        // participants
+        // Participants.
         criteria.participants?.let {
-            val participants = criteria.participants as List<AbstractParty>
+            val participants = criteria.participants!!
-            val joinLinearStateToParty = vaultFungibleStates.joinSet<VaultSchemaV1.VaultFungibleStates, AbstractParty>("participants")
+
-            predicateSet.add(criteriaBuilder.and(joinLinearStateToParty.`in`(participants)))
+            // Get the persistent party entity.
-            criteriaQuery.distinct(true)
+            val persistentPartyEntity = VaultSchemaV1.PersistentParty::class.java
+            val entityRoot = rootEntities.getOrElse(persistentPartyEntity) {
+                val entityRoot = criteriaQuery.from(persistentPartyEntity)
+                rootEntities[persistentPartyEntity] = entityRoot
+                entityRoot
+            }
+
+            // Add the join and participants predicates.
+            val statePartyJoin = criteriaBuilder.equal(vaultStates.get<VaultSchemaV1.VaultStates>("stateRef"), entityRoot.get<VaultSchemaV1.PersistentParty>("stateRef"))
+            val participantsPredicate = criteriaBuilder.and(entityRoot.get<VaultSchemaV1.PersistentParty>("x500Name").`in`(participants))
+            predicateSet.add(statePartyJoin)
+            predicateSet.add(participantsPredicate)
         }
+
         return predicateSet
     }
 
@@ -452,17 +465,29 @@ class HibernateQueryCriteriaParser(val contractStateType: Class<out ContractStat
                 predicateSet.add(criteriaBuilder.and(vaultLinearStates.get<String>("externalId").`in`(externalIds)))
         }
 
-        // deal participants
+        // Participants.
         criteria.participants?.let {
-            val participants = criteria.participants as List<AbstractParty>
+            val participants = criteria.participants!!
-            val joinLinearStateToParty = vaultLinearStates.joinSet<VaultSchemaV1.VaultLinearStates, AbstractParty>("participants")
+
-            predicateSet.add(criteriaBuilder.and(joinLinearStateToParty.`in`(participants)))
+            // Get the persistent party entity.
-            criteriaQuery.distinct(true)
+            val persistentPartyEntity = VaultSchemaV1.PersistentParty::class.java
+            val entityRoot = rootEntities.getOrElse(persistentPartyEntity) {
+                val entityRoot = criteriaQuery.from(persistentPartyEntity)
+                rootEntities[persistentPartyEntity] = entityRoot
+                entityRoot
+            }
+
+            // Add the join and participants predicates.
+            val statePartyJoin = criteriaBuilder.equal(vaultStates.get<VaultSchemaV1.VaultStates>("stateRef"), entityRoot.get<VaultSchemaV1.PersistentParty>("stateRef"))
+            val participantsPredicate = criteriaBuilder.and(entityRoot.get<VaultSchemaV1.PersistentParty>("x500Name").`in`(participants))
+            predicateSet.add(statePartyJoin)
+            predicateSet.add(participantsPredicate)
         }
+
         return predicateSet
     }
 
-    override fun <L : PersistentState> parseCriteria(criteria: QueryCriteria.VaultCustomQueryCriteria<L>): Collection<Predicate> {
+    override fun <L : StatePersistable> parseCriteria(criteria: QueryCriteria.VaultCustomQueryCriteria<L>): Collection<Predicate> {
         log.trace { "Parsing VaultCustomQueryCriteria: $criteria" }
 
         val predicateSet = mutableSetOf<Predicate>()

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -132,8 +132,15 @@ class NodeVaultService(
                 //
                 // Adding a new column in the "VaultStates" table was considered the best approach.
                 val keys = stateOnly.participants.map { it.owningKey }
+                val persistentStateRef = PersistentStateRef(stateAndRef.key)
                 val isRelevant = isRelevant(stateOnly, keyManagementService.filterMyKeys(keys).toSet())
                 val constraintInfo = Vault.ConstraintInfo(stateAndRef.value.state.constraint)
+                // Save a row for each party in the state_party table.
+                // TODO: Perhaps these can be stored in a batch?
+                stateOnly.participants.forEach { participant ->
+                    val persistentParty = VaultSchemaV1.PersistentParty(persistentStateRef, participant)
+                    session.save(persistentParty)
+                }
                 val stateToAdd = VaultSchemaV1.VaultStates(
                         notary = stateAndRef.value.state.notary,
                         contractStateClassName = stateAndRef.value.state.data.javaClass.name,
@@ -143,7 +150,7 @@ class NodeVaultService(
                         constraintType = constraintInfo.type(),
                         constraintData = constraintInfo.data()
                 )
-                stateToAdd.stateRef = PersistentStateRef(stateAndRef.key)
+                stateToAdd.stateRef = persistentStateRef
                 session.save(stateToAdd)
             }
             consumedStateRefs.forEach { stateRef ->

node/src/main/kotlin/net/corda/node/services/vault/VaultSchema.kt

@@ -3,14 +3,18 @@ package net.corda.node.services.vault
 import net.corda.core.contracts.ContractState
 import net.corda.core.contracts.MAX_ISSUER_REF_SIZE
 import net.corda.core.contracts.UniqueIdentifier
+import net.corda.core.crypto.toStringShort
 import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.Party
 import net.corda.core.node.services.MAX_CONSTRAINT_DATA_SIZE
 import net.corda.core.node.services.Vault
 import net.corda.core.schemas.MappedSchema
 import net.corda.core.schemas.PersistentState
+import net.corda.core.schemas.PersistentStateRef
+import net.corda.core.schemas.StatePersistable
 import net.corda.core.serialization.CordaSerializable
 import net.corda.core.utilities.OpaqueBytes
+import org.hibernate.annotations.Immutable
 import org.hibernate.annotations.Type
 import java.time.Instant
 import java.util.*
@@ -25,8 +29,18 @@ object VaultSchema
  * First version of the Vault ORM schema
  */
 @CordaSerializable
-object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, version = 1,
+object VaultSchemaV1 : MappedSchema(
-        mappedTypes = listOf(VaultStates::class.java, VaultLinearStates::class.java, VaultFungibleStates::class.java, VaultTxnNote::class.java)) {
+        schemaFamily = VaultSchema.javaClass,
+        version = 1,
+        mappedTypes = listOf(
+                VaultStates::class.java,
+                VaultLinearStates::class.java,
+                VaultFungibleStates::class.java,
+                VaultTxnNote::class.java,
+                PersistentParty::class.java,
+                StateToExternalId::class.java
+        )
+) {
 
     override val migrationResource = "vault-schema.changelog-master"
 
@@ -84,16 +98,6 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio
     class VaultLinearStates(
             /** [ContractState] attributes */
 
-            /** X500Name of participant parties **/
-            @ElementCollection
-            @CollectionTable(name = "vault_linear_states_parts",
-                    joinColumns = [(JoinColumn(name = "output_index", referencedColumnName = "output_index")), (JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))],
-                    foreignKey = ForeignKey(name = "FK__lin_stat_parts__lin_stat"))
-            @Column(name = "participants")
-            var participants: MutableSet<AbstractParty?>? = null,
-            // Reason for not using Set is described here:
-            // https://stackoverflow.com/questions/44213074/kotlin-collection-has-neither-generic-type-or-onetomany-targetentity
-
             /**
              *  Represents a [LinearState] [UniqueIdentifier]
              */
@@ -104,25 +108,12 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio
             @Type(type = "uuid-char")
             var uuid: UUID
     ) : PersistentState() {
-        constructor(uid: UniqueIdentifier, _participants: List<AbstractParty>) :
+        constructor(uid: UniqueIdentifier) : this(externalId = uid.externalId, uuid = uid.id)
-                this(externalId = uid.externalId,
-                        uuid = uid.id,
-                        participants = _participants.toMutableSet())
     }
 
     @Entity
     @Table(name = "vault_fungible_states")
     class VaultFungibleStates(
-            /** [ContractState] attributes */
-
-            /** X500Name of participant parties **/
-            @ElementCollection
-            @CollectionTable(name = "vault_fungible_states_parts",
-                    joinColumns = [(JoinColumn(name = "output_index", referencedColumnName = "output_index")), (JoinColumn(name = "transaction_id", referencedColumnName = "transaction_id"))],
-                    foreignKey = ForeignKey(name = "FK__fung_st_parts__fung_st"))
-            @Column(name = "participants", nullable = true)
-            var participants: MutableSet<AbstractParty>? = null,
-
             /** [OwnableState] attributes */
 
             /** X500Name of owner party **/
@@ -149,12 +140,8 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio
             @Type(type = "corda-wrapper-binary")
             var issuerRef: ByteArray?
     ) : PersistentState() {
-        constructor(_owner: AbstractParty, _quantity: Long, _issuerParty: AbstractParty, _issuerRef: OpaqueBytes, _participants: List<AbstractParty>) :
+        constructor(_owner: AbstractParty, _quantity: Long, _issuerParty: AbstractParty, _issuerRef: OpaqueBytes) :
-                this(owner = _owner,
+                this(owner = _owner, quantity = _quantity, issuer = _issuerParty, issuerRef = _issuerRef.bytes)
-                        quantity = _quantity,
-                        issuer = _issuerParty,
-                        issuerRef = _issuerRef.bytes,
-                        participants = _participants.toMutableSet())
     }
 
     @Entity
@@ -173,4 +160,47 @@ object VaultSchemaV1 : MappedSchema(schemaFamily = VaultSchema.javaClass, versio
     ) {
         constructor(txId: String, note: String) : this(0, txId, note)
     }
-}
+
+    @Entity
+    @Table(name = "state_party", indexes = [Index(name = "state_party_idx", columnList = "public_key_hash")])
+    class PersistentParty(
+            @Id
+            @GeneratedValue
+            @Column(name = "id", unique = true, nullable = false)
+            var id: Long? = null,
+
+            // Foreign key.
+            @Column(name = "state_ref")
+            var stateRef: PersistentStateRef,
+
+            @Column(name = "public_key_hash", nullable = false)
+            var publicKeyHash: String,
+
+            @Column(name = "x500_name", nullable = true)
+            var x500Name: AbstractParty? = null
+    ) : StatePersistable {
+        constructor(stateRef: PersistentStateRef, abstractParty: AbstractParty)
+                : this(null, stateRef, abstractParty.owningKey.toStringShort(), abstractParty)
+    }
+
+    @Entity
+    @Immutable
+    @Table(name = "v_pkey_hash_ex_id_map")
+    class StateToExternalId(
+            @Id
+            @GeneratedValue
+            @Column(name = "id", unique = true, nullable = false)
+            var id: Long? = null,
+
+            // Foreign key.
+            @Column(name = "state_ref")
+            var stateRef: PersistentStateRef,
+
+            @Column(name = "public_key_hash")
+            var publicKeyHash: String,
+
+            @Column(name = "external_id")
+            var externalId: UUID
+    ) : StatePersistable
+}
+

node/src/main/resources/migration/vault-schema.changelog-master.xml

@@ -11,4 +11,5 @@
     <include file="migration/vault-schema.changelog-v5.xml"/>
     <include file="migration/vault-schema.changelog-v6.xml"/>
     <include file="migration/vault-schema.changelog-v7.xml"/>
+    <include file="migration/vault-schema.changelog-v8.xml"/>
 </databaseChangeLog>

node/src/main/resources/migration/vault-schema.changelog-v8.xml

@@ -0,0 +1,36 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+    <changeSet author="R3.Corda" id="create-external-id-to-state-party-view">
+        <createTable tableName="state_party">
+            <column name="output_index" type="INT"/>
+            <column name="transaction_id" type="NVARCHAR(64)"/>
+            <column name="id" type="INT"/>
+            <column name="public_key_hash" type="NVARCHAR(255)"/>
+            <column name="x500_name" type="NVARCHAR(255)"/>
+        </createTable>
+        <createIndex indexName="state_pk_hash_idx" tableName="state_party">
+            <column name="public_key_hash"/>
+        </createIndex>
+        <createTable tableName="pk_hash_to_ext_id_map">
+            <column name="id" type="INT"/>
+            <column name="external_id" type="NVARCHAR(255)"/>
+            <column name="public_key_hash" type="NVARCHAR(255)"/>
+        </createTable>
+        <createIndex indexName="pk_hash_to_xid_idx" tableName="pk_hash_to_ext_id_map">
+            <column name="public_key_hash"/>
+        </createIndex>
+        <createView viewName="v_pkey_hash_ex_id_map">
+            select
+                state_party.id,
+                state_party.public_key_hash,
+                state_party.transaction_id,
+                state_party.output_index,
+                pk_hash_to_ext_id_map.external_id
+            from state_party
+            join pk_hash_to_ext_id_map
+            on state_party.public_key_hash = pk_hash_to_ext_id_map.public_key_hash
+        </createView>
+    </changeSet>
+</databaseChangeLog>

node/src/test/kotlin/net/corda/node/services/vault/ExternalIdMappingTest.kt

@@ -0,0 +1,131 @@
+package net.corda.node.services.vault
+
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
+import net.corda.core.identity.AbstractParty
+import net.corda.core.identity.AnonymousParty
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.node.services.queryBy
+import net.corda.core.node.services.vault.QueryCriteria
+import net.corda.core.node.services.vault.builder
+import net.corda.core.transactions.TransactionBuilder
+import net.corda.node.services.api.IdentityServiceInternal
+import net.corda.node.services.keys.PersistentKeyManagementService
+import net.corda.nodeapi.internal.persistence.CordaPersistence
+import net.corda.testing.common.internal.testNetworkParameters
+import net.corda.testing.contracts.DummyContract
+import net.corda.testing.contracts.DummyState
+import net.corda.testing.core.SerializationEnvironmentRule
+import net.corda.testing.core.TestIdentity
+import net.corda.testing.internal.rigorousMock
+import net.corda.testing.node.MockServices
+import org.junit.Rule
+import org.junit.Test
+import java.util.*
+import kotlin.test.assertEquals
+
+class ExternalIdMappingTest {
+
+    @Rule
+    @JvmField
+    val testSerialization = SerializationEnvironmentRule()
+
+    private val cordapps = listOf(
+            "net.corda.node.services.persistence",
+            "net.corda.testing.contracts"
+    )
+
+    private val myself = TestIdentity(CordaX500Name("Me", "London", "GB"))
+    private val notary = TestIdentity(CordaX500Name("NotaryService", "London", "GB"), 1337L)
+    private val databaseAndServices = MockServices.makeTestDatabaseAndMockServices(
+            cordappPackages = cordapps,
+            identityService = rigorousMock<IdentityServiceInternal>().also {
+                doReturn(notary.party).whenever(it).partyFromKey(notary.publicKey)
+                doReturn(notary.party).whenever(it).wellKnownPartyFromAnonymous(notary.party)
+                doReturn(notary.party).whenever(it).wellKnownPartyFromX500Name(notary.name)
+            },
+            initialIdentity = myself,
+            networkParameters = testNetworkParameters(minimumPlatformVersion = 4)
+    )
+
+    private val services: MockServices = databaseAndServices.second
+    private val database: CordaPersistence = databaseAndServices.first
+
+    private fun freshKeyForExternalId(externalId: UUID): AnonymousParty {
+        val anonymousParty = freshKey()
+        database.transaction {
+            services.withEntityManager {
+                val mapping = PersistentKeyManagementService.PublicKeyHashToExternalId(externalId, anonymousParty.owningKey)
+                persist(mapping)
+                flush()
+            }
+        }
+        return anonymousParty
+    }
+
+    private fun freshKey(): AnonymousParty {
+        val key = services.keyManagementService.freshKey()
+        val anonymousParty = AnonymousParty(key)
+        // Add behaviour to the mock identity management service for dealing with the new key.
+        // It won't be able to resolve it as it's just an anonymous key that is not linked to an identity.
+        services.identityService.also { doReturn(null).whenever(it).wellKnownPartyFromAnonymous(anonymousParty) }
+        return anonymousParty
+    }
+
+    private fun createDummyState(participants: List<AbstractParty>): DummyState {
+        val tx = TransactionBuilder(notary = notary.party).apply {
+            addOutputState(DummyState(1, participants), DummyContract.PROGRAM_ID)
+            addCommand(DummyContract.Commands.Create(), participants.map { it.owningKey })
+        }
+        val stx = services.signInitialTransaction(tx)
+        database.transaction { services.recordTransactions(stx) }
+        return stx.tx.outputsOfType<DummyState>().single()
+    }
+
+    @Test
+    fun `Two states can be mapped to a single externalId`() {
+        val vaultService = services.vaultService
+        // Create new external ID and two keys mapped to it.
+        val id = UUID.randomUUID()
+        val keyOne = freshKeyForExternalId(id)
+        val keyTwo = freshKeyForExternalId(id)
+        // Create states with a public key assigned to the new external ID.
+        val dummyStateOne = createDummyState(listOf(keyOne))
+        val dummyStateTwo = createDummyState(listOf(keyTwo))
+        // This query should return two states!
+        val result = database.transaction {
+            val externalId = builder { VaultSchemaV1.StateToExternalId::externalId.`in`(listOf(id)) }
+            val queryCriteria = QueryCriteria.VaultCustomQueryCriteria(externalId)
+            vaultService.queryBy<DummyState>(queryCriteria).states
+        }
+        assertEquals(setOf(dummyStateOne, dummyStateTwo), result.map { it.state.data }.toSet())
+
+        // This query should return two states!
+        val resultTwo = database.transaction {
+            val externalId = builder { VaultSchemaV1.StateToExternalId::externalId.equal(id) }
+            val queryCriteria = QueryCriteria.VaultCustomQueryCriteria(externalId)
+            vaultService.queryBy<DummyState>(queryCriteria).states
+        }
+        assertEquals(setOf(dummyStateOne, dummyStateTwo), resultTwo.map { it.state.data }.toSet())
+    }
+
+    @Test
+    fun `One state can be mapped to multiple externalIds`() {
+        val vaultService = services.vaultService
+        // Create new external ID.
+        val idOne = UUID.randomUUID()
+        val keyOne = freshKeyForExternalId(idOne)
+        val idTwo = UUID.randomUUID()
+        val keyTwo = freshKeyForExternalId(idTwo)
+        // Create state with a public key assigned to the new external ID.
+        val dummyState = createDummyState(listOf(keyOne, keyTwo))
+        // This query should return one state!
+        val result = database.transaction {
+            val externalId = builder { VaultSchemaV1.StateToExternalId::externalId.`in`(listOf(idOne, idTwo)) }
+            val queryCriteria = QueryCriteria.VaultCustomQueryCriteria(externalId)
+            vaultService.queryBy<DummyState>(queryCriteria).states
+        }
+        assertEquals(dummyState, result.single().state.data)
+    }
+
+}