CORDA-2432 - Update documentation for filtering by state relevancy in vault queries (#4597)

* [CORDA-2432] Update documentation for filtering by state relevancy in vault queries

* Add missing file to fix

docs/source/api-vault-query.rst

@@ -86,7 +86,7 @@ There are four implementations of this interface which can be chained together t
 
 1. ``VaultQueryCriteria`` provides filterable criteria on attributes within the Vault states table: status (UNCONSUMED,
    CONSUMED), state reference(s), contract state type(s), notaries, soft locked states, timestamps (RECORDED, CONSUMED),
-   state constraints (see :ref:`Constraint Types <implicit_constraint_types>`).
+   state constraints (see :ref:`Constraint Types <implicit_constraint_types>`), relevancy (ALL, RELEVANT, NON_RELEVANT).
 
 	.. note:: Sensible defaults are defined for frequently used attributes (status = UNCONSUMED, always include soft
 	   locked states).
@@ -330,6 +330,14 @@ pages available:
     :end-before: DOCEND VaultQueryExample24
     :dedent: 8
 
+Query for only relevant states in the vault:
+
+.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt
+    :language: kotlin
+    :start-after: DOCSTART VaultQueryExample25
+    :end-before: DOCEND VaultQueryExample25
+    :dedent: 8
+
 **LinearState and DealState queries using** ``LinearStateQueryCriteria``:
 
 Query for unconsumed linear states for given linear ids:
@@ -364,6 +372,14 @@ Query for unconsumed deal states with deals parties:
     :end-before: DOCEND VaultQueryExample11
     :dedent: 12
 
+Query for only relevant linear states in the vault:
+
+.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt
+    :language: kotlin
+    :start-after: DOCSTART VaultQueryExample26
+    :end-before: DOCEND VaultQueryExample26
+    :dedent: 8
+
 **FungibleAsset and DealState queries using** ``FungibleAssetQueryCriteria``:
 
 Query for fungible assets for a given currency:
@@ -392,6 +408,14 @@ Query for fungible assets for a specific issuer party:
     :end-before: DOCEND VaultQueryExample14
     :dedent: 12
 
+Query for only relevant fungible states in the vault:
+
+.. literalinclude:: ../../node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt
+    :language: kotlin
+    :start-after: DOCSTART VaultQueryExample27
+    :end-before: DOCEND VaultQueryExample27
+    :dedent: 12
+
 **Aggregate Function queries using** ``VaultCustomQueryCriteria``:
 
 .. note:: Query results for aggregate functions are contained in the ``otherResults`` attribute of a results Page.

docs/source/app-upgrade-notes.rst

@@ -333,6 +333,15 @@ into shared business logic, but it makes perfect sense to put into a user-specif
 
 If your flows could benefit from being extended in this way, read ":doc:`flow-overriding`" to learn more.
 
+Step 10. Possibly update Vault state queries
+--------------------------------------------
+
+Queries made on a node's vault can filter by the relevancy of those states to the node in Corda 4. As this functionality does not exist in
+Corda 3, apps targeting that release will continue to receive all states in any vault queries. In Corda 4, the default is to return all
+states in the vault, to maintain backwards compatibility. However, it may make sense to migrate queries expecting just those states relevant
+to the node in question to query for only relevant states. See :doc:`api-vault-query.rst` for more details on how to do this. Not doing this
+may result in queries returning more states than expected if the node is using Observer node functionality (see ":doc:`tutorial-observer-nodes.rst`").
+
 Step 10. Explore other new features that may be useful
 ------------------------------------------------------
 

docs/source/changelog.rst

@@ -320,6 +320,9 @@ Version 4.0
 
 * Logging for P2P and RPC has been separated, to make it easier to enable all P2P or RPC logging without hand-picking loggers for individual classes.
 
+* Vault Query Criteria have been enhanced to allow filtering by state relevancy. Queries can request all states, just relevant ones, or just non relevant ones. The default is to return all states, to maintain backwards compatibility.
+  Note that this means apps running on nodes using Observer node functionality should update their queries to request only relevant states if they are only expecting to see states in which they participate.
+
 Version 3.3
 -----------
 

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -16,7 +16,6 @@ import net.corda.core.schemas.PersistentStateRef
 import net.corda.core.serialization.SingletonSerializeAsToken
 import net.corda.core.transactions.*
 import net.corda.core.utilities.*
-import net.corda.node.cordapp.CordappLoader
 import net.corda.node.services.api.SchemaService
 import net.corda.node.services.api.VaultServiceInternal
 import net.corda.node.services.schema.PersistentStateService

node/src/test/kotlin/net/corda/node/services/vault/NodeVaultServiceTest.kt

@@ -7,6 +7,7 @@ import net.corda.core.crypto.NullKeys
 import net.corda.core.crypto.generateKeyPair
 import net.corda.core.identity.*
 import net.corda.core.internal.NotaryChangeTransactionBuilder
+import net.corda.core.internal.cordapp.CordappResolver
 import net.corda.core.internal.packageName
 import net.corda.core.node.NotaryInfo
 import net.corda.core.node.StatesToRecord
@@ -681,7 +682,7 @@ class NodeVaultServiceTest {
     fun observerMode() {
         fun countCash(): Long {
             return database.transaction {
-                vaultService.queryBy(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(), PageSpecification(1)).totalStatesAvailable
+                vaultService.queryBy(Cash.State::class.java, QueryCriteria.VaultQueryCriteria(relevancyStatus = Vault.RelevancyStatus.ALL), PageSpecification(1)).totalStatesAvailable
             }
         }
         val currentCashStates = countCash()
@@ -775,7 +776,7 @@ class NodeVaultServiceTest {
         services.recordTransactions(StatesToRecord.NONE, listOf(createTx(7, bankOfCorda.party)))
 
         // Test one.
-        // RelevancyStatus is RELEVANT by default. This should return two states.
+        // RelevancyStatus is ALL by default. This should return five states.
         val resultOne = vaultService.queryBy<DummyState>().states.getNumbers()
         assertEquals(setOf(1, 3, 4, 5, 6), resultOne)
 
@@ -786,7 +787,7 @@ class NodeVaultServiceTest {
         assertEquals(setOf(4, 5), resultTwo)
 
         // Test three.
-        // RelevancyStatus set to ALL.
+        // RelevancyStatus set to RELEVANT.
         val criteriaThree = VaultQueryCriteria(relevancyStatus = Vault.RelevancyStatus.RELEVANT)
         val resultThree = vaultService.queryBy<DummyState>(criteriaThree).states.getNumbers()
         assertEquals(setOf(1, 3, 6), resultThree)
@@ -855,6 +856,35 @@ class NodeVaultServiceTest {
         })
     }
 
+    @Test
+    fun `V3 vault queries return all states by default`() {
+        fun createTx(number: Int, vararg participants: Party): SignedTransaction {
+            return services.signInitialTransaction(TransactionBuilder(DUMMY_NOTARY).apply {
+                addOutputState(DummyState(number, participants.toList()), DummyContract.PROGRAM_ID)
+                addCommand(DummyCommandData, listOf(megaCorp.publicKey))
+            })
+        }
+
+        fun List<StateAndRef<DummyState>>.getNumbers() = map { it.state.data.magicNumber }.toSet()
+
+        CordappResolver.withCordapp(targetPlatformVersion = 3) {
+            services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(1, megaCorp.party)))
+            services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(2, miniCorp.party)))
+            services.recordTransactions(StatesToRecord.ONLY_RELEVANT, listOf(createTx(3, miniCorp.party, megaCorp.party)))
+            services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(4, miniCorp.party)))
+            services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(5, bankOfCorda.party)))
+            services.recordTransactions(StatesToRecord.ALL_VISIBLE, listOf(createTx(6, megaCorp.party, bankOfCorda.party)))
+            services.recordTransactions(StatesToRecord.NONE, listOf(createTx(7, bankOfCorda.party)))
+
+            // Test one.
+            // RelevancyStatus is ALL by default. This should return five states.
+            val resultOne = vaultService.queryBy<DummyState>().states.getNumbers()
+            assertEquals(setOf(1, 3, 4, 5, 6), resultOne)
+        }
+
+        // We should never see 2 or 7.
+    }
+
     @Test
     @Ignore
     fun `trackByCriteria filters updates and snapshots`() {

node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt

@@ -7,6 +7,7 @@ import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.internal.packageName
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.*
 import net.corda.core.node.services.Vault.ConstraintInfo.Type.*
 import net.corda.core.node.services.vault.*
@@ -831,6 +832,45 @@ abstract class VaultQueryTestsBase : VaultQueryParties {
         }
     }
 
+    @Test
+    fun `state relevancy queries`() {
+        database.transaction {
+            vaultFiller.fillWithSomeTestDeals(listOf("123", "456", "789"), includeMe = true)
+            vaultFiller.fillWithSomeTestDeals(listOf("ABC", "DEF", "GHI"), includeMe = false)
+            vaultFillerCashNotary.fillWithSomeTestCash(100.DOLLARS, notaryServices, 10, DUMMY_CASH_ISSUER, statesToRecord = StatesToRecord.ALL_VISIBLE)
+            vaultFillerCashNotary.fillWithSomeTestCash(100.DOLLARS, notaryServices, 10, DUMMY_CASH_ISSUER, charlie.party, statesToRecord = StatesToRecord.ALL_VISIBLE)
+            vaultFiller.fillWithSomeTestLinearStates(1, "XYZ", includeMe = true)
+            vaultFiller.fillWithSomeTestLinearStates(2, "JKL", includeMe = false)
+
+            val dealStates = vaultService.queryBy<DummyDealContract.State>().states
+            assertThat(dealStates).hasSize(6)
+
+            //DOCSTART VaultQueryExample25
+            val relevancyAllCriteria = VaultQueryCriteria(relevancyStatus = Vault.RelevancyStatus.RELEVANT)
+            val allDealStateCount = vaultService.queryBy<DummyDealContract.State>(relevancyAllCriteria).states
+            //DOCEND VaultQueryExample25
+            assertThat(allDealStateCount).hasSize(3)
+
+            val cashStates = vaultService.queryBy<Cash.State>().states
+            assertThat(cashStates).hasSize(20)
+
+            //DOCSTART VaultQueryExample27
+            val allCashCriteria = FungibleStateQueryCriteria(relevancyStatus = Vault.RelevancyStatus.RELEVANT)
+            val allCashStates = vaultService.queryBy<Cash.State>(allCashCriteria).states
+            //DOCEND VaultQueryExample27
+            assertThat(allCashStates).hasSize(10)
+
+            val linearStates = vaultService.queryBy<DummyLinearContract.State>().states
+            assertThat(linearStates).hasSize(3)
+
+            //DOCSTART VaultQueryExample26
+            val allLinearStateCriteria = LinearStateQueryCriteria(relevancyStatus = Vault.RelevancyStatus.RELEVANT)
+            val allLinearStates = vaultService.queryBy<DummyLinearContract.State>(allLinearStateCriteria).states
+            //DOCEND VaultQueryExample26
+            assertThat(allLinearStates).hasSize(1)
+        }
+    }
+
     @Test
     fun `logical operator EQUAL`() {
         database.transaction {

testing/test-utils/src/main/kotlin/net/corda/testing/internal/vault/VaultFiller.kt

@@ -7,6 +7,7 @@ import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.AnonymousParty
 import net.corda.core.identity.Party
 import net.corda.core.node.ServiceHub
+import net.corda.core.node.StatesToRecord
 import net.corda.core.node.services.Vault
 import net.corda.core.toFuture
 import net.corda.core.transactions.SignedTransaction
@@ -72,20 +73,23 @@ class VaultFiller @JvmOverloads constructor(
     @JvmOverloads
     fun fillWithSomeTestDeals(dealIds: List<String>,
                               issuerServices: ServiceHub = services,
-                              participants: List<AbstractParty> = emptyList()): Vault<DealState> {
+                              participants: List<AbstractParty> = emptyList(),
+                              includeMe: Boolean = true): Vault<DealState> {
         val myKey: PublicKey = services.myInfo.chooseIdentity().owningKey
         val me = AnonymousParty(myKey)
+        val participantsToUse = if (includeMe) participants.plus(me) else participants
 
         val transactions: List<SignedTransaction> = dealIds.map {
             // Issue a deal state
             val dummyIssue = TransactionBuilder(notary = defaultNotary.party).apply {
-                addOutputState(DummyDealContract.State(ref = it, participants = participants.plus(me)), DUMMY_DEAL_PROGRAM_ID)
+                addOutputState(DummyDealContract.State(ref = it, participants = participantsToUse), DUMMY_DEAL_PROGRAM_ID)
                 addCommand(dummyCommand())
             }
             val stx = issuerServices.signInitialTransaction(dummyIssue)
             return@map services.addSignature(stx, defaultNotary.publicKey)
         }
-        services.recordTransactions(transactions)
+        val statesToRecord = if (includeMe) StatesToRecord.ONLY_RELEVANT else StatesToRecord.ALL_VISIBLE
+        services.recordTransactions(statesToRecord, transactions)
         // Get all the StateAndRefs of all the generated transactions.
         val states = transactions.flatMap { stx ->
             stx.tx.outputs.indices.map { i -> stx.tx.outRef<DealState>(i) }
@@ -103,17 +107,19 @@ class VaultFiller @JvmOverloads constructor(
                                      linearNumber: Long = 0L,
                                      linearBoolean: Boolean = false,
                                      linearTimestamp: Instant = now(),
-                                     constraint: AttachmentConstraint = AutomaticPlaceholderConstraint): Vault<LinearState> {
+                                     constraint: AttachmentConstraint = AutomaticPlaceholderConstraint,
+                                     includeMe: Boolean = true): Vault<LinearState> {
         val myKey: PublicKey = services.myInfo.chooseIdentity().owningKey
         val me = AnonymousParty(myKey)
         val issuerKey = defaultNotary.keyPair
         val signatureMetadata = SignatureMetadata(services.myInfo.platformVersion, Crypto.findSignatureScheme(issuerKey.public).schemeNumberID)
+        val participantsToUse = if (includeMe) participants.plus(me) else participants
         val transactions: List<SignedTransaction> = (1..numberToCreate).map {
             // Issue a Linear state
             val dummyIssue = TransactionBuilder(notary = defaultNotary.party).apply {
                 addOutputState(DummyLinearContract.State(
                         linearId = uniqueIdentifier ?: UniqueIdentifier(externalId),
-                        participants = participants.plus(me),
+                        participants = participantsToUse,
                         linearString = linearString,
                         linearNumber = linearNumber,
                         linearBoolean = linearBoolean,
@@ -123,7 +129,8 @@ class VaultFiller @JvmOverloads constructor(
             }
             return@map services.signInitialTransaction(dummyIssue).withAdditionalSignature(issuerKey, signatureMetadata)
         }
-        services.recordTransactions(transactions)
+        val statesToRecord = if (includeMe) StatesToRecord.ONLY_RELEVANT else StatesToRecord.ALL_VISIBLE
+        services.recordTransactions(statesToRecord, transactions)
         // Get all the StateAndRefs of all the generated transactions.
         val states = transactions.flatMap { stx ->
             stx.tx.outputs.indices.map { i -> stx.tx.outRef<LinearState>(i) }
@@ -174,7 +181,8 @@ class VaultFiller @JvmOverloads constructor(
                              thisManyStates: Int,
                              issuedBy: PartyAndReference,
                              owner: AbstractParty? = null,
-                             rng: Random? = null) = fillWithSomeTestCash(howMuch, issuerServices, thisManyStates, thisManyStates, issuedBy, owner, rng)
+                             rng: Random? = null,
+                             statesToRecord: StatesToRecord = StatesToRecord.ONLY_RELEVANT) = fillWithSomeTestCash(howMuch, issuerServices, thisManyStates, thisManyStates, issuedBy, owner, rng, statesToRecord)
 
     /**
      * Creates a random set of between (by default) 3 and 10 cash states that add up to the given amount and adds them
@@ -190,7 +198,8 @@ class VaultFiller @JvmOverloads constructor(
                              atMostThisManyStates: Int,
                              issuedBy: PartyAndReference,
                              owner: AbstractParty? = null,
-                             rng: Random? = null): Vault<Cash.State> {
+                             rng: Random? = null,
+                             statesToRecord: StatesToRecord = StatesToRecord.ONLY_RELEVANT): Vault<Cash.State> {
         val amounts = calculateRandomlySizedAmounts(howMuch, atLeastThisManyStates, atMostThisManyStates, rng ?: rngFactory())
         // We will allocate one state to one transaction, for simplicities sake.
         val cash = Cash()
@@ -199,7 +208,7 @@ class VaultFiller @JvmOverloads constructor(
             cash.generateIssue(issuance, Amount(pennies, Issued(issuedBy, howMuch.token)), owner ?: services.myInfo.singleIdentity(), altNotary)
             return@map issuerServices.signInitialTransaction(issuance, issuedBy.party.owningKey)
         }
-        services.recordTransactions(transactions)
+        services.recordTransactions(statesToRecord, transactions)
         // Get all the StateRefs of all the generated transactions.
         val states = transactions.flatMap { stx ->
             stx.tx.outputs.indices.map { i -> stx.tx.outRef<Cash.State>(i) }