Merge branch 'release/os/4.10' into shams-4.11-merge-53fe0e6c

# Conflicts:
#	node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt

@@ -272,7 +272,7 @@ class HibernateAttachmentQueryCriteriaParser<T,R>(override val criteriaBuilder:
 class HibernateQueryCriteriaParser(val contractStateType: Class<out ContractState>,
                                    val contractStateTypeMappings: Map<String, Set<String>>,
                                    override val criteriaBuilder: CriteriaBuilder,
-                                   val criteriaQuery: CriteriaQuery<Tuple>,
+                                   val criteriaQuery: CriteriaQuery<*>,
                                    val vaultStates: Root<VaultSchemaV1.VaultStates>) : AbstractQueryCriteriaParser<QueryCriteria, IQueryCriteriaParser, Sort>(), IQueryCriteriaParser {
     private companion object {
         private val log = contextLogger()

node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt

@@ -35,7 +35,6 @@ import net.corda.core.node.services.vault.PageSpecification
 import net.corda.core.node.services.vault.QueryCriteria
 import net.corda.core.node.services.vault.Sort
 import net.corda.core.node.services.vault.SortAttribute
-import net.corda.core.node.services.vault.builder
 import net.corda.core.observable.internal.OnResilientSubscribe
 import net.corda.core.schemas.PersistentStateRef
 import net.corda.core.serialization.SingletonSerializeAsToken
@@ -69,17 +68,21 @@ import java.security.PublicKey
 import java.sql.SQLException
 import java.time.Clock
 import java.time.Instant
-import java.util.Arrays
+import java.util.*
-import java.util.UUID
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.CopyOnWriteArraySet
 import java.util.stream.Stream
 import javax.persistence.PersistenceException
 import javax.persistence.Tuple
 import javax.persistence.criteria.CriteriaBuilder
+import javax.persistence.criteria.CriteriaQuery
 import javax.persistence.criteria.CriteriaUpdate
 import javax.persistence.criteria.Predicate
 import javax.persistence.criteria.Root
+import kotlin.collections.ArrayList
+import kotlin.collections.LinkedHashSet
+import kotlin.collections.component1
+import kotlin.collections.component2
 
 /**
  * The vault service handles storage, retrieval and querying of states.
@@ -707,10 +710,8 @@ class NodeVaultService(
                                             paging: PageSpecification,
                                             sorting: Sort,
                                             contractStateType: Class<out T>): Vault.Page<T> {
-        // calculate total results where a page specification has been defined
+        val (criteriaQuery, criteriaParser) = buildCriteriaQuery<Tuple>(criteria, contractStateType, sorting)
-        val totalStatesAvailable = if (paging.isDefault) -1 else queryTotalStateCount(criteria, contractStateType)
+        val query = getSession().createQuery(criteriaQuery)
-
-        val (query, stateTypes) = createQuery(criteria, contractStateType, sorting)
         query.setResultWindow(paging)
 
         var previousPageAnchor: StateRef? = null
@@ -742,7 +743,14 @@ class NodeVaultService(
                 ArrayList()
         )
 
-        return Vault.Page(states, statesMetadata, totalStatesAvailable, stateTypes, otherResults, previousPageAnchor)
+        val totalStatesAvailable = when {
+            paging.isDefault -> -1L
+            // If the first page isn't full then we know that's all the states that are available
+            paging.pageNumber == DEFAULT_PAGE_NUM && states.size < paging.pageSize -> states.size.toLong()
+            else -> queryTotalStateCount(criteria, contractStateType)
+        }
+
+        return Vault.Page(states, statesMetadata, totalStatesAvailable, criteriaParser.stateTypes, otherResults, previousPageAnchor)
     }
 
     private fun <R> Query<R>.resultStream(paging: PageSpecification): Stream<R> {
@@ -775,19 +783,17 @@ class NodeVaultService(
         }
     }
 
-    private fun <T : ContractState> queryTotalStateCount(baseCriteria: QueryCriteria, contractStateType: Class<out T>): Long {
+    private fun <T : ContractState> queryTotalStateCount(criteria: QueryCriteria, contractStateType: Class<out T>): Long {
-        val count = builder { VaultSchemaV1.VaultStates::recordedTime.count() }
+        val (criteriaQuery, criteriaParser) = buildCriteriaQuery<Long>(criteria, contractStateType, null)
-        val countCriteria = QueryCriteria.VaultCustomQueryCriteria(count, Vault.StateStatus.ALL)
+        criteriaQuery.select(criteriaBuilder.countDistinct(criteriaParser.vaultStates))
-        val criteria = baseCriteria.and(countCriteria)
+        val query = getSession().createQuery(criteriaQuery)
-        val (query) = createQuery(criteria, contractStateType, null)
+        return query.singleResult
-        val results = query.resultList
-        return results.last().toArray().last() as Long
     }
 
-    private fun <T : ContractState> createQuery(criteria: QueryCriteria,
+    private inline fun <reified T> buildCriteriaQuery(criteria: QueryCriteria,
-                                                contractStateType: Class<out T>,
+                                                      contractStateType: Class<out ContractState>,
-                                                sorting: Sort?): Pair<Query<Tuple>, Vault.StateStatus> {
+                                                      sorting: Sort?): Pair<CriteriaQuery<T>, HibernateQueryCriteriaParser> {
-        val criteriaQuery = criteriaBuilder.createQuery(Tuple::class.java)
+        val criteriaQuery = criteriaBuilder.createQuery(T::class.java)
         val criteriaParser = HibernateQueryCriteriaParser(
                 contractStateType,
                 contractStateTypeMappings,
@@ -796,8 +802,7 @@ class NodeVaultService(
                 criteriaQuery.from(VaultSchemaV1.VaultStates::class.java)
         )
         criteriaParser.parse(criteria, sorting)
-        val query = getSession().createQuery(criteriaQuery)
+        return Pair(criteriaQuery, criteriaParser)
-        return Pair(query, criteriaParser.stateTypes)
     }
 
     /**

node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt

@@ -22,7 +22,7 @@ class VaultQueryExceptionsTests : VaultQueryParties by rule {
 
         @ClassRule
         @JvmField
-        val rule = object : VaultQueryTestRule() {
+        val rule = object : VaultQueryTestRule(persistentServices = false) {
             override val cordappPackages = listOf(
                     "net.corda.testing.contracts",
                     "net.corda.finance.contracts",

node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt

@@ -4,6 +4,7 @@ import com.nhaarman.mockito_kotlin.mock
 import net.corda.core.contracts.*
 import net.corda.core.crypto.*
 import net.corda.core.identity.AbstractParty
+import net.corda.core.identity.AnonymousParty
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
 import net.corda.core.internal.packageName
@@ -37,6 +38,7 @@ import net.corda.testing.internal.configureDatabase
 import net.corda.testing.internal.vault.*
 import net.corda.testing.node.MockServices
 import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndMockServices
+import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndPersistentServices
 import net.corda.testing.node.makeTestIdentityService
 import org.assertj.core.api.Assertions.assertThat
 import org.assertj.core.api.Assertions.assertThatCode
@@ -102,7 +104,7 @@ interface VaultQueryParties {
     val cordappPackages: List<String>
 }
 
-open class VaultQueryTestRule : ExternalResource(), VaultQueryParties {
+open class VaultQueryTestRule(private val persistentServices: Boolean) : ExternalResource(), VaultQueryParties {
     override val alice = TestIdentity(ALICE_NAME, 70)
     override val bankOfCorda = TestIdentity(BOC_NAME)
     override val bigCorp = TestIdentity(CordaX500Name("BigCorporation", "New York", "US"))
@@ -135,12 +137,22 @@ open class VaultQueryTestRule : ExternalResource(), VaultQueryParties {
 
 
     override fun before() {
-        // register additional identities
+        val databaseAndServices = if (persistentServices) {
-        val databaseAndServices = makeTestDatabaseAndMockServices(
+            makeTestDatabaseAndPersistentServices(
-                cordappPackages,
+                    cordappPackages,
-                makeTestIdentityService(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity),
+                    megaCorp,
-                megaCorp,
+                    moreKeys = setOf(DUMMY_NOTARY_KEY),
-                moreKeys = *arrayOf(DUMMY_NOTARY_KEY))
+                    moreIdentities = setOf(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity)
+            )
+        } else {
+            @Suppress("SpreadOperator")
+            makeTestDatabaseAndMockServices(
+                    cordappPackages,
+                    makeTestIdentityService(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity),
+                    megaCorp,
+                    moreKeys = *arrayOf(DUMMY_NOTARY_KEY)
+            )
+        }
         database = databaseAndServices.first
         services = databaseAndServices.second
         vaultFiller = VaultFiller(services, dummyNotary)
@@ -2890,9 +2902,8 @@ abstract class VaultQueryTestsBase : VaultQueryParties {
 }
 
 class VaultQueryTests : VaultQueryTestsBase(), VaultQueryParties by delegate {
-
     companion object {
-        val delegate = VaultQueryTestRule()
+        val delegate = VaultQueryTestRule(persistentServices = false)
     }
 
     @Rule
@@ -3195,4 +3206,34 @@ class VaultQueryTests : VaultQueryTestsBase(), VaultQueryParties by delegate {
             )
         }
     }
-}
+}
+
+
+class PersistentServicesVaultQueryTests : VaultQueryParties by delegate {
+    companion object {
+        val delegate = VaultQueryTestRule(persistentServices = true)
+
+        @ClassRule
+        @JvmField
+        val testSerialization = SerializationEnvironmentRule()
+    }
+
+    @Rule
+    @JvmField
+    val vaultQueryTestRule = delegate
+
+    @Test(timeout = 300_000)
+    fun `query on externalId which maps to multiple keys`() {
+        val externalId = UUID.randomUUID()
+        val page = database.transaction {
+            val keys = Array(2) { services.keyManagementService.freshKey(externalId) }
+            vaultFiller.fillWithDummyState(participants = keys.map(::AnonymousParty))
+            services.vaultService.queryBy<ContractState>(
+                    VaultQueryCriteria(externalIds = listOf(externalId)),
+                    paging = PageSpecification(DEFAULT_PAGE_NUM, 10)
+            )
+        }
+        assertThat(page.states).hasSize(1)
+        assertThat(page.totalStatesAvailable).isEqualTo(1)
+    }
+}

serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EvolvabilityTests.kt

@@ -637,6 +637,45 @@ class EvolvabilityTests {
         assertEquals(DUMMY_NOTARY_PARTY, networkParams.notaries.firstOrNull()?.identity)
     }
 
+    //
+    // This test uses a NetworkParameters signed set of bytes generated by a Corda 4.11 build and
+    // is here to ensure we can still read them. This test exists because Corda 4.11 Adds in new
+    // network parameters that should be backwards compatible with older corda versions
+    //
+    // The file itself was generated from a corda-os 4.11 build at commit
+    //      58ecce1 Ent-9875: New Network Parameters
+    //
+    // To regenerate the file un-ignore the test below this one (regenerate broken network parameters),
+    // to regenerate at a specific version add that test to a checkout at the desired sha then take
+    // the resulting file and add to the repo, changing the filename as appropriate
+    //
+    @Test(timeout=300_000)
+    fun `read corda 4-11 network parameters`() {
+        val sf = testDefaultFactory()
+        sf.register(net.corda.serialization.internal.amqp.custom.InstantSerializer(sf))
+        sf.register(net.corda.serialization.internal.amqp.custom.PublicKeySerializer)
+        sf.register(net.corda.serialization.internal.amqp.custom.DurationSerializer(sf))
+
+        //
+        // filename breakdown
+        // networkParams - because this is a serialised set of network parameters
+        // r3corda - generated by Corda Enterprise instead of Corda
+        // 6a6b6f256 - Commit sha of the build that generated the file we're testing against
+        //
+        val resource = "networkParams.4.11.58ecce1"
+
+        val url = EvolvabilityTests::class.java.getResource(resource)
+        val sc2 = url.readBytes()
+        val deserializedC = DeserializationInput(sf).deserialize(SerializedBytes<SignedData<NetworkParameters>>(sc2))
+        val networkParams = DeserializationInput(sf).deserialize(deserializedC.raw)
+
+        assertEquals(1000, networkParams.maxMessageSize)
+        assertEquals(1000, networkParams.maxTransactionSize)
+        assertEquals(3, networkParams.minimumPlatformVersion)
+        assertEquals(1, networkParams.notaries.size)
+        assertEquals(DUMMY_NOTARY_PARTY, networkParams.notaries.firstOrNull()?.identity)
+    }
+
     //
     // This test created a serialized and signed set of Network Parameters to test whether we
     // can still deserialize them