From 639acc4c6327d34368e6e026a7f98b0dcbe86432 Mon Sep 17 00:00:00 2001 From: tomstark99 Date: Mon, 17 Jul 2023 14:35:16 +0100 Subject: [PATCH 1/4] Added test to deserialise 4.11 network params --- .../internal/amqp/EvolvabilityTests.kt | 39 ++++++++++++++++++ .../internal/amqp/networkParams.4.11.58ecce1 | Bin 0 -> 4655 bytes 2 files changed, 39 insertions(+) create mode 100644 serialization/src/test/resources/net/corda/serialization/internal/amqp/networkParams.4.11.58ecce1 diff --git a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EvolvabilityTests.kt b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EvolvabilityTests.kt index 4c4d4c43a1..9514a65626 100644 --- a/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EvolvabilityTests.kt +++ b/serialization/src/test/kotlin/net/corda/serialization/internal/amqp/EvolvabilityTests.kt @@ -637,6 +637,45 @@ class EvolvabilityTests { assertEquals(DUMMY_NOTARY_PARTY, networkParams.notaries.firstOrNull()?.identity) } + // + // This test uses a NetworkParameters signed set of bytes generated by a Corda 4.11 build and + // is here to ensure we can still read them. This test exists because Corda 4.11 Adds in new + // network parameters that should be backwards compatible with older corda versions + // + // The file itself was generated from a corda-os 4.11 build at commit + // 58ecce1 Ent-9875: New Network Parameters + // + // To regenerate the file un-ignore the test below this one (regenerate broken network parameters), + // to regenerate at a specific version add that test to a checkout at the desired sha then take + // the resulting file and add to the repo, changing the filename as appropriate + // + @Test(timeout=300_000) + fun `read corda 4-11 network parameters`() { + val sf = testDefaultFactory() + sf.register(net.corda.serialization.internal.amqp.custom.InstantSerializer(sf)) + sf.register(net.corda.serialization.internal.amqp.custom.PublicKeySerializer) + sf.register(net.corda.serialization.internal.amqp.custom.DurationSerializer(sf)) + + // + // filename breakdown + // networkParams - because this is a serialised set of network parameters + // r3corda - generated by Corda Enterprise instead of Corda + // 6a6b6f256 - Commit sha of the build that generated the file we're testing against + // + val resource = "networkParams.4.11.58ecce1" + + val url = EvolvabilityTests::class.java.getResource(resource) + val sc2 = url.readBytes() + val deserializedC = DeserializationInput(sf).deserialize(SerializedBytes>(sc2)) + val networkParams = DeserializationInput(sf).deserialize(deserializedC.raw) + + assertEquals(1000, networkParams.maxMessageSize) + assertEquals(1000, networkParams.maxTransactionSize) + assertEquals(3, networkParams.minimumPlatformVersion) + assertEquals(1, networkParams.notaries.size) + assertEquals(DUMMY_NOTARY_PARTY, networkParams.notaries.firstOrNull()?.identity) + } + // // This test created a serialized and signed set of Network Parameters to test whether we // can still deserialize them diff --git a/serialization/src/test/resources/net/corda/serialization/internal/amqp/networkParams.4.11.58ecce1 b/serialization/src/test/resources/net/corda/serialization/internal/amqp/networkParams.4.11.58ecce1 new file mode 100644 index 0000000000000000000000000000000000000000..fa7ea435dfd2ef92d3abeed5ec193479124d6386 GIT binary patch literal 4655 zcmdT{S!^5S79KBSCxu4PQVP8UrDZ4J*d<$`&7F8@l7=j;oy18c(C_hYdzzVlCNuwd z$-P$s3Elf}A9z3&5<-Gs6%s;-rMw_`=pclI#3B~4hzC{)A$UL_ArK(S83&x1afaO7 z(g$7~$8-Mwob#RUeCLd)>O~abIPR{O3h>Esfj2mA$7S#(#GTqlFpY_}#jV;f#zs;* zV2<@Jl&c9;8%Pak+3S0IXPa+2tIRDbQ;R8ckmP%B&*l3{iG?-a!7x}~z@55cJ<=1Xiegk!a4c== zh)OE4E)3?lyTK5g_Hntu|7iZ`nX|tXKMtUDGN>dZQgPU%#5^=-)-3FT8X7!{g>~@c2(%YR3nkJo|Pu>B@9zX*iLZA6T4QHWIj&9n;21 zf9~j^LoWakwo=9xVLOg>w7v&{j@zDxI@>JXb{f81(0azhPoZ1kC!Sr?%Ft#vjuEwp zW7C+{RDF3C=}5s8>xPhNoef#n2A$>dW268+g73QAhOJDL-5{!@Bo#5C5|SsHe`#Y% zB9dY%NhB_t+5`cYRV1?=?flIMLNeKkC2U<1WjxkYs9s1SG{XXMVbuBj`Y5{)SHao1 zs!OXX@wwA=raQ{I6jYzWhJngBE3IOm%lAgv4lvB=h!{w;+v^XEuZgltm9;cyWkgG= zuFM0naLDKO>mYPRElMQ`7jw|V?BaDeerq#8!xcHe+d*s+OW0t68?8iu#hQmqD#@|i zC4(N^u-r}FPmq#2;<<1t$}ZBND1gIEjX<~+No&7&3TX!os!ODNQ%}=fdz*MKJ`iQy zIu_L`)*Dl({{I!f9%cJ#6^UXQGQ-7`N~jKXm{2qsD^T;M;x&Od5@2KNK(xi{3y-Kb zG^)n3cH84|WK>4ni9{?pvSo-fENlndVzIZQ{DjicguG?(?8-l`rNi9o!B1iZOA5Ba z3R`_Zr?pO{4c6`P_WOevs03|znFy`14O&<#LJdfnMGLAbV?;8HFCp2$PTKO;cl;)| z3)WQu*5e7FB&#(emc|BnXjZxvxi?+ENg-JOaHe}pl!XZhUYjPiQccdjf2?Pdm8UjJN*DZ=T&bV$gT^Kzuwsou8b}rW-_D?OicS8q698 zOYEr-NN+e5Hb%g4)(g4c&=suDBCR@wtvx^!QB@R`*od;onu>Tues`2bMb(7vZa7R_ z%6A!1RaQk{98h)r8kfx#AgHQdMnp22eDSmE0nu~X%@K(@Y+dtb7t9~nc&Jb85o_6{ z4{ge`ky2(@NI_Z)B`>soTZ5w-5$m4FpC@>m#FI_rhRQ=a$;$fCl4$hJq#Ly?>~*eq zNQA}L1$d)L)GoEcWSk}r}=Zr@B9HYq>zl3 zMdZNh+_E{GQ0E8pbpqSIqlTT$ec17aAdk+ECFhda6GO90^`X8RS{=wL_M5~>*!;J; zvOn_WnX}WEe-!)ViEr+Ynje1L*t7fd$GZRXy}7OP-Uo-y6dx82*EQwkH&4Iy`jgMS zcl@#LrT6ncK67X7-;aOMwd>%H&%W>Py!q3_9h3DZzkTK8&Hv5a{rBaQ_r3P7Z*#k! ze(%ej$&rYAB@g|+l85Y-Jlx^{Z~fp;RH;mGF^wp4pyHJUzqeAsddNBCCZjB*qZ%9W zOj85vu#ZctO=X0IXYD^+v}tY}TyKq;!U-(W5%2K34X!n$GK+bx{jJr~Qdy!%&O#!H z!YXz3fh~fqSo;F4GH%++f9+P}t)Ka=*7*EY#7_qVYu-tD Rcn1s?wzQFLT18uuegy#n2ju_& literal 0 HcmV?d00001 From 4b7e2a399502cb8cefc6a18487355041c1753f25 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Wed, 26 Jul 2023 10:46:25 +0100 Subject: [PATCH 2/4] ENT-10045: Fix vault query bug on externalId and mapping with multiple keys --- .../vault/HibernateQueryCriteriaParser.kt | 2 +- .../node/services/vault/NodeVaultService.kt | 37 +++++------ .../vault/VaultQueryExceptionsTests.kt | 2 +- .../node/services/vault/VaultQueryTests.kt | 61 ++++++++++++++++--- 4 files changed, 72 insertions(+), 30 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt b/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt index 3e8387803f..b5f9b327c2 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/HibernateQueryCriteriaParser.kt @@ -272,7 +272,7 @@ class HibernateAttachmentQueryCriteriaParser(override val criteriaBuilder: class HibernateQueryCriteriaParser(val contractStateType: Class, val contractStateTypeMappings: Map>, override val criteriaBuilder: CriteriaBuilder, - val criteriaQuery: CriteriaQuery, + val criteriaQuery: CriteriaQuery<*>, val vaultStates: Root) : AbstractQueryCriteriaParser(), IQueryCriteriaParser { private companion object { private val log = contextLogger() diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index ac0913604c..cccac84910 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -35,7 +35,6 @@ import net.corda.core.node.services.vault.PageSpecification import net.corda.core.node.services.vault.QueryCriteria import net.corda.core.node.services.vault.Sort import net.corda.core.node.services.vault.SortAttribute -import net.corda.core.node.services.vault.builder import net.corda.core.observable.internal.OnResilientSubscribe import net.corda.core.schemas.PersistentStateRef import net.corda.core.serialization.SingletonSerializeAsToken @@ -69,17 +68,21 @@ import java.security.PublicKey import java.sql.SQLException import java.time.Clock import java.time.Instant -import java.util.Arrays -import java.util.UUID +import java.util.* import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CopyOnWriteArraySet import java.util.stream.Stream import javax.persistence.PersistenceException import javax.persistence.Tuple import javax.persistence.criteria.CriteriaBuilder +import javax.persistence.criteria.CriteriaQuery import javax.persistence.criteria.CriteriaUpdate import javax.persistence.criteria.Predicate import javax.persistence.criteria.Root +import kotlin.collections.ArrayList +import kotlin.collections.LinkedHashSet +import kotlin.collections.component1 +import kotlin.collections.component2 /** * The vault service handles storage, retrieval and querying of states. @@ -709,7 +712,8 @@ class NodeVaultService( // calculate total results where a page specification has been defined val totalStatesAvailable = if (paging.isDefault) -1 else queryTotalStateCount(criteria, contractStateType) - val (query, stateTypes) = createQuery(criteria, contractStateType, sorting) + val (criteriaQuery, criteriaParser) = buildCriteriaQuery(criteria, contractStateType, sorting) + val query = getSession().createQuery(criteriaQuery) query.setResultWindow(paging) val statesMetadata: MutableList = mutableListOf() @@ -732,7 +736,7 @@ class NodeVaultService( ArrayList() ) - return Vault.Page(states, statesMetadata, totalStatesAvailable, stateTypes, otherResults) + return Vault.Page(states, statesMetadata, totalStatesAvailable, criteriaParser.stateTypes, otherResults) } private fun Query.resultStream(paging: PageSpecification): Stream { @@ -761,19 +765,17 @@ class NodeVaultService( } } - private fun queryTotalStateCount(baseCriteria: QueryCriteria, contractStateType: Class): Long { - val count = builder { VaultSchemaV1.VaultStates::recordedTime.count() } - val countCriteria = QueryCriteria.VaultCustomQueryCriteria(count, Vault.StateStatus.ALL) - val criteria = baseCriteria.and(countCriteria) - val (query) = createQuery(criteria, contractStateType, null) - val results = query.resultList - return results.last().toArray().last() as Long + private fun queryTotalStateCount(criteria: QueryCriteria, contractStateType: Class): Long { + val (criteriaQuery, criteriaParser) = buildCriteriaQuery(criteria, contractStateType, null) + criteriaQuery.select(criteriaBuilder.countDistinct(criteriaParser.vaultStates)) + val query = getSession().createQuery(criteriaQuery) + return query.singleResult } - private fun createQuery(criteria: QueryCriteria, - contractStateType: Class, - sorting: Sort?): Pair, Vault.StateStatus> { - val criteriaQuery = criteriaBuilder.createQuery(Tuple::class.java) + private inline fun buildCriteriaQuery(criteria: QueryCriteria, + contractStateType: Class, + sorting: Sort?): Pair, HibernateQueryCriteriaParser> { + val criteriaQuery = criteriaBuilder.createQuery(T::class.java) val criteriaParser = HibernateQueryCriteriaParser( contractStateType, contractStateTypeMappings, @@ -782,8 +784,7 @@ class NodeVaultService( criteriaQuery.from(VaultSchemaV1.VaultStates::class.java) ) criteriaParser.parse(criteria, sorting) - val query = getSession().createQuery(criteriaQuery) - return Pair(query, criteriaParser.stateTypes) + return Pair(criteriaQuery, criteriaParser) } /** diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt index d1a96ccda5..2dbd77b3ed 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryExceptionsTests.kt @@ -22,7 +22,7 @@ class VaultQueryExceptionsTests : VaultQueryParties by rule { @ClassRule @JvmField - val rule = object : VaultQueryTestRule() { + val rule = object : VaultQueryTestRule(persistentServices = false) { override val cordappPackages = listOf( "net.corda.testing.contracts", "net.corda.finance.contracts", diff --git a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt index b06518667c..94a6eda019 100644 --- a/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt +++ b/node/src/test/kotlin/net/corda/node/services/vault/VaultQueryTests.kt @@ -4,6 +4,7 @@ import com.nhaarman.mockito_kotlin.mock import net.corda.core.contracts.* import net.corda.core.crypto.* import net.corda.core.identity.AbstractParty +import net.corda.core.identity.AnonymousParty import net.corda.core.identity.CordaX500Name import net.corda.core.identity.Party import net.corda.core.internal.packageName @@ -37,6 +38,7 @@ import net.corda.testing.internal.configureDatabase import net.corda.testing.internal.vault.* import net.corda.testing.node.MockServices import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndMockServices +import net.corda.testing.node.MockServices.Companion.makeTestDatabaseAndPersistentServices import net.corda.testing.node.makeTestIdentityService import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThatCode @@ -102,7 +104,7 @@ interface VaultQueryParties { val cordappPackages: List } -open class VaultQueryTestRule : ExternalResource(), VaultQueryParties { +open class VaultQueryTestRule(private val persistentServices: Boolean) : ExternalResource(), VaultQueryParties { override val alice = TestIdentity(ALICE_NAME, 70) override val bankOfCorda = TestIdentity(BOC_NAME) override val bigCorp = TestIdentity(CordaX500Name("BigCorporation", "New York", "US")) @@ -135,12 +137,22 @@ open class VaultQueryTestRule : ExternalResource(), VaultQueryParties { override fun before() { - // register additional identities - val databaseAndServices = makeTestDatabaseAndMockServices( - cordappPackages, - makeTestIdentityService(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity), - megaCorp, - moreKeys = *arrayOf(DUMMY_NOTARY_KEY)) + val databaseAndServices = if (persistentServices) { + makeTestDatabaseAndPersistentServices( + cordappPackages, + megaCorp, + moreKeys = setOf(DUMMY_NOTARY_KEY), + moreIdentities = setOf(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity) + ) + } else { + @Suppress("SpreadOperator") + makeTestDatabaseAndMockServices( + cordappPackages, + makeTestIdentityService(MEGA_CORP_IDENTITY, MINI_CORP_IDENTITY, dummyCashIssuer.identity, dummyNotary.identity), + megaCorp, + moreKeys = *arrayOf(DUMMY_NOTARY_KEY) + ) + } database = databaseAndServices.first services = databaseAndServices.second vaultFiller = VaultFiller(services, dummyNotary) @@ -2832,9 +2844,8 @@ abstract class VaultQueryTestsBase : VaultQueryParties { } class VaultQueryTests : VaultQueryTestsBase(), VaultQueryParties by delegate { - companion object { - val delegate = VaultQueryTestRule() + val delegate = VaultQueryTestRule(persistentServices = false) } @Rule @@ -3137,4 +3148,34 @@ class VaultQueryTests : VaultQueryTestsBase(), VaultQueryParties by delegate { ) } } -} \ No newline at end of file +} + + +class PersistentServicesVaultQueryTests : VaultQueryParties by delegate { + companion object { + val delegate = VaultQueryTestRule(persistentServices = true) + + @ClassRule + @JvmField + val testSerialization = SerializationEnvironmentRule() + } + + @Rule + @JvmField + val vaultQueryTestRule = delegate + + @Test(timeout = 300_000) + fun `query on externalId which maps to multiple keys`() { + val externalId = UUID.randomUUID() + val page = database.transaction { + val keys = Array(2) { services.keyManagementService.freshKey(externalId) } + vaultFiller.fillWithDummyState(participants = keys.map(::AnonymousParty)) + services.vaultService.queryBy( + VaultQueryCriteria(externalIds = listOf(externalId)), + paging = PageSpecification(DEFAULT_PAGE_NUM, 10) + ) + } + assertThat(page.states).hasSize(1) + assertThat(page.totalStatesAvailable).isEqualTo(1) + } +} From 5cdbec9ddf8236d4f381fc5b654e487da31a4d94 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Mon, 21 Aug 2023 10:30:42 +0100 Subject: [PATCH 3/4] ENT-6876: Optimised vault query to not query for total state count if the first page isn't full (#7449) --- .../net/corda/node/services/vault/NodeVaultService.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index ac0913604c..9c90c36dd3 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -706,9 +706,6 @@ class NodeVaultService( paging: PageSpecification, sorting: Sort, contractStateType: Class): Vault.Page { - // calculate total results where a page specification has been defined - val totalStatesAvailable = if (paging.isDefault) -1 else queryTotalStateCount(criteria, contractStateType) - val (query, stateTypes) = createQuery(criteria, contractStateType, sorting) query.setResultWindow(paging) @@ -732,6 +729,13 @@ class NodeVaultService( ArrayList() ) + val totalStatesAvailable = when { + paging.isDefault -> -1L + // If the first page isn't full then we know that's all the states that are available + paging.pageNumber == DEFAULT_PAGE_NUM && states.size < paging.pageSize -> states.size.toLong() + else -> queryTotalStateCount(criteria, contractStateType) + } + return Vault.Page(states, statesMetadata, totalStatesAvailable, stateTypes, otherResults) } From 6edb1b779c2151c2160fe19853ec8cdced23f983 Mon Sep 17 00:00:00 2001 From: Shams Asari Date: Wed, 23 Aug 2023 15:03:08 +0100 Subject: [PATCH 4/4] Placate Detekt --- .../kotlin/net/corda/node/services/vault/NodeVaultService.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt index a1bf1893b0..1b6c43e33b 100644 --- a/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt +++ b/node/src/main/kotlin/net/corda/node/services/vault/NodeVaultService.kt @@ -93,6 +93,7 @@ import kotlin.collections.component2 * TODO: keep an audit trail with time stamps of previously unconsumed states "as of" a particular point in time. * TODO: have transaction storage do some caching. */ +@Suppress("LargeClass") class NodeVaultService( private val clock: Clock, private val keyManagementService: KeyManagementService,