From 98af7f10f9758d429d050c96ddcdb9e2144b20fe Mon Sep 17 00:00:00 2001 From: Denis Rekalov Date: Tue, 19 May 2020 15:53:49 +0100 Subject: [PATCH] CORDA-3805: Make custom migration scripts independent on the latest PersistentIdentityService schema --- .../migration/PersistentIdentityMigration.kt | 7 +- .../PersistentIdentityMigrationNewTable.kt | 37 +---- .../migration/node-core.changelog-master.xml | 3 +- ...entityServiceToStringShortMigrationTest.kt | 18 +- .../node/migration/MigrationTestSchema.kt | 78 +++++++++ ...PersistentIdentityMigrationNewTableTest.kt | 157 +++++------------- 6 files changed, 143 insertions(+), 157 deletions(-) create mode 100644 node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt diff --git a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt index 93db88b4ca..ffbb0c1b8e 100644 --- a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt +++ b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt @@ -9,16 +9,13 @@ import liquibase.statement.core.UpdateStatement import net.corda.core.crypto.toStringShort import net.corda.core.identity.CordaX500Name import net.corda.core.identity.PartyAndCertificate -import net.corda.core.utilities.contextLogger -import net.corda.node.services.identity.PersistentIdentityService import net.corda.nodeapi.internal.crypto.X509CertificateFactory class PersistentIdentityMigration : CustomSqlChange { companion object { - private val logger = contextLogger() - const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME - const val X500_NAME_TO_PUB_KEY_HASH_TABLE = PersistentIdentityService.NAME_TO_HASH_TABLE_NAME + const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = "node_identities" + const val X500_NAME_TO_PUB_KEY_HASH_TABLE = "node_named_identities" } override fun validate(database: Database?): ValidationErrors? { diff --git a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt index 59408499c6..a7381cff58 100644 --- a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt +++ b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt @@ -1,28 +1,27 @@ package net.corda.node.migration +import liquibase.change.custom.CustomTaskChange import liquibase.database.Database import liquibase.database.jvm.JdbcConnection import liquibase.exception.ValidationErrors import liquibase.resource.ResourceAccessor import net.corda.core.identity.CordaX500Name import net.corda.core.identity.PartyAndCertificate -import net.corda.core.schemas.MappedSchema import net.corda.core.utilities.contextLogger -import net.corda.node.internal.DBNetworkParametersStorage -import net.corda.node.services.identity.PersistentIdentityService -import net.corda.node.services.persistence.DBTransactionStorage -import net.corda.node.services.persistence.NodeAttachmentService import net.corda.nodeapi.internal.crypto.X509CertificateFactory +import net.corda.nodeapi.internal.persistence.SchemaMigration import java.security.PublicKey /** * Migration that reads data from the [PersistentIdentityCert] table, extracts the parameters required to insert into the [PersistentIdentity] table. */ -class PersistentIdentityMigrationNewTable : CordaMigration() { +class PersistentIdentityMigrationNewTable : CustomTaskChange { companion object { private val logger = contextLogger() } + private lateinit var ourName: CordaX500Name + override fun execute(database: Database?) { logger.info("Migrating persistent identities with certificates table into persistent table with no certificate data.") @@ -30,7 +29,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() { logger.error("Cannot migrate persistent identities: Liquibase failed to provide a suitable database connection") throw PersistentIdentitiesMigrationException("Cannot migrate persistent identities as liquibase failed to provide a suitable database connection") } - initialiseNodeServices(database, setOf(PersistentIdentitiesMigrationSchemaBuilder.getMappedSchema())) + ourName = CordaX500Name.parse(System.getProperty(SchemaMigration.NODE_X500_NAME)) val connection = database.connection as JdbcConnection val hashToKeyAndName = extractKeyAndName(connection) @@ -68,7 +67,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() { it.setString(2, name.toString()) it.executeUpdate() } - if (name !in identityService.ourNames) { + if (name != ourName) { connection.prepareStatement("INSERT INTO node_hash_to_key (pk_hash, public_key) VALUES (?,?)").use { it.setString(1, publicKeyHash) it.setBytes(2, publicKey.encoded) @@ -92,26 +91,4 @@ class PersistentIdentityMigrationNewTable : CordaMigration() { } } -/** - * A minimal set of schema for retrieving data from the database. - * - * Note that adding an extra schema here may cause migrations to fail if it ends up creating a table before the same table - * is created in a migration script. As such, this migration must be run after the tables for the following have been created (and, - * if they are removed in the future, before they are deleted). - */ -object PersistentIdentitiesMigrationSchema - -object PersistentIdentitiesMigrationSchemaBuilder { - fun getMappedSchema() = - MappedSchema(schemaFamily = PersistentIdentitiesMigrationSchema.javaClass, version = 1, - mappedTypes = listOf( - DBTransactionStorage.DBTransaction::class.java, - PersistentIdentityService.PersistentPublicKeyHashToCertificate::class.java, - PersistentIdentityService.PersistentPartyToPublicKeyHash::class.java, - PersistentIdentityService.PersistentPublicKeyHashToParty::class.java, - PersistentIdentityService.PersistentHashToPublicKey::class.java, - NodeAttachmentService.DBAttachment::class.java, - DBNetworkParametersStorage.PersistentNetworkParameters::class.java - )) -} class PersistentIdentitiesMigrationException(msg: String, cause: Exception? = null) : Exception(msg, cause) \ No newline at end of file diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index 9e96e93d01..ec7e8035e1 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -24,13 +24,12 @@ + - - diff --git a/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt b/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt index c539a83a07..5f9d7f4785 100644 --- a/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt +++ b/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt @@ -1,5 +1,7 @@ package net.corda.node.migration +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever import liquibase.database.core.H2Database import liquibase.database.jvm.JdbcConnection import net.corda.core.crypto.toStringShort @@ -7,7 +9,8 @@ import net.corda.core.identity.CordaX500Name import net.corda.core.identity.PartyAndCertificate import net.corda.core.internal.hash import net.corda.core.utilities.contextLogger -import net.corda.node.services.identity.PersistentIdentityService +import net.corda.coretesting.internal.rigorousMock +import net.corda.node.services.api.SchemaService import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig import net.corda.nodeapi.internal.persistence.contextTransactionOrNull @@ -44,11 +47,16 @@ class IdentityServiceToStringShortMigrationTest { @Before fun setUp() { + val schemaService = rigorousMock() + doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas + cordaDB = configureDatabase( makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, + schemaService = schemaService, + internalSchemas = setOf(), ourName = BOB_IDENTITY.name) liquibaseDB = H2Database() liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection) @@ -66,8 +74,8 @@ class IdentityServiceToStringShortMigrationTest { cordaDB.transaction { val groupedIdentities = identities.groupBy { it.name } groupedIdentities.forEach { name, certs -> - val persistentIDs = certs.map { PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded) } - val persistentName = PersistentIdentityService.PersistentPartyToPublicKeyHash(name.toString(), certs.first().owningKey.hash.toString()) + val persistentIDs = certs.map { IdentityTestSchemaV1.NodeIdentities(it.owningKey.hash.toString(), it.certPath.encoded) } + val persistentName = IdentityTestSchemaV1.NodeNamedIdentities(name.toString(), certs.first().owningKey.hash.toString()) persistentIDs.forEach { session.persist(it) } @@ -87,7 +95,7 @@ class IdentityServiceToStringShortMigrationTest { identities.forEach { logger.info("Checking: ${it.name}") cordaDB.transaction { - val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT ${PersistentIdentityService.PK_HASH_COLUMN_NAME} FROM ${PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME} WHERE pk_hash=?") + val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT pk_hash FROM node_identities WHERE pk_hash=?") hashToIdentityStatement.setString(1, it.owningKey.toStringShort()) val hashToIdentityResultSet = hashToIdentityStatement.executeQuery() @@ -96,7 +104,7 @@ class IdentityServiceToStringShortMigrationTest { //check that the pk_hash actually matches what we expect (kinda redundant, but deserializing the whole PartyAndCertificate feels like overkill) Assert.assertThat(hashToIdentityResultSet.getString(1), `is`(it.owningKey.toStringShort())) - val nameToHashStatement = connection.prepareStatement("SELECT ${PersistentIdentityService.NAME_COLUMN_NAME} FROM ${PersistentIdentityService.NAME_TO_HASH_TABLE_NAME} WHERE pk_hash=?") + val nameToHashStatement = connection.prepareStatement("SELECT name FROM node_named_identities WHERE pk_hash=?") nameToHashStatement.setString(1, it.owningKey.toStringShort()) val nameToHashResultSet = nameToHashStatement.executeQuery() diff --git a/node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt b/node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt new file mode 100644 index 0000000000..1f744ec5d1 --- /dev/null +++ b/node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt @@ -0,0 +1,78 @@ +package net.corda.node.migration + +import net.corda.core.schemas.MappedSchema +import net.corda.core.utilities.MAX_HASH_HEX_SIZE +import org.apache.commons.lang3.ArrayUtils +import org.hibernate.annotations.Type +import javax.persistence.Column +import javax.persistence.Entity +import javax.persistence.Id +import javax.persistence.Table + +object MigrationTestSchema + +/** + * Schema definition for testing PersistentIdentityService custom migration scripts at the moment when scripts were written. + * Used to break dependency on the latest PersistentIdentityService which schema version may be different. + * + * This will allow: + * - to fix the position of relevant scripts in the node-core.changelog-master.xml (instead of placing them at the end) + * - to perform further modifications of PersistentIdentityService schema without impacting existing migration scripts and their tests + */ +object IdentityTestSchemaV1 : MappedSchema( + schemaFamily = MigrationTestSchema::class.java, + version = 1, + mappedTypes = listOf( + NodeIdentities::class.java, + NodeNamedIdentities::class.java, + NodeIdentitiesNoCert::class.java, + NodeHashToKey::class.java + ) +) { + @Entity + @Table(name = "node_identities") + class NodeIdentities( + @Id + @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false) + var publicKeyHash: String = "", + + @Type(type = "corda-blob") + @Column(name = "identity_value", nullable = false) + var identity: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY + ) + + @Entity + @Table(name = "node_named_identities") + class NodeNamedIdentities( + @Id + @Suppress("MagicNumber") // database column width + @Column(name = "name", length = 128, nullable = false) + var name: String = "", + + @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false) + var publicKeyHash: String = "" + ) + + @Entity + @Table(name = "node_identities_no_cert") + class NodeIdentitiesNoCert( + @Id + @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false) + var publicKeyHash: String = "", + + @Column(name = "name", length = 128, nullable = false) + var name: String = "" + ) + + @Entity + @Table(name = "node_hash_to_key") + class NodeHashToKey( + @Id + @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false) + var publicKeyHash: String = "", + + @Type(type = "corda-blob") + @Column(name = "public_key", nullable = false) + var publicKey: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY + ) +} \ No newline at end of file diff --git a/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt b/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt index 2aaf1e9cca..2e65364932 100644 --- a/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt +++ b/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt @@ -1,48 +1,35 @@ package net.corda.node.migration -import liquibase.database.Database +import com.nhaarman.mockito_kotlin.doReturn +import com.nhaarman.mockito_kotlin.whenever +import liquibase.database.core.H2Database import liquibase.database.jvm.JdbcConnection -import net.corda.core.crypto.SecureHash -import net.corda.core.identity.CordaX500Name +import net.corda.core.crypto.Crypto +import net.corda.core.crypto.toStringShort import net.corda.core.identity.PartyAndCertificate -import net.corda.core.internal.hash -import net.corda.core.internal.signWithCert -import net.corda.core.node.NetworkParameters -import net.corda.core.node.NotaryInfo -import net.corda.node.internal.DBNetworkParametersStorage -import net.corda.node.migration.VaultStateMigrationTest.Companion.CHARLIE -import net.corda.node.migration.VaultStateMigrationTest.Companion.DUMMY_NOTARY -import net.corda.node.services.identity.PersistentIdentityService -import net.corda.node.services.keys.BasicHSMKeyManagementService -import net.corda.nodeapi.internal.crypto.X509Utilities +import net.corda.coretesting.internal.rigorousMock +import net.corda.node.services.api.SchemaService import net.corda.nodeapi.internal.persistence.CordaPersistence import net.corda.nodeapi.internal.persistence.DatabaseConfig +import net.corda.nodeapi.internal.persistence.SchemaMigration import net.corda.testing.core.* import net.corda.testing.internal.configureDatabase import net.corda.testing.node.MockServices -import net.corda.testing.node.makeTestIdentityService +import org.assertj.core.api.Assertions.assertThat import org.junit.After import org.junit.Before import org.junit.ClassRule import org.junit.Test -import org.mockito.Mockito -import java.security.KeyPair -import java.sql.Connection -import java.time.Clock -import java.time.Duration -import java.util.* -class PersistentIdentityMigrationNewTableTest{ +class PersistentIdentityMigrationNewTableTest { companion object { val alice = TestIdentity(ALICE_NAME, 70) val bankOfCorda = TestIdentity(BOC_NAME) val bob = TestIdentity(BOB_NAME, 80) - val dummyCashIssuer = TestIdentity(CordaX500Name("Snake Oil Issuer", "London", "GB"), 10) val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20) val ALICE_IDENTITY get() = alice.identity val BOB_IDENTITY get() = bob.identity val BOC_IDENTITY get() = bankOfCorda.identity - val BOC_KEY get() = bankOfCorda.keyPair val bob2 = TestIdentity(BOB_NAME, 40) val BOB2_IDENTITY = bob2.identity @@ -51,122 +38,62 @@ class PersistentIdentityMigrationNewTableTest{ val testSerialization = SerializationEnvironmentRule() } - lateinit var liquidBaseDB: Database + lateinit var liquibaseDB: H2Database lateinit var cordaDB: CordaPersistence - lateinit var notaryServices: MockServices @Before fun setUp() { - val identityService = makeTestIdentityService(PersistentIdentityMigrationNewTableTest.dummyNotary.identity, BOB_IDENTITY, ALICE_IDENTITY) - notaryServices = MockServices(listOf("net.corda.finance.contracts"), dummyNotary, identityService, dummyCashIssuer.keyPair, BOC_KEY) + val schemaService = rigorousMock() + doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas + System.setProperty(SchemaMigration.NODE_X500_NAME, BOB_IDENTITY.name.toString()) - // Runs migration tasks cordaDB = configureDatabase( MockServices.makeTestDataSourceProperties(), DatabaseConfig(), - notaryServices.identityService::wellKnownPartyFromX500Name, - notaryServices.identityService::wellKnownPartyFromAnonymous, + { null }, + { null }, + schemaService = schemaService, + internalSchemas = setOf(), ourName = BOB_IDENTITY.name) - val liquidbaseConnection = Mockito.mock(JdbcConnection::class.java) - Mockito.`when`(liquidbaseConnection.url).thenReturn(cordaDB.jdbcUrl) - Mockito.`when`(liquidbaseConnection.wrappedConnection).thenReturn(cordaDB.dataSource.connection) - liquidBaseDB = Mockito.mock(Database::class.java) - Mockito.`when`(liquidBaseDB.connection).thenReturn(liquidbaseConnection) - - cordaDB.dataSource.connection - saveOurKeys(listOf(bob.keyPair, bob2.keyPair)) - saveAllIdentities(listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY)) - addNetworkParameters() + liquibaseDB = H2Database() + liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection) + liquibaseDB.isAutoCommit = true } @After - fun `close`() { + fun close() { cordaDB.close() } - @Test(timeout=300_000) - fun `migrate identities to new table`() { - val pkHash = addTestMapping(cordaDB.dataSource.connection, alice) - PersistentIdentityMigrationNewTable() - verifyTestMigration(cordaDB.dataSource.connection, pkHash, alice.name.toString()) + @Test(timeout = 300_000) + fun `migrate identities to new table`() { + val identities = listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY) + saveAllIdentities(identities) + + PersistentIdentityMigrationNewTable().execute(liquibaseDB) + + val expectedParties = identities.map { it.owningKey.toStringShort() to it.toString() } + val actualParties = selectAll().map { it.publicKeyHash to it.name } + assertThat(actualParties).isEqualTo(expectedParties) + + val expectedKeys = listOf(ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity).map { it.owningKey.toStringShort() to it.owningKey } + val actualKeys = selectAll().map { it.publicKeyHash to Crypto.decodePublicKey(it.publicKey) } + assertThat(actualKeys).isEqualTo(expectedKeys) } private fun saveAllIdentities(identities: List) { cordaDB.transaction { identities.forEach { - session.save(PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded)) + session.save(IdentityTestSchemaV1.NodeIdentities(it.owningKey.toStringShort(), it.certPath.encoded)) } } } - private fun saveOurKeys(keys: List) { - cordaDB.transaction { - keys.forEach { - val persistentKey = BasicHSMKeyManagementService.PersistentKey(it.public, it.private) - session.save(persistentKey) - } - } - } - - private fun addNetworkParameters() { - cordaDB.transaction { - val clock = Clock.systemUTC() - val params = NetworkParameters( - 1, - listOf(NotaryInfo(DUMMY_NOTARY, false), NotaryInfo(CHARLIE, false)), - 1, - 1, - clock.instant(), - 1, - mapOf(), - Duration.ZERO, - mapOf() - ) - val signedParams = params.signWithCert(bob.keyPair.private, BOB_IDENTITY.certificate) - val persistentParams = DBNetworkParametersStorage.PersistentNetworkParameters( - SecureHash.allOnesHash.toString(), - params.epoch, - signedParams.raw.bytes, - signedParams.sig.bytes, - signedParams.sig.by.encoded, - X509Utilities.buildCertPath(signedParams.sig.parentCertsChain).encoded - ) - session.save(persistentParams) - } - } - - private fun addTestMapping(connection: Connection, testIdentity: TestIdentity): String { - val pkHash = UUID.randomUUID().toString() - val cert = testIdentity.identity.certPath.encoded - - connection.prepareStatement("INSERT INTO node_identities (pk_hash, identity_value) VALUES (?,?)").use { - it.setString(1, pkHash) - it.setBytes(2, cert) - it.executeUpdate() - } - return pkHash - } - -// private fun deleteTestMapping(connection: Connection, pkHash: String) { -// connection.prepareStatement("DELETE FROM node_identities WHERE pk_hash = ?").use { -// it.setString(1, pkHash) -// it.executeUpdate() -// } -// } - - private fun verifyTestMigration(connection: Connection, pk: String, name: String) { - connection.createStatement().use { - try { - val rs = it.executeQuery("SELECT (pk_hash, name) FROM node_identities_no_cert") - while (rs.next()) { - val result = rs.getString(1) - require(result.contains(pk)) - require(result.contains(name)) - } - rs.close() - } catch (e: Exception) { - println(e.localizedMessage) - } + private inline fun selectAll(): List { + return cordaDB.transaction { + val criteria = session.criteriaBuilder.createQuery(T::class.java) + criteria.select(criteria.from(T::class.java)) + session.createQuery(criteria).resultList } } } \ No newline at end of file