Merge pull request #6266 from corda/denis/CORDA-3805-custom-migration-scripts

CORDA-3805: cut dependency from PersistentIdentityService for custom migration scripts
2024-12-19 04:57:58 +00:00 · 2020-06-04 14:20:26 +01:00 · 2020-06-04 14:20:26 +01:00 · 45614cf29e
commit 45614cf29e
parent f0d2c9fe71 98af7f10f9
6 changed files with 143 additions and 157 deletions
--- a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt
+++ b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt
@ -9,16 +9,13 @@ import liquibase.statement.core.UpdateStatement
 import net.corda.core.crypto.toStringShort
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.utilities.contextLogger
-import net.corda.node.services.identity.PersistentIdentityService
 import net.corda.nodeapi.internal.crypto.X509CertificateFactory

 class PersistentIdentityMigration : CustomSqlChange {

    companion object {
-        private val logger = contextLogger()
-        const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME
-        const val X500_NAME_TO_PUB_KEY_HASH_TABLE = PersistentIdentityService.NAME_TO_HASH_TABLE_NAME
+        const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = "node_identities"
+        const val X500_NAME_TO_PUB_KEY_HASH_TABLE = "node_named_identities"
    }

    override fun validate(database: Database?): ValidationErrors? {
--- a/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt
+++ b/node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt
@ -1,28 +1,27 @@
 package net.corda.node.migration

+import liquibase.change.custom.CustomTaskChange
 import liquibase.database.Database
 import liquibase.database.jvm.JdbcConnection
 import liquibase.exception.ValidationErrors
 import liquibase.resource.ResourceAccessor
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.schemas.MappedSchema
 import net.corda.core.utilities.contextLogger
-import net.corda.node.internal.DBNetworkParametersStorage
-import net.corda.node.services.identity.PersistentIdentityService
-import net.corda.node.services.persistence.DBTransactionStorage
-import net.corda.node.services.persistence.NodeAttachmentService
 import net.corda.nodeapi.internal.crypto.X509CertificateFactory
+import net.corda.nodeapi.internal.persistence.SchemaMigration
 import java.security.PublicKey

 /**
 * Migration that reads data from the [PersistentIdentityCert] table, extracts the parameters required to insert into the [PersistentIdentity] table.
 */
-class PersistentIdentityMigrationNewTable : CordaMigration() {
+class PersistentIdentityMigrationNewTable : CustomTaskChange {
    companion object {
        private val logger = contextLogger()
    }

+    private lateinit var ourName: CordaX500Name
+
    override fun execute(database: Database?) {
        logger.info("Migrating persistent identities with certificates table into persistent table with no certificate data.")

@ -30,7 +29,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
            logger.error("Cannot migrate persistent identities: Liquibase failed to provide a suitable database connection")
            throw PersistentIdentitiesMigrationException("Cannot migrate persistent identities as liquibase failed to provide a suitable database connection")
        }
-        initialiseNodeServices(database, setOf(PersistentIdentitiesMigrationSchemaBuilder.getMappedSchema()))
+        ourName = CordaX500Name.parse(System.getProperty(SchemaMigration.NODE_X500_NAME))

        val connection = database.connection as JdbcConnection
        val hashToKeyAndName = extractKeyAndName(connection)
@ -68,7 +67,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
            it.setString(2, name.toString())
            it.executeUpdate()
        }
-        if (name !in identityService.ourNames) {
+        if (name != ourName) {
            connection.prepareStatement("INSERT INTO node_hash_to_key (pk_hash, public_key) VALUES (?,?)").use {
                it.setString(1, publicKeyHash)
                it.setBytes(2, publicKey.encoded)
@ -92,26 +91,4 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
    }
 }

-/**
- * A minimal set of schema for retrieving data from the database.
- *
- * Note that adding an extra schema here may cause migrations to fail if it ends up creating a table before the same table
- * is created in a migration script. As such, this migration must be run after the tables for the following have been created (and,
- * if they are removed in the future, before they are deleted).
- */
-object PersistentIdentitiesMigrationSchema
-
-object PersistentIdentitiesMigrationSchemaBuilder {
-    fun getMappedSchema() =
-            MappedSchema(schemaFamily = PersistentIdentitiesMigrationSchema.javaClass, version = 1,
-                    mappedTypes = listOf(
-                            DBTransactionStorage.DBTransaction::class.java,
-                            PersistentIdentityService.PersistentPublicKeyHashToCertificate::class.java,
-                            PersistentIdentityService.PersistentPartyToPublicKeyHash::class.java,
-                            PersistentIdentityService.PersistentPublicKeyHashToParty::class.java,
-                            PersistentIdentityService.PersistentHashToPublicKey::class.java,
-                            NodeAttachmentService.DBAttachment::class.java,
-                            DBNetworkParametersStorage.PersistentNetworkParameters::class.java
-                    ))
-}
 class PersistentIdentitiesMigrationException(msg: String, cause: Exception? = null) : Exception(msg, cause)
--- a/node/src/main/resources/migration/node-core.changelog-master.xml
+++ b/node/src/main/resources/migration/node-core.changelog-master.xml
@ -24,13 +24,12 @@
    <include file="migration/node-core.changelog-v13.xml"/>
    <!-- This change should be done before the v14-data migration. -->
    <include file="migration/node-core.changelog-v15.xml"/>
+    <include file="migration/node-core.changelog-v14-data.xml"/>
    <include file="migration/node-core.changelog-v16.xml"/>

    <!-- This must run after node-core.changelog-init.xml, to prevent database columns being created twice. -->
    <include file="migration/vault-schema.changelog-v9.xml"/>

-    <include file="migration/node-core.changelog-v14-data.xml"/>
-
    <include file="migration/node-core.changelog-v19.xml"/>
    <include file="migration/node-core.changelog-v19-postgres.xml"/>
    <include file="migration/node-core.changelog-v19-keys.xml"/>
--- a/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt
+++ b/node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt
@ -1,5 +1,7 @@
 package net.corda.node.migration

+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
 import liquibase.database.core.H2Database
 import liquibase.database.jvm.JdbcConnection
 import net.corda.core.crypto.toStringShort
@ -7,7 +9,8 @@ import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
 import net.corda.core.internal.hash
 import net.corda.core.utilities.contextLogger
-import net.corda.node.services.identity.PersistentIdentityService
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.node.services.api.SchemaService
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.nodeapi.internal.persistence.contextTransactionOrNull
@ -44,11 +47,16 @@ class IdentityServiceToStringShortMigrationTest {

    @Before
    fun setUp() {
+        val schemaService = rigorousMock<SchemaService>()
+        doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
+
        cordaDB = configureDatabase(
                makeTestDataSourceProperties(),
                DatabaseConfig(),
                { null },
                { null },
+                schemaService = schemaService,
+                internalSchemas = setOf(),
                ourName = BOB_IDENTITY.name)
        liquibaseDB = H2Database()
        liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
@ -66,8 +74,8 @@ class IdentityServiceToStringShortMigrationTest {
        cordaDB.transaction {
            val groupedIdentities = identities.groupBy { it.name }
            groupedIdentities.forEach { name, certs ->
-                val persistentIDs = certs.map { PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded) }
-                val persistentName = PersistentIdentityService.PersistentPartyToPublicKeyHash(name.toString(), certs.first().owningKey.hash.toString())
+                val persistentIDs = certs.map { IdentityTestSchemaV1.NodeIdentities(it.owningKey.hash.toString(), it.certPath.encoded) }
+                val persistentName = IdentityTestSchemaV1.NodeNamedIdentities(name.toString(), certs.first().owningKey.hash.toString())
                persistentIDs.forEach {
                    session.persist(it)
                }
@ -87,7 +95,7 @@ class IdentityServiceToStringShortMigrationTest {
        identities.forEach {
            logger.info("Checking: ${it.name}")
            cordaDB.transaction {
-                val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT ${PersistentIdentityService.PK_HASH_COLUMN_NAME} FROM ${PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME} WHERE pk_hash=?")
+                val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT pk_hash FROM node_identities WHERE pk_hash=?")
                hashToIdentityStatement.setString(1, it.owningKey.toStringShort())
                val hashToIdentityResultSet = hashToIdentityStatement.executeQuery()

@ -96,7 +104,7 @@ class IdentityServiceToStringShortMigrationTest {
                //check that the pk_hash actually matches what we expect (kinda redundant, but deserializing the whole PartyAndCertificate feels like overkill)
                Assert.assertThat(hashToIdentityResultSet.getString(1), `is`(it.owningKey.toStringShort()))

-                val nameToHashStatement = connection.prepareStatement("SELECT ${PersistentIdentityService.NAME_COLUMN_NAME} FROM ${PersistentIdentityService.NAME_TO_HASH_TABLE_NAME} WHERE pk_hash=?")
+                val nameToHashStatement = connection.prepareStatement("SELECT name FROM node_named_identities WHERE pk_hash=?")
                nameToHashStatement.setString(1, it.owningKey.toStringShort())
                val nameToHashResultSet = nameToHashStatement.executeQuery()

--- a/node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt
+++ b/node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt
@ -0,0 +1,78 @@
+package net.corda.node.migration
+
+import net.corda.core.schemas.MappedSchema
+import net.corda.core.utilities.MAX_HASH_HEX_SIZE
+import org.apache.commons.lang3.ArrayUtils
+import org.hibernate.annotations.Type
+import javax.persistence.Column
+import javax.persistence.Entity
+import javax.persistence.Id
+import javax.persistence.Table
+
+object MigrationTestSchema
+
+/**
+ * Schema definition for testing PersistentIdentityService custom migration scripts at the moment when scripts were written.
+ * Used to break dependency on the latest PersistentIdentityService which schema version may be different.
+ *
+ * This will allow:
+ * - to fix the position of relevant scripts in the node-core.changelog-master.xml (instead of placing them at the end)
+ * - to perform further modifications of PersistentIdentityService schema without impacting existing migration scripts and their tests
+ */
+object IdentityTestSchemaV1 : MappedSchema(
+        schemaFamily = MigrationTestSchema::class.java,
+        version = 1,
+        mappedTypes = listOf(
+                NodeIdentities::class.java,
+                NodeNamedIdentities::class.java,
+                NodeIdentitiesNoCert::class.java,
+                NodeHashToKey::class.java
+        )
+) {
+    @Entity
+    @Table(name = "node_identities")
+    class NodeIdentities(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Type(type = "corda-blob")
+            @Column(name = "identity_value", nullable = false)
+            var identity: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
+    )
+
+    @Entity
+    @Table(name = "node_named_identities")
+    class NodeNamedIdentities(
+            @Id
+            @Suppress("MagicNumber") // database column width
+            @Column(name = "name", length = 128, nullable = false)
+            var name: String = "",
+
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = ""
+    )
+
+    @Entity
+    @Table(name = "node_identities_no_cert")
+    class NodeIdentitiesNoCert(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Column(name = "name", length = 128, nullable = false)
+            var name: String = ""
+    )
+
+    @Entity
+    @Table(name = "node_hash_to_key")
+    class NodeHashToKey(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Type(type = "corda-blob")
+            @Column(name = "public_key", nullable = false)
+            var publicKey: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
+    )
+}
--- a/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt
+++ b/node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt
@ -1,48 +1,35 @@
 package net.corda.node.migration

-import liquibase.database.Database
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
+import liquibase.database.core.H2Database
 import liquibase.database.jvm.JdbcConnection
-import net.corda.core.crypto.SecureHash
-import net.corda.core.identity.CordaX500Name
+import net.corda.core.crypto.Crypto
+import net.corda.core.crypto.toStringShort
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.internal.hash
-import net.corda.core.internal.signWithCert
-import net.corda.core.node.NetworkParameters
-import net.corda.core.node.NotaryInfo
-import net.corda.node.internal.DBNetworkParametersStorage
-import net.corda.node.migration.VaultStateMigrationTest.Companion.CHARLIE
-import net.corda.node.migration.VaultStateMigrationTest.Companion.DUMMY_NOTARY
-import net.corda.node.services.identity.PersistentIdentityService
-import net.corda.node.services.keys.BasicHSMKeyManagementService
-import net.corda.nodeapi.internal.crypto.X509Utilities
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.node.services.api.SchemaService
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
+import net.corda.nodeapi.internal.persistence.SchemaMigration
 import net.corda.testing.core.*
 import net.corda.testing.internal.configureDatabase
 import net.corda.testing.node.MockServices
-import net.corda.testing.node.makeTestIdentityService
+import org.assertj.core.api.Assertions.assertThat
 import org.junit.After
 import org.junit.Before
 import org.junit.ClassRule
 import org.junit.Test
-import org.mockito.Mockito
-import java.security.KeyPair
-import java.sql.Connection
-import java.time.Clock
-import java.time.Duration
-import java.util.*

-class PersistentIdentityMigrationNewTableTest{
+class PersistentIdentityMigrationNewTableTest {
    companion object {
        val alice = TestIdentity(ALICE_NAME, 70)
        val bankOfCorda = TestIdentity(BOC_NAME)
        val bob = TestIdentity(BOB_NAME, 80)
-        val dummyCashIssuer = TestIdentity(CordaX500Name("Snake Oil Issuer", "London", "GB"), 10)
        val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20)
        val ALICE_IDENTITY get() = alice.identity
        val BOB_IDENTITY get() = bob.identity
        val BOC_IDENTITY get() = bankOfCorda.identity
-        val BOC_KEY get() = bankOfCorda.keyPair
        val bob2 = TestIdentity(BOB_NAME, 40)
        val BOB2_IDENTITY = bob2.identity

@ -51,122 +38,62 @@ class PersistentIdentityMigrationNewTableTest{
        val testSerialization = SerializationEnvironmentRule()
    }

-    lateinit var liquidBaseDB: Database
+    lateinit var liquibaseDB: H2Database
    lateinit var cordaDB: CordaPersistence
-    lateinit var notaryServices: MockServices

    @Before
    fun setUp() {
-        val identityService = makeTestIdentityService(PersistentIdentityMigrationNewTableTest.dummyNotary.identity, BOB_IDENTITY, ALICE_IDENTITY)
-        notaryServices = MockServices(listOf("net.corda.finance.contracts"), dummyNotary, identityService, dummyCashIssuer.keyPair, BOC_KEY)
+        val schemaService = rigorousMock<SchemaService>()
+        doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
+        System.setProperty(SchemaMigration.NODE_X500_NAME, BOB_IDENTITY.name.toString())

-        // Runs migration tasks
        cordaDB = configureDatabase(
                MockServices.makeTestDataSourceProperties(),
                DatabaseConfig(),
-                notaryServices.identityService::wellKnownPartyFromX500Name,
-                notaryServices.identityService::wellKnownPartyFromAnonymous,
+                { null },
+                { null },
+                schemaService = schemaService,
+                internalSchemas = setOf(),
                ourName = BOB_IDENTITY.name)
-        val liquidbaseConnection = Mockito.mock(JdbcConnection::class.java)
-        Mockito.`when`(liquidbaseConnection.url).thenReturn(cordaDB.jdbcUrl)
-        Mockito.`when`(liquidbaseConnection.wrappedConnection).thenReturn(cordaDB.dataSource.connection)
-        liquidBaseDB = Mockito.mock(Database::class.java)
-        Mockito.`when`(liquidBaseDB.connection).thenReturn(liquidbaseConnection)
-
-        cordaDB.dataSource.connection
-        saveOurKeys(listOf(bob.keyPair, bob2.keyPair))
-        saveAllIdentities(listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY))
-        addNetworkParameters()
+        liquibaseDB = H2Database()
+        liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
+        liquibaseDB.isAutoCommit = true
    }

    @After
-    fun `close`() {
+    fun close() {
        cordaDB.close()
    }

-    @Test(timeout=300_000)
-	fun `migrate identities to new table`() {
-         val pkHash = addTestMapping(cordaDB.dataSource.connection, alice)
-         PersistentIdentityMigrationNewTable()
-         verifyTestMigration(cordaDB.dataSource.connection, pkHash, alice.name.toString())
+    @Test(timeout = 300_000)
+    fun `migrate identities to new table`() {
+        val identities = listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY)
+        saveAllIdentities(identities)
+
+        PersistentIdentityMigrationNewTable().execute(liquibaseDB)
+
+        val expectedParties = identities.map { it.owningKey.toStringShort() to it.toString() }
+        val actualParties = selectAll<IdentityTestSchemaV1.NodeIdentitiesNoCert>().map { it.publicKeyHash to it.name }
+        assertThat(actualParties).isEqualTo(expectedParties)
+
+        val expectedKeys = listOf(ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity).map { it.owningKey.toStringShort() to it.owningKey }
+        val actualKeys = selectAll<IdentityTestSchemaV1.NodeHashToKey>().map { it.publicKeyHash to Crypto.decodePublicKey(it.publicKey) }
+        assertThat(actualKeys).isEqualTo(expectedKeys)
    }

    private fun saveAllIdentities(identities: List<PartyAndCertificate>) {
        cordaDB.transaction {
            identities.forEach {
-                session.save(PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded))
+                session.save(IdentityTestSchemaV1.NodeIdentities(it.owningKey.toStringShort(), it.certPath.encoded))
            }
        }
    }

-    private fun saveOurKeys(keys: List<KeyPair>) {
-        cordaDB.transaction {
-            keys.forEach {
-                val persistentKey = BasicHSMKeyManagementService.PersistentKey(it.public, it.private)
-                session.save(persistentKey)
-            }
-        }
-    }
-
-    private fun addNetworkParameters() {
-        cordaDB.transaction {
-            val clock = Clock.systemUTC()
-            val params = NetworkParameters(
-                    1,
-                    listOf(NotaryInfo(DUMMY_NOTARY, false), NotaryInfo(CHARLIE, false)),
-                    1,
-                    1,
-                    clock.instant(),
-                    1,
-                    mapOf(),
-                    Duration.ZERO,
-                    mapOf()
-            )
-            val signedParams = params.signWithCert(bob.keyPair.private, BOB_IDENTITY.certificate)
-            val persistentParams = DBNetworkParametersStorage.PersistentNetworkParameters(
-                    SecureHash.allOnesHash.toString(),
-                    params.epoch,
-                    signedParams.raw.bytes,
-                    signedParams.sig.bytes,
-                    signedParams.sig.by.encoded,
-                    X509Utilities.buildCertPath(signedParams.sig.parentCertsChain).encoded
-            )
-            session.save(persistentParams)
-        }
-    }
-
-        private fun addTestMapping(connection: Connection, testIdentity: TestIdentity): String {
-        val pkHash = UUID.randomUUID().toString()
-        val cert = testIdentity.identity.certPath.encoded
-
-        connection.prepareStatement("INSERT INTO node_identities (pk_hash, identity_value) VALUES (?,?)").use {
-            it.setString(1, pkHash)
-            it.setBytes(2, cert)
-            it.executeUpdate()
-        }
-        return pkHash
-    }
-
-//    private fun deleteTestMapping(connection: Connection, pkHash: String) {
-//        connection.prepareStatement("DELETE FROM node_identities WHERE pk_hash = ?").use {
-//            it.setString(1, pkHash)
-//            it.executeUpdate()
-//        }
-//    }
-
-    private fun verifyTestMigration(connection: Connection, pk: String, name: String) {
-        connection.createStatement().use {
-            try {
-                val rs = it.executeQuery("SELECT (pk_hash, name) FROM node_identities_no_cert")
-                while (rs.next()) {
-                    val result = rs.getString(1)
-                    require(result.contains(pk))
-                    require(result.contains(name))
-                }
-                rs.close()
-            } catch (e: Exception) {
-                println(e.localizedMessage)
-            }
+    private inline fun <reified T> selectAll(): List<T> {
+        return cordaDB.transaction {
+            val criteria = session.criteriaBuilder.createQuery(T::class.java)
+            criteria.select(criteria.from(T::class.java))
+            session.createQuery(criteria).resultList
        }
    }
 }