Merge pull request #6266 from corda/denis/CORDA-3805-custom-migration-scripts

CORDA-3805: cut dependency from PersistentIdentityService for custom migration scripts

node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigration.kt

@@ -9,16 +9,13 @@ import liquibase.statement.core.UpdateStatement
 import net.corda.core.crypto.toStringShort
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.utilities.contextLogger
-import net.corda.node.services.identity.PersistentIdentityService
 import net.corda.nodeapi.internal.crypto.X509CertificateFactory
 
 class PersistentIdentityMigration : CustomSqlChange {
 
     companion object {
-        private val logger = contextLogger()
+        const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = "node_identities"
-        const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME
+        const val X500_NAME_TO_PUB_KEY_HASH_TABLE = "node_named_identities"
-        const val X500_NAME_TO_PUB_KEY_HASH_TABLE = PersistentIdentityService.NAME_TO_HASH_TABLE_NAME
     }
 
     override fun validate(database: Database?): ValidationErrors? {

node/src/main/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTable.kt

@@ -1,28 +1,27 @@
 package net.corda.node.migration
 
+import liquibase.change.custom.CustomTaskChange
 import liquibase.database.Database
 import liquibase.database.jvm.JdbcConnection
 import liquibase.exception.ValidationErrors
 import liquibase.resource.ResourceAccessor
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.schemas.MappedSchema
 import net.corda.core.utilities.contextLogger
-import net.corda.node.internal.DBNetworkParametersStorage
-import net.corda.node.services.identity.PersistentIdentityService
-import net.corda.node.services.persistence.DBTransactionStorage
-import net.corda.node.services.persistence.NodeAttachmentService
 import net.corda.nodeapi.internal.crypto.X509CertificateFactory
+import net.corda.nodeapi.internal.persistence.SchemaMigration
 import java.security.PublicKey
 
 /**
  * Migration that reads data from the [PersistentIdentityCert] table, extracts the parameters required to insert into the [PersistentIdentity] table.
  */
-class PersistentIdentityMigrationNewTable : CordaMigration() {
+class PersistentIdentityMigrationNewTable : CustomTaskChange {
     companion object {
         private val logger = contextLogger()
     }
 
+    private lateinit var ourName: CordaX500Name
+
     override fun execute(database: Database?) {
         logger.info("Migrating persistent identities with certificates table into persistent table with no certificate data.")
 
@@ -30,7 +29,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
             logger.error("Cannot migrate persistent identities: Liquibase failed to provide a suitable database connection")
             throw PersistentIdentitiesMigrationException("Cannot migrate persistent identities as liquibase failed to provide a suitable database connection")
         }
-        initialiseNodeServices(database, setOf(PersistentIdentitiesMigrationSchemaBuilder.getMappedSchema()))
+        ourName = CordaX500Name.parse(System.getProperty(SchemaMigration.NODE_X500_NAME))
 
         val connection = database.connection as JdbcConnection
         val hashToKeyAndName = extractKeyAndName(connection)
@@ -68,7 +67,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
             it.setString(2, name.toString())
             it.executeUpdate()
         }
-        if (name !in identityService.ourNames) {
+        if (name != ourName) {
             connection.prepareStatement("INSERT INTO node_hash_to_key (pk_hash, public_key) VALUES (?,?)").use {
                 it.setString(1, publicKeyHash)
                 it.setBytes(2, publicKey.encoded)
@@ -92,26 +91,4 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
     }
 }
 
-/**
- * A minimal set of schema for retrieving data from the database.
- *
- * Note that adding an extra schema here may cause migrations to fail if it ends up creating a table before the same table
- * is created in a migration script. As such, this migration must be run after the tables for the following have been created (and,
- * if they are removed in the future, before they are deleted).
- */
-object PersistentIdentitiesMigrationSchema
-
-object PersistentIdentitiesMigrationSchemaBuilder {
-    fun getMappedSchema() =
-            MappedSchema(schemaFamily = PersistentIdentitiesMigrationSchema.javaClass, version = 1,
-                    mappedTypes = listOf(
-                            DBTransactionStorage.DBTransaction::class.java,
-                            PersistentIdentityService.PersistentPublicKeyHashToCertificate::class.java,
-                            PersistentIdentityService.PersistentPartyToPublicKeyHash::class.java,
-                            PersistentIdentityService.PersistentPublicKeyHashToParty::class.java,
-                            PersistentIdentityService.PersistentHashToPublicKey::class.java,
-                            NodeAttachmentService.DBAttachment::class.java,
-                            DBNetworkParametersStorage.PersistentNetworkParameters::class.java
-                    ))
-}
 class PersistentIdentitiesMigrationException(msg: String, cause: Exception? = null) : Exception(msg, cause)

node/src/main/resources/migration/node-core.changelog-master.xml

@@ -24,13 +24,12 @@
     <include file="migration/node-core.changelog-v13.xml"/>
     <!-- This change should be done before the v14-data migration. -->
     <include file="migration/node-core.changelog-v15.xml"/>
+    <include file="migration/node-core.changelog-v14-data.xml"/>
     <include file="migration/node-core.changelog-v16.xml"/>
 
     <!-- This must run after node-core.changelog-init.xml, to prevent database columns being created twice. -->
     <include file="migration/vault-schema.changelog-v9.xml"/>
 
-    <include file="migration/node-core.changelog-v14-data.xml"/>
-
     <include file="migration/node-core.changelog-v19.xml"/>
     <include file="migration/node-core.changelog-v19-postgres.xml"/>
     <include file="migration/node-core.changelog-v19-keys.xml"/>

node/src/test/kotlin/net/corda/node/migration/IdentityServiceToStringShortMigrationTest.kt

@@ -1,5 +1,7 @@
 package net.corda.node.migration
 
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
 import liquibase.database.core.H2Database
 import liquibase.database.jvm.JdbcConnection
 import net.corda.core.crypto.toStringShort
@@ -7,7 +9,8 @@ import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.PartyAndCertificate
 import net.corda.core.internal.hash
 import net.corda.core.utilities.contextLogger
-import net.corda.node.services.identity.PersistentIdentityService
+import net.corda.coretesting.internal.rigorousMock
+import net.corda.node.services.api.SchemaService
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
 import net.corda.nodeapi.internal.persistence.contextTransactionOrNull
@@ -44,11 +47,16 @@ class IdentityServiceToStringShortMigrationTest {
 
     @Before
     fun setUp() {
+        val schemaService = rigorousMock<SchemaService>()
+        doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
+
         cordaDB = configureDatabase(
                 makeTestDataSourceProperties(),
                 DatabaseConfig(),
                 { null },
                 { null },
+                schemaService = schemaService,
+                internalSchemas = setOf(),
                 ourName = BOB_IDENTITY.name)
         liquibaseDB = H2Database()
         liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
@@ -66,8 +74,8 @@ class IdentityServiceToStringShortMigrationTest {
         cordaDB.transaction {
             val groupedIdentities = identities.groupBy { it.name }
             groupedIdentities.forEach { name, certs ->
-                val persistentIDs = certs.map { PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded) }
+                val persistentIDs = certs.map { IdentityTestSchemaV1.NodeIdentities(it.owningKey.hash.toString(), it.certPath.encoded) }
-                val persistentName = PersistentIdentityService.PersistentPartyToPublicKeyHash(name.toString(), certs.first().owningKey.hash.toString())
+                val persistentName = IdentityTestSchemaV1.NodeNamedIdentities(name.toString(), certs.first().owningKey.hash.toString())
                 persistentIDs.forEach {
                     session.persist(it)
                 }
@@ -87,7 +95,7 @@ class IdentityServiceToStringShortMigrationTest {
         identities.forEach {
             logger.info("Checking: ${it.name}")
             cordaDB.transaction {
-                val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT ${PersistentIdentityService.PK_HASH_COLUMN_NAME} FROM ${PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME} WHERE pk_hash=?")
+                val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT pk_hash FROM node_identities WHERE pk_hash=?")
                 hashToIdentityStatement.setString(1, it.owningKey.toStringShort())
                 val hashToIdentityResultSet = hashToIdentityStatement.executeQuery()
 
@@ -96,7 +104,7 @@ class IdentityServiceToStringShortMigrationTest {
                 //check that the pk_hash actually matches what we expect (kinda redundant, but deserializing the whole PartyAndCertificate feels like overkill)
                 Assert.assertThat(hashToIdentityResultSet.getString(1), `is`(it.owningKey.toStringShort()))
 
-                val nameToHashStatement = connection.prepareStatement("SELECT ${PersistentIdentityService.NAME_COLUMN_NAME} FROM ${PersistentIdentityService.NAME_TO_HASH_TABLE_NAME} WHERE pk_hash=?")
+                val nameToHashStatement = connection.prepareStatement("SELECT name FROM node_named_identities WHERE pk_hash=?")
                 nameToHashStatement.setString(1, it.owningKey.toStringShort())
                 val nameToHashResultSet = nameToHashStatement.executeQuery()
 

node/src/test/kotlin/net/corda/node/migration/MigrationTestSchema.kt

@@ -0,0 +1,78 @@
+package net.corda.node.migration
+
+import net.corda.core.schemas.MappedSchema
+import net.corda.core.utilities.MAX_HASH_HEX_SIZE
+import org.apache.commons.lang3.ArrayUtils
+import org.hibernate.annotations.Type
+import javax.persistence.Column
+import javax.persistence.Entity
+import javax.persistence.Id
+import javax.persistence.Table
+
+object MigrationTestSchema
+
+/**
+ * Schema definition for testing PersistentIdentityService custom migration scripts at the moment when scripts were written.
+ * Used to break dependency on the latest PersistentIdentityService which schema version may be different.
+ *
+ * This will allow:
+ * - to fix the position of relevant scripts in the node-core.changelog-master.xml (instead of placing them at the end)
+ * - to perform further modifications of PersistentIdentityService schema without impacting existing migration scripts and their tests
+ */
+object IdentityTestSchemaV1 : MappedSchema(
+        schemaFamily = MigrationTestSchema::class.java,
+        version = 1,
+        mappedTypes = listOf(
+                NodeIdentities::class.java,
+                NodeNamedIdentities::class.java,
+                NodeIdentitiesNoCert::class.java,
+                NodeHashToKey::class.java
+        )
+) {
+    @Entity
+    @Table(name = "node_identities")
+    class NodeIdentities(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Type(type = "corda-blob")
+            @Column(name = "identity_value", nullable = false)
+            var identity: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
+    )
+
+    @Entity
+    @Table(name = "node_named_identities")
+    class NodeNamedIdentities(
+            @Id
+            @Suppress("MagicNumber") // database column width
+            @Column(name = "name", length = 128, nullable = false)
+            var name: String = "",
+
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = ""
+    )
+
+    @Entity
+    @Table(name = "node_identities_no_cert")
+    class NodeIdentitiesNoCert(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Column(name = "name", length = 128, nullable = false)
+            var name: String = ""
+    )
+
+    @Entity
+    @Table(name = "node_hash_to_key")
+    class NodeHashToKey(
+            @Id
+            @Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
+            var publicKeyHash: String = "",
+
+            @Type(type = "corda-blob")
+            @Column(name = "public_key", nullable = false)
+            var publicKey: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
+    )
+}

node/src/test/kotlin/net/corda/node/migration/PersistentIdentityMigrationNewTableTest.kt

@@ -1,48 +1,35 @@
 package net.corda.node.migration
 
-import liquibase.database.Database
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
+import liquibase.database.core.H2Database
 import liquibase.database.jvm.JdbcConnection
-import net.corda.core.crypto.SecureHash
+import net.corda.core.crypto.Crypto
-import net.corda.core.identity.CordaX500Name
+import net.corda.core.crypto.toStringShort
 import net.corda.core.identity.PartyAndCertificate
-import net.corda.core.internal.hash
+import net.corda.coretesting.internal.rigorousMock
-import net.corda.core.internal.signWithCert
+import net.corda.node.services.api.SchemaService
-import net.corda.core.node.NetworkParameters
-import net.corda.core.node.NotaryInfo
-import net.corda.node.internal.DBNetworkParametersStorage
-import net.corda.node.migration.VaultStateMigrationTest.Companion.CHARLIE
-import net.corda.node.migration.VaultStateMigrationTest.Companion.DUMMY_NOTARY
-import net.corda.node.services.identity.PersistentIdentityService
-import net.corda.node.services.keys.BasicHSMKeyManagementService
-import net.corda.nodeapi.internal.crypto.X509Utilities
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
+import net.corda.nodeapi.internal.persistence.SchemaMigration
 import net.corda.testing.core.*
 import net.corda.testing.internal.configureDatabase
 import net.corda.testing.node.MockServices
-import net.corda.testing.node.makeTestIdentityService
+import org.assertj.core.api.Assertions.assertThat
 import org.junit.After
 import org.junit.Before
 import org.junit.ClassRule
 import org.junit.Test
-import org.mockito.Mockito
-import java.security.KeyPair
-import java.sql.Connection
-import java.time.Clock
-import java.time.Duration
-import java.util.*
 
-class PersistentIdentityMigrationNewTableTest{
+class PersistentIdentityMigrationNewTableTest {
     companion object {
         val alice = TestIdentity(ALICE_NAME, 70)
         val bankOfCorda = TestIdentity(BOC_NAME)
         val bob = TestIdentity(BOB_NAME, 80)
-        val dummyCashIssuer = TestIdentity(CordaX500Name("Snake Oil Issuer", "London", "GB"), 10)
         val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20)
         val ALICE_IDENTITY get() = alice.identity
         val BOB_IDENTITY get() = bob.identity
         val BOC_IDENTITY get() = bankOfCorda.identity
-        val BOC_KEY get() = bankOfCorda.keyPair
         val bob2 = TestIdentity(BOB_NAME, 40)
         val BOB2_IDENTITY = bob2.identity
 
@@ -51,122 +38,62 @@ class PersistentIdentityMigrationNewTableTest{
         val testSerialization = SerializationEnvironmentRule()
     }
 
-    lateinit var liquidBaseDB: Database
+    lateinit var liquibaseDB: H2Database
     lateinit var cordaDB: CordaPersistence
-    lateinit var notaryServices: MockServices
 
     @Before
     fun setUp() {
-        val identityService = makeTestIdentityService(PersistentIdentityMigrationNewTableTest.dummyNotary.identity, BOB_IDENTITY, ALICE_IDENTITY)
+        val schemaService = rigorousMock<SchemaService>()
-        notaryServices = MockServices(listOf("net.corda.finance.contracts"), dummyNotary, identityService, dummyCashIssuer.keyPair, BOC_KEY)
+        doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
+        System.setProperty(SchemaMigration.NODE_X500_NAME, BOB_IDENTITY.name.toString())
 
-        // Runs migration tasks
         cordaDB = configureDatabase(
                 MockServices.makeTestDataSourceProperties(),
                 DatabaseConfig(),
-                notaryServices.identityService::wellKnownPartyFromX500Name,
+                { null },
-                notaryServices.identityService::wellKnownPartyFromAnonymous,
+                { null },
+                schemaService = schemaService,
+                internalSchemas = setOf(),
                 ourName = BOB_IDENTITY.name)
-        val liquidbaseConnection = Mockito.mock(JdbcConnection::class.java)
+        liquibaseDB = H2Database()
-        Mockito.`when`(liquidbaseConnection.url).thenReturn(cordaDB.jdbcUrl)
+        liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
-        Mockito.`when`(liquidbaseConnection.wrappedConnection).thenReturn(cordaDB.dataSource.connection)
+        liquibaseDB.isAutoCommit = true
-        liquidBaseDB = Mockito.mock(Database::class.java)
-        Mockito.`when`(liquidBaseDB.connection).thenReturn(liquidbaseConnection)
-
-        cordaDB.dataSource.connection
-        saveOurKeys(listOf(bob.keyPair, bob2.keyPair))
-        saveAllIdentities(listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY))
-        addNetworkParameters()
     }
 
     @After
-    fun `close`() {
+    fun close() {
         cordaDB.close()
     }
 
-    @Test(timeout=300_000)
+    @Test(timeout = 300_000)
-	fun `migrate identities to new table`() {
+    fun `migrate identities to new table`() {
-         val pkHash = addTestMapping(cordaDB.dataSource.connection, alice)
+        val identities = listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY)
-         PersistentIdentityMigrationNewTable()
+        saveAllIdentities(identities)
-         verifyTestMigration(cordaDB.dataSource.connection, pkHash, alice.name.toString())
+
+        PersistentIdentityMigrationNewTable().execute(liquibaseDB)
+
+        val expectedParties = identities.map { it.owningKey.toStringShort() to it.toString() }
+        val actualParties = selectAll<IdentityTestSchemaV1.NodeIdentitiesNoCert>().map { it.publicKeyHash to it.name }
+        assertThat(actualParties).isEqualTo(expectedParties)
+
+        val expectedKeys = listOf(ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity).map { it.owningKey.toStringShort() to it.owningKey }
+        val actualKeys = selectAll<IdentityTestSchemaV1.NodeHashToKey>().map { it.publicKeyHash to Crypto.decodePublicKey(it.publicKey) }
+        assertThat(actualKeys).isEqualTo(expectedKeys)
     }
 
     private fun saveAllIdentities(identities: List<PartyAndCertificate>) {
         cordaDB.transaction {
             identities.forEach {
-                session.save(PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded))
+                session.save(IdentityTestSchemaV1.NodeIdentities(it.owningKey.toStringShort(), it.certPath.encoded))
             }
         }
     }
 
-    private fun saveOurKeys(keys: List<KeyPair>) {
+    private inline fun <reified T> selectAll(): List<T> {
-        cordaDB.transaction {
+        return cordaDB.transaction {
-            keys.forEach {
+            val criteria = session.criteriaBuilder.createQuery(T::class.java)
-                val persistentKey = BasicHSMKeyManagementService.PersistentKey(it.public, it.private)
+            criteria.select(criteria.from(T::class.java))
-                session.save(persistentKey)
+            session.createQuery(criteria).resultList
-            }
-        }
-    }
-
-    private fun addNetworkParameters() {
-        cordaDB.transaction {
-            val clock = Clock.systemUTC()
-            val params = NetworkParameters(
-                    1,
-                    listOf(NotaryInfo(DUMMY_NOTARY, false), NotaryInfo(CHARLIE, false)),
-                    1,
-                    1,
-                    clock.instant(),
-                    1,
-                    mapOf(),
-                    Duration.ZERO,
-                    mapOf()
-            )
-            val signedParams = params.signWithCert(bob.keyPair.private, BOB_IDENTITY.certificate)
-            val persistentParams = DBNetworkParametersStorage.PersistentNetworkParameters(
-                    SecureHash.allOnesHash.toString(),
-                    params.epoch,
-                    signedParams.raw.bytes,
-                    signedParams.sig.bytes,
-                    signedParams.sig.by.encoded,
-                    X509Utilities.buildCertPath(signedParams.sig.parentCertsChain).encoded
-            )
-            session.save(persistentParams)
-        }
-    }
-
-        private fun addTestMapping(connection: Connection, testIdentity: TestIdentity): String {
-        val pkHash = UUID.randomUUID().toString()
-        val cert = testIdentity.identity.certPath.encoded
-
-        connection.prepareStatement("INSERT INTO node_identities (pk_hash, identity_value) VALUES (?,?)").use {
-            it.setString(1, pkHash)
-            it.setBytes(2, cert)
-            it.executeUpdate()
-        }
-        return pkHash
-    }
-
-//    private fun deleteTestMapping(connection: Connection, pkHash: String) {
-//        connection.prepareStatement("DELETE FROM node_identities WHERE pk_hash = ?").use {
-//            it.setString(1, pkHash)
-//            it.executeUpdate()
-//        }
-//    }
-
-    private fun verifyTestMigration(connection: Connection, pk: String, name: String) {
-        connection.createStatement().use {
-            try {
-                val rs = it.executeQuery("SELECT (pk_hash, name) FROM node_identities_no_cert")
-                while (rs.next()) {
-                    val result = rs.getString(1)
-                    require(result.contains(pk))
-                    require(result.contains(name))
-                }
-                rs.close()
-            } catch (e: Exception) {
-                println(e.localizedMessage)
-            }
         }
     }
 }