Merge pull request #6266 from corda/denis/CORDA-3805-custom-migration-scripts

CORDA-3805: cut dependency from PersistentIdentityService for custom migration scripts
This commit is contained in:
Denis Rekalov 2020-06-04 14:20:26 +01:00 committed by GitHub
commit 45614cf29e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 143 additions and 157 deletions

View File

@ -9,16 +9,13 @@ import liquibase.statement.core.UpdateStatement
import net.corda.core.crypto.toStringShort
import net.corda.core.identity.CordaX500Name
import net.corda.core.identity.PartyAndCertificate
import net.corda.core.utilities.contextLogger
import net.corda.node.services.identity.PersistentIdentityService
import net.corda.nodeapi.internal.crypto.X509CertificateFactory
class PersistentIdentityMigration : CustomSqlChange {
companion object {
private val logger = contextLogger()
const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME
const val X500_NAME_TO_PUB_KEY_HASH_TABLE = PersistentIdentityService.NAME_TO_HASH_TABLE_NAME
const val PUB_KEY_HASH_TO_PARTY_AND_CERT_TABLE = "node_identities"
const val X500_NAME_TO_PUB_KEY_HASH_TABLE = "node_named_identities"
}
override fun validate(database: Database?): ValidationErrors? {

View File

@ -1,28 +1,27 @@
package net.corda.node.migration
import liquibase.change.custom.CustomTaskChange
import liquibase.database.Database
import liquibase.database.jvm.JdbcConnection
import liquibase.exception.ValidationErrors
import liquibase.resource.ResourceAccessor
import net.corda.core.identity.CordaX500Name
import net.corda.core.identity.PartyAndCertificate
import net.corda.core.schemas.MappedSchema
import net.corda.core.utilities.contextLogger
import net.corda.node.internal.DBNetworkParametersStorage
import net.corda.node.services.identity.PersistentIdentityService
import net.corda.node.services.persistence.DBTransactionStorage
import net.corda.node.services.persistence.NodeAttachmentService
import net.corda.nodeapi.internal.crypto.X509CertificateFactory
import net.corda.nodeapi.internal.persistence.SchemaMigration
import java.security.PublicKey
/**
* Migration that reads data from the [PersistentIdentityCert] table, extracts the parameters required to insert into the [PersistentIdentity] table.
*/
class PersistentIdentityMigrationNewTable : CordaMigration() {
class PersistentIdentityMigrationNewTable : CustomTaskChange {
companion object {
private val logger = contextLogger()
}
private lateinit var ourName: CordaX500Name
override fun execute(database: Database?) {
logger.info("Migrating persistent identities with certificates table into persistent table with no certificate data.")
@ -30,7 +29,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
logger.error("Cannot migrate persistent identities: Liquibase failed to provide a suitable database connection")
throw PersistentIdentitiesMigrationException("Cannot migrate persistent identities as liquibase failed to provide a suitable database connection")
}
initialiseNodeServices(database, setOf(PersistentIdentitiesMigrationSchemaBuilder.getMappedSchema()))
ourName = CordaX500Name.parse(System.getProperty(SchemaMigration.NODE_X500_NAME))
val connection = database.connection as JdbcConnection
val hashToKeyAndName = extractKeyAndName(connection)
@ -68,7 +67,7 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
it.setString(2, name.toString())
it.executeUpdate()
}
if (name !in identityService.ourNames) {
if (name != ourName) {
connection.prepareStatement("INSERT INTO node_hash_to_key (pk_hash, public_key) VALUES (?,?)").use {
it.setString(1, publicKeyHash)
it.setBytes(2, publicKey.encoded)
@ -92,26 +91,4 @@ class PersistentIdentityMigrationNewTable : CordaMigration() {
}
}
/**
* A minimal set of schema for retrieving data from the database.
*
* Note that adding an extra schema here may cause migrations to fail if it ends up creating a table before the same table
* is created in a migration script. As such, this migration must be run after the tables for the following have been created (and,
* if they are removed in the future, before they are deleted).
*/
object PersistentIdentitiesMigrationSchema
object PersistentIdentitiesMigrationSchemaBuilder {
fun getMappedSchema() =
MappedSchema(schemaFamily = PersistentIdentitiesMigrationSchema.javaClass, version = 1,
mappedTypes = listOf(
DBTransactionStorage.DBTransaction::class.java,
PersistentIdentityService.PersistentPublicKeyHashToCertificate::class.java,
PersistentIdentityService.PersistentPartyToPublicKeyHash::class.java,
PersistentIdentityService.PersistentPublicKeyHashToParty::class.java,
PersistentIdentityService.PersistentHashToPublicKey::class.java,
NodeAttachmentService.DBAttachment::class.java,
DBNetworkParametersStorage.PersistentNetworkParameters::class.java
))
}
class PersistentIdentitiesMigrationException(msg: String, cause: Exception? = null) : Exception(msg, cause)

View File

@ -24,13 +24,12 @@
<include file="migration/node-core.changelog-v13.xml"/>
<!-- This change should be done before the v14-data migration. -->
<include file="migration/node-core.changelog-v15.xml"/>
<include file="migration/node-core.changelog-v14-data.xml"/>
<include file="migration/node-core.changelog-v16.xml"/>
<!-- This must run after node-core.changelog-init.xml, to prevent database columns being created twice. -->
<include file="migration/vault-schema.changelog-v9.xml"/>
<include file="migration/node-core.changelog-v14-data.xml"/>
<include file="migration/node-core.changelog-v19.xml"/>
<include file="migration/node-core.changelog-v19-postgres.xml"/>
<include file="migration/node-core.changelog-v19-keys.xml"/>

View File

@ -1,5 +1,7 @@
package net.corda.node.migration
import com.nhaarman.mockito_kotlin.doReturn
import com.nhaarman.mockito_kotlin.whenever
import liquibase.database.core.H2Database
import liquibase.database.jvm.JdbcConnection
import net.corda.core.crypto.toStringShort
@ -7,7 +9,8 @@ import net.corda.core.identity.CordaX500Name
import net.corda.core.identity.PartyAndCertificate
import net.corda.core.internal.hash
import net.corda.core.utilities.contextLogger
import net.corda.node.services.identity.PersistentIdentityService
import net.corda.coretesting.internal.rigorousMock
import net.corda.node.services.api.SchemaService
import net.corda.nodeapi.internal.persistence.CordaPersistence
import net.corda.nodeapi.internal.persistence.DatabaseConfig
import net.corda.nodeapi.internal.persistence.contextTransactionOrNull
@ -44,11 +47,16 @@ class IdentityServiceToStringShortMigrationTest {
@Before
fun setUp() {
val schemaService = rigorousMock<SchemaService>()
doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
cordaDB = configureDatabase(
makeTestDataSourceProperties(),
DatabaseConfig(),
{ null },
{ null },
schemaService = schemaService,
internalSchemas = setOf(),
ourName = BOB_IDENTITY.name)
liquibaseDB = H2Database()
liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
@ -66,8 +74,8 @@ class IdentityServiceToStringShortMigrationTest {
cordaDB.transaction {
val groupedIdentities = identities.groupBy { it.name }
groupedIdentities.forEach { name, certs ->
val persistentIDs = certs.map { PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded) }
val persistentName = PersistentIdentityService.PersistentPartyToPublicKeyHash(name.toString(), certs.first().owningKey.hash.toString())
val persistentIDs = certs.map { IdentityTestSchemaV1.NodeIdentities(it.owningKey.hash.toString(), it.certPath.encoded) }
val persistentName = IdentityTestSchemaV1.NodeNamedIdentities(name.toString(), certs.first().owningKey.hash.toString())
persistentIDs.forEach {
session.persist(it)
}
@ -87,7 +95,7 @@ class IdentityServiceToStringShortMigrationTest {
identities.forEach {
logger.info("Checking: ${it.name}")
cordaDB.transaction {
val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT ${PersistentIdentityService.PK_HASH_COLUMN_NAME} FROM ${PersistentIdentityService.HASH_TO_IDENTITY_TABLE_NAME} WHERE pk_hash=?")
val hashToIdentityStatement = database.dataSource.connection.prepareStatement("SELECT pk_hash FROM node_identities WHERE pk_hash=?")
hashToIdentityStatement.setString(1, it.owningKey.toStringShort())
val hashToIdentityResultSet = hashToIdentityStatement.executeQuery()
@ -96,7 +104,7 @@ class IdentityServiceToStringShortMigrationTest {
//check that the pk_hash actually matches what we expect (kinda redundant, but deserializing the whole PartyAndCertificate feels like overkill)
Assert.assertThat(hashToIdentityResultSet.getString(1), `is`(it.owningKey.toStringShort()))
val nameToHashStatement = connection.prepareStatement("SELECT ${PersistentIdentityService.NAME_COLUMN_NAME} FROM ${PersistentIdentityService.NAME_TO_HASH_TABLE_NAME} WHERE pk_hash=?")
val nameToHashStatement = connection.prepareStatement("SELECT name FROM node_named_identities WHERE pk_hash=?")
nameToHashStatement.setString(1, it.owningKey.toStringShort())
val nameToHashResultSet = nameToHashStatement.executeQuery()

View File

@ -0,0 +1,78 @@
package net.corda.node.migration
import net.corda.core.schemas.MappedSchema
import net.corda.core.utilities.MAX_HASH_HEX_SIZE
import org.apache.commons.lang3.ArrayUtils
import org.hibernate.annotations.Type
import javax.persistence.Column
import javax.persistence.Entity
import javax.persistence.Id
import javax.persistence.Table
object MigrationTestSchema
/**
* Schema definition for testing PersistentIdentityService custom migration scripts at the moment when scripts were written.
* Used to break dependency on the latest PersistentIdentityService which schema version may be different.
*
* This will allow:
* - to fix the position of relevant scripts in the node-core.changelog-master.xml (instead of placing them at the end)
* - to perform further modifications of PersistentIdentityService schema without impacting existing migration scripts and their tests
*/
object IdentityTestSchemaV1 : MappedSchema(
schemaFamily = MigrationTestSchema::class.java,
version = 1,
mappedTypes = listOf(
NodeIdentities::class.java,
NodeNamedIdentities::class.java,
NodeIdentitiesNoCert::class.java,
NodeHashToKey::class.java
)
) {
@Entity
@Table(name = "node_identities")
class NodeIdentities(
@Id
@Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
var publicKeyHash: String = "",
@Type(type = "corda-blob")
@Column(name = "identity_value", nullable = false)
var identity: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
)
@Entity
@Table(name = "node_named_identities")
class NodeNamedIdentities(
@Id
@Suppress("MagicNumber") // database column width
@Column(name = "name", length = 128, nullable = false)
var name: String = "",
@Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
var publicKeyHash: String = ""
)
@Entity
@Table(name = "node_identities_no_cert")
class NodeIdentitiesNoCert(
@Id
@Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
var publicKeyHash: String = "",
@Column(name = "name", length = 128, nullable = false)
var name: String = ""
)
@Entity
@Table(name = "node_hash_to_key")
class NodeHashToKey(
@Id
@Column(name = "pk_hash", length = MAX_HASH_HEX_SIZE, nullable = false)
var publicKeyHash: String = "",
@Type(type = "corda-blob")
@Column(name = "public_key", nullable = false)
var publicKey: ByteArray = ArrayUtils.EMPTY_BYTE_ARRAY
)
}

View File

@ -1,48 +1,35 @@
package net.corda.node.migration
import liquibase.database.Database
import com.nhaarman.mockito_kotlin.doReturn
import com.nhaarman.mockito_kotlin.whenever
import liquibase.database.core.H2Database
import liquibase.database.jvm.JdbcConnection
import net.corda.core.crypto.SecureHash
import net.corda.core.identity.CordaX500Name
import net.corda.core.crypto.Crypto
import net.corda.core.crypto.toStringShort
import net.corda.core.identity.PartyAndCertificate
import net.corda.core.internal.hash
import net.corda.core.internal.signWithCert
import net.corda.core.node.NetworkParameters
import net.corda.core.node.NotaryInfo
import net.corda.node.internal.DBNetworkParametersStorage
import net.corda.node.migration.VaultStateMigrationTest.Companion.CHARLIE
import net.corda.node.migration.VaultStateMigrationTest.Companion.DUMMY_NOTARY
import net.corda.node.services.identity.PersistentIdentityService
import net.corda.node.services.keys.BasicHSMKeyManagementService
import net.corda.nodeapi.internal.crypto.X509Utilities
import net.corda.coretesting.internal.rigorousMock
import net.corda.node.services.api.SchemaService
import net.corda.nodeapi.internal.persistence.CordaPersistence
import net.corda.nodeapi.internal.persistence.DatabaseConfig
import net.corda.nodeapi.internal.persistence.SchemaMigration
import net.corda.testing.core.*
import net.corda.testing.internal.configureDatabase
import net.corda.testing.node.MockServices
import net.corda.testing.node.makeTestIdentityService
import org.assertj.core.api.Assertions.assertThat
import org.junit.After
import org.junit.Before
import org.junit.ClassRule
import org.junit.Test
import org.mockito.Mockito
import java.security.KeyPair
import java.sql.Connection
import java.time.Clock
import java.time.Duration
import java.util.*
class PersistentIdentityMigrationNewTableTest {
companion object {
val alice = TestIdentity(ALICE_NAME, 70)
val bankOfCorda = TestIdentity(BOC_NAME)
val bob = TestIdentity(BOB_NAME, 80)
val dummyCashIssuer = TestIdentity(CordaX500Name("Snake Oil Issuer", "London", "GB"), 10)
val dummyNotary = TestIdentity(DUMMY_NOTARY_NAME, 20)
val ALICE_IDENTITY get() = alice.identity
val BOB_IDENTITY get() = bob.identity
val BOC_IDENTITY get() = bankOfCorda.identity
val BOC_KEY get() = bankOfCorda.keyPair
val bob2 = TestIdentity(BOB_NAME, 40)
val BOB2_IDENTITY = bob2.identity
@ -51,122 +38,62 @@ class PersistentIdentityMigrationNewTableTest{
val testSerialization = SerializationEnvironmentRule()
}
lateinit var liquidBaseDB: Database
lateinit var liquibaseDB: H2Database
lateinit var cordaDB: CordaPersistence
lateinit var notaryServices: MockServices
@Before
fun setUp() {
val identityService = makeTestIdentityService(PersistentIdentityMigrationNewTableTest.dummyNotary.identity, BOB_IDENTITY, ALICE_IDENTITY)
notaryServices = MockServices(listOf("net.corda.finance.contracts"), dummyNotary, identityService, dummyCashIssuer.keyPair, BOC_KEY)
val schemaService = rigorousMock<SchemaService>()
doReturn(setOf(IdentityTestSchemaV1)).whenever(schemaService).schemas
System.setProperty(SchemaMigration.NODE_X500_NAME, BOB_IDENTITY.name.toString())
// Runs migration tasks
cordaDB = configureDatabase(
MockServices.makeTestDataSourceProperties(),
DatabaseConfig(),
notaryServices.identityService::wellKnownPartyFromX500Name,
notaryServices.identityService::wellKnownPartyFromAnonymous,
{ null },
{ null },
schemaService = schemaService,
internalSchemas = setOf(),
ourName = BOB_IDENTITY.name)
val liquidbaseConnection = Mockito.mock(JdbcConnection::class.java)
Mockito.`when`(liquidbaseConnection.url).thenReturn(cordaDB.jdbcUrl)
Mockito.`when`(liquidbaseConnection.wrappedConnection).thenReturn(cordaDB.dataSource.connection)
liquidBaseDB = Mockito.mock(Database::class.java)
Mockito.`when`(liquidBaseDB.connection).thenReturn(liquidbaseConnection)
cordaDB.dataSource.connection
saveOurKeys(listOf(bob.keyPair, bob2.keyPair))
saveAllIdentities(listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY))
addNetworkParameters()
liquibaseDB = H2Database()
liquibaseDB.connection = JdbcConnection(cordaDB.dataSource.connection)
liquibaseDB.isAutoCommit = true
}
@After
fun `close`() {
fun close() {
cordaDB.close()
}
@Test(timeout = 300_000)
fun `migrate identities to new table`() {
val pkHash = addTestMapping(cordaDB.dataSource.connection, alice)
PersistentIdentityMigrationNewTable()
verifyTestMigration(cordaDB.dataSource.connection, pkHash, alice.name.toString())
val identities = listOf(BOB_IDENTITY, ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity, BOB2_IDENTITY)
saveAllIdentities(identities)
PersistentIdentityMigrationNewTable().execute(liquibaseDB)
val expectedParties = identities.map { it.owningKey.toStringShort() to it.toString() }
val actualParties = selectAll<IdentityTestSchemaV1.NodeIdentitiesNoCert>().map { it.publicKeyHash to it.name }
assertThat(actualParties).isEqualTo(expectedParties)
val expectedKeys = listOf(ALICE_IDENTITY, BOC_IDENTITY, dummyNotary.identity).map { it.owningKey.toStringShort() to it.owningKey }
val actualKeys = selectAll<IdentityTestSchemaV1.NodeHashToKey>().map { it.publicKeyHash to Crypto.decodePublicKey(it.publicKey) }
assertThat(actualKeys).isEqualTo(expectedKeys)
}
private fun saveAllIdentities(identities: List<PartyAndCertificate>) {
cordaDB.transaction {
identities.forEach {
session.save(PersistentIdentityService.PersistentPublicKeyHashToCertificate(it.owningKey.hash.toString(), it.certPath.encoded))
session.save(IdentityTestSchemaV1.NodeIdentities(it.owningKey.toStringShort(), it.certPath.encoded))
}
}
}
private fun saveOurKeys(keys: List<KeyPair>) {
cordaDB.transaction {
keys.forEach {
val persistentKey = BasicHSMKeyManagementService.PersistentKey(it.public, it.private)
session.save(persistentKey)
}
}
}
private fun addNetworkParameters() {
cordaDB.transaction {
val clock = Clock.systemUTC()
val params = NetworkParameters(
1,
listOf(NotaryInfo(DUMMY_NOTARY, false), NotaryInfo(CHARLIE, false)),
1,
1,
clock.instant(),
1,
mapOf(),
Duration.ZERO,
mapOf()
)
val signedParams = params.signWithCert(bob.keyPair.private, BOB_IDENTITY.certificate)
val persistentParams = DBNetworkParametersStorage.PersistentNetworkParameters(
SecureHash.allOnesHash.toString(),
params.epoch,
signedParams.raw.bytes,
signedParams.sig.bytes,
signedParams.sig.by.encoded,
X509Utilities.buildCertPath(signedParams.sig.parentCertsChain).encoded
)
session.save(persistentParams)
}
}
private fun addTestMapping(connection: Connection, testIdentity: TestIdentity): String {
val pkHash = UUID.randomUUID().toString()
val cert = testIdentity.identity.certPath.encoded
connection.prepareStatement("INSERT INTO node_identities (pk_hash, identity_value) VALUES (?,?)").use {
it.setString(1, pkHash)
it.setBytes(2, cert)
it.executeUpdate()
}
return pkHash
}
// private fun deleteTestMapping(connection: Connection, pkHash: String) {
// connection.prepareStatement("DELETE FROM node_identities WHERE pk_hash = ?").use {
// it.setString(1, pkHash)
// it.executeUpdate()
// }
// }
private fun verifyTestMigration(connection: Connection, pk: String, name: String) {
connection.createStatement().use {
try {
val rs = it.executeQuery("SELECT (pk_hash, name) FROM node_identities_no_cert")
while (rs.next()) {
val result = rs.getString(1)
require(result.contains(pk))
require(result.contains(name))
}
rs.close()
} catch (e: Exception) {
println(e.localizedMessage)
}
private inline fun <reified T> selectAll(): List<T> {
return cordaDB.transaction {
val criteria = session.criteriaBuilder.createQuery(T::class.java)
criteria.select(criteria.from(T::class.java))
session.createQuery(criteria).resultList
}
}
}