From 3259c4b64a1d87367519682793b71a67d28d5412 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Wed, 25 Jul 2018 15:21:51 +0100 Subject: [PATCH] Tudor merge os 25 jul (#1299) * Feature/corda 1813/change postgres column type (#3631) * CORDA-1813 fix Postgres db bloat issue * CORDA-1813 merge fixes * CORDA-1813 change column type and size to a standard corda type * CORDA-1813 docs * CORDA-1813 create custom hibernate type for the checkpoint blob and align with enterprise * CORDA-1813 Remove max col size * CORDA-1813 Remove max col size * CORDA-1813 Fix merge * CORDA-1813 Remove buggy :serverNameTablePrefix: configuration * CORDA-1813 fix merge --- docs/source/changelog.rst | 2 ++ docs/source/corda-configuration-file.rst | 4 +-- .../internal/persistence/CordaPersistence.kt | 4 +-- .../persistence/HibernateConfiguration.kt | 27 +++++++++++++++++++ .../persistence/DBCheckpointStorage.kt | 4 +-- .../migration/node-core.changelog-master.xml | 1 + .../node-core.changelog-postgres-blob.xml | 16 +++++++++++ 7 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 node/src/main/resources/migration/node-core.changelog-postgres-blob.xml diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 8b19a55f83..fec39b13e2 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -8,6 +8,8 @@ Unreleased ---------- * Introduced ``TestCorDapp`` and utilities to support asymmetric setups for nodes through ``DriverDSL``, ``MockNetwork`` and ``MockServices``. +* Change type of the `checkpoint_value` column. Please check the upgrade-notes on how to update your database. + * ``freeLocalHostAndPort``, ``freePort``, and ``getFreeLocalPorts`` from ``TestUtils`` have been deprecated as they don't provide any guarantee the returned port will be available which can result in flaky tests. Use ``PortAllocation.Incremental`` instead. diff --git a/docs/source/corda-configuration-file.rst b/docs/source/corda-configuration-file.rst index dfe425ebcf..91806630da 100644 --- a/docs/source/corda-configuration-file.rst +++ b/docs/source/corda-configuration-file.rst @@ -76,10 +76,10 @@ absolute path to the node's base directory. :database: This section is used to configure JDBC and Hibernate related properties: - :transactionIsolationLevel: Transaction isolation level as defined by the ``TRANSACTION_`` constants in + :transactionIsolationLevel: Transaction isolation level as defined by the ``TRANSACTION_`` constants in ``java.sql.Connection``, but without the "TRANSACTION_" prefix. Defaults to REPEATABLE_READ. - :exportHibernateJMXStatistics: Whether to export Hibernate JMX statistics (caution: expensive run-time overhead) + :exportHibernateJMXStatistics: Whether to export Hibernate JMX statistics (caution: expensive run-time overhead) :runMigration: Boolean on whether to run the database migration scripts at startup. Defaults to false. In production please keep it false. For more information please check :doc:`database-management` diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index 82f8ac918d..e1e5720c2c 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -100,9 +100,9 @@ class CordaPersistence( transaction { check(!connection.metaData.isReadOnly) { "Database should not be readonly." } - } + } } - object DataSourceConfigTag { + object DataSourceConfigTag { const val DATA_SOURCE_URL = "dataSource.url" } diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index cbaaba123d..2ef8517c41 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -26,6 +26,7 @@ import org.hibernate.cfg.Configuration import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider import org.hibernate.service.UnknownUnwrapTypeException import org.hibernate.type.AbstractSingleColumnStandardBasicType +import org.hibernate.type.MaterializedBlobType import org.hibernate.type.descriptor.java.PrimitiveByteArrayTypeDescriptor import org.hibernate.type.descriptor.sql.BlobTypeDescriptor import org.hibernate.type.descriptor.sql.VarbinaryTypeDescriptor @@ -52,6 +53,15 @@ class HibernateConfiguration( // to avoid OOM when large blobs might get logged. applyBasicType(CordaMaterializedBlobType, CordaMaterializedBlobType.name) applyBasicType(CordaWrapperBinaryType, CordaWrapperBinaryType.name) + + // Create a custom type that will map a blob to byteA in postgres and as a normal blob for all other dbms. + // This is required for the Checkpoints as a workaround for the issue that postgres has on azure. + if (jdbcUrl.contains(":postgresql:", ignoreCase = true)) { + applyBasicType(MapBlobToPostgresByteA, MapBlobToPostgresByteA.name) + } else { + applyBasicType(MapBlobToNormalBlob, MapBlobToNormalBlob.name) + } + // When connecting to SqlServer or Oracle, do we need to tell hibernate to use // nationalised (i.e. Unicode) strings by default val forceUnicodeForSqlServer = listOf(":oracle:", ":sqlserver:").any { jdbcUrl.contains(it, ignoreCase = true) } @@ -202,6 +212,23 @@ class HibernateConfiguration( return "corda-wrapper-binary" } } + + // Maps to a byte array on postgres. + object MapBlobToPostgresByteA : AbstractSingleColumnStandardBasicType(VarbinaryTypeDescriptor.INSTANCE, PrimitiveByteArrayTypeDescriptor.INSTANCE) { + override fun getRegistrationKeys(): Array { + return arrayOf(name, "ByteArray", ByteArray::class.java.name) + } + + override fun getName(): String { + return "corda-blob" + } + } + + object MapBlobToNormalBlob : MaterializedBlobType() { + override fun getName(): String { + return "corda-blob" + } + } } /** Allow Oracle database drivers ojdbc7.jar and ojdbc8.jar to deserialize classes from oracle.sql.converter package. */ diff --git a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt index ac71d2604b..c8048c382c 100644 --- a/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt +++ b/node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt @@ -28,7 +28,7 @@ import java.util.stream.Stream import javax.persistence.Column import javax.persistence.Entity import javax.persistence.Id -import javax.persistence.Lob +import org.hibernate.annotations.Type /** * Simple checkpoint key value storage in DB. @@ -43,7 +43,7 @@ class DBCheckpointStorage : CheckpointStorage { @Column(name = "checkpoint_id", length = 64, nullable = false) var checkpointId: String = "", - @Lob + @Type(type = "corda-blob") @Column(name = "checkpoint_value", nullable = false) var checkpoint: ByteArray = EMPTY_BYTE_ARRAY ) diff --git a/node/src/main/resources/migration/node-core.changelog-master.xml b/node/src/main/resources/migration/node-core.changelog-master.xml index 398350560d..6775478002 100644 --- a/node/src/main/resources/migration/node-core.changelog-master.xml +++ b/node/src/main/resources/migration/node-core.changelog-master.xml @@ -18,5 +18,6 @@ + diff --git a/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml b/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml new file mode 100644 index 0000000000..c115bb1e74 --- /dev/null +++ b/node/src/main/resources/migration/node-core.changelog-postgres-blob.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + \ No newline at end of file