Tudor merge os 25 jul (#1299)

* Feature/corda 1813/change postgres column type (#3631)

* CORDA-1813 fix Postgres db bloat issue

* CORDA-1813 merge fixes

* CORDA-1813 change column type and size to a standard corda type

* CORDA-1813 docs

* CORDA-1813 create custom hibernate type for the checkpoint blob and align with enterprise

* CORDA-1813 Remove max col size

* CORDA-1813 Remove max col size

* CORDA-1813 Fix merge

* CORDA-1813 Remove buggy :serverNameTablePrefix: configuration

* CORDA-1813 fix merge

docs/source/changelog.rst

@@ -8,6 +8,8 @@ Unreleased
 ----------
 * Introduced ``TestCorDapp`` and utilities to support asymmetric setups for nodes through ``DriverDSL``, ``MockNetwork`` and ``MockServices``.
 
+* Change type of the `checkpoint_value` column. Please check the upgrade-notes on how to update your database.
+
 * ``freeLocalHostAndPort``, ``freePort``, and ``getFreeLocalPorts`` from ``TestUtils`` have been deprecated as they
   don't provide any guarantee the returned port will be available which can result in flaky tests. Use ``PortAllocation.Incremental``
   instead.

docs/source/corda-configuration-file.rst

@@ -76,10 +76,10 @@ absolute path to the node's base directory.
 
 :database:  This section is used to configure JDBC and Hibernate related properties:
 
-    :transactionIsolationLevel: Transaction isolation level as defined by the ``TRANSACTION_`` constants in
+        :transactionIsolationLevel: Transaction isolation level as defined by the ``TRANSACTION_`` constants in
             ``java.sql.Connection``, but without the "TRANSACTION_" prefix. Defaults to REPEATABLE_READ.
 
-    :exportHibernateJMXStatistics: Whether to export Hibernate JMX statistics (caution: expensive run-time overhead)
+        :exportHibernateJMXStatistics: Whether to export Hibernate JMX statistics (caution: expensive run-time overhead)
 
     :runMigration: Boolean on whether to run the database migration scripts at startup. Defaults to false.
                    In production please keep it false. For more information please check :doc:`database-management`

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt

@@ -100,9 +100,9 @@ class CordaPersistence(
         transaction {
             check(!connection.metaData.isReadOnly) { "Database should not be readonly." }
 
-            }
+        }
     }
-        object DataSourceConfigTag {
+    object DataSourceConfigTag {
         const val DATA_SOURCE_URL = "dataSource.url"
     }
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt

@@ -26,6 +26,7 @@ import org.hibernate.cfg.Configuration
 import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider
 import org.hibernate.service.UnknownUnwrapTypeException
 import org.hibernate.type.AbstractSingleColumnStandardBasicType
+import org.hibernate.type.MaterializedBlobType
 import org.hibernate.type.descriptor.java.PrimitiveByteArrayTypeDescriptor
 import org.hibernate.type.descriptor.sql.BlobTypeDescriptor
 import org.hibernate.type.descriptor.sql.VarbinaryTypeDescriptor
@@ -52,6 +53,15 @@ class HibernateConfiguration(
                 // to avoid OOM when large blobs might get logged.
                 applyBasicType(CordaMaterializedBlobType, CordaMaterializedBlobType.name)
                 applyBasicType(CordaWrapperBinaryType, CordaWrapperBinaryType.name)
+
+                // Create a custom type that will map a blob to byteA in postgres and as a normal blob for all other dbms.
+                // This is required for the Checkpoints as a workaround for the issue that postgres has on azure.
+                if (jdbcUrl.contains(":postgresql:", ignoreCase = true)) {
+                    applyBasicType(MapBlobToPostgresByteA, MapBlobToPostgresByteA.name)
+                } else {
+                    applyBasicType(MapBlobToNormalBlob, MapBlobToNormalBlob.name)
+                }
+
                 // When connecting to SqlServer or Oracle, do we need to tell hibernate to use
                 // nationalised (i.e. Unicode) strings by default
                 val forceUnicodeForSqlServer = listOf(":oracle:", ":sqlserver:").any { jdbcUrl.contains(it, ignoreCase = true) }
@@ -202,6 +212,23 @@ class HibernateConfiguration(
             return "corda-wrapper-binary"
         }
     }
+
+    // Maps to a byte array on postgres.
+    object MapBlobToPostgresByteA : AbstractSingleColumnStandardBasicType<ByteArray>(VarbinaryTypeDescriptor.INSTANCE, PrimitiveByteArrayTypeDescriptor.INSTANCE) {
+        override fun getRegistrationKeys(): Array<String> {
+            return arrayOf(name, "ByteArray", ByteArray::class.java.name)
+        }
+
+        override fun getName(): String {
+            return "corda-blob"
+        }
+    }
+
+    object MapBlobToNormalBlob : MaterializedBlobType() {
+        override fun getName(): String {
+            return "corda-blob"
+        }
+    }
 }
 
 /** Allow Oracle database drivers ojdbc7.jar and ojdbc8.jar to deserialize classes from oracle.sql.converter package. */

node/src/main/kotlin/net/corda/node/services/persistence/DBCheckpointStorage.kt

@@ -28,7 +28,7 @@ import java.util.stream.Stream
 import javax.persistence.Column
 import javax.persistence.Entity
 import javax.persistence.Id
-import javax.persistence.Lob
+import org.hibernate.annotations.Type
 
 /**
  * Simple checkpoint key value storage in DB.
@@ -43,7 +43,7 @@ class DBCheckpointStorage : CheckpointStorage {
             @Column(name = "checkpoint_id", length = 64, nullable = false)
             var checkpointId: String = "",
 
-            @Lob
+            @Type(type = "corda-blob")
             @Column(name = "checkpoint_value", nullable = false)
             var checkpoint: ByteArray = EMPTY_BYTE_ARRAY
     )

node/src/main/resources/migration/node-core.changelog-master.xml

@@ -18,5 +18,6 @@
     <include file="migration/node-core.changelog-v3.xml"/>
     <include file="migration/node-core.changelog-v4.xml"/>
     <include file="migration/node-core.changelog-pkey.xml"/>
+    <include file="migration/node-core.changelog-postgres-blob.xml"/>
 
 </databaseChangeLog>

node/src/main/resources/migration/node-core.changelog-postgres-blob.xml

@@ -0,0 +1,16 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
+
+    <changeSet author="R3.Corda" id="modify checkpoint_value column type" dbms="postgresql">
+        <dropColumn tableName="node_checkpoints" columnName="checkpoint_value"/>
+        <addColumn tableName="node_checkpoints">
+            <column name="checkpoint_value" type="varbinary(33554432)">
+                <constraints nullable="false"/>
+            </column>
+        </addColumn>
+    </changeSet>
+
+</databaseChangeLog>