From a2344f15fb1e4a5137bfbdda3f022a86e673d9c7 Mon Sep 17 00:00:00 2001 From: Christian Sailer Date: Thu, 21 Dec 2017 17:34:21 +0000 Subject: [PATCH] Unicode columns for SqlServer (#248) * ENT-1291 Switch liquibase changelogs to use nvarchar instead of varchar * Configure Hibernate to use nationalised strings * Configure Hibernate to use nationalised strings * Change schema so that UUIDs are varchars * Update schema certificate signing request status is not unicode * Upper case suffix for audit tables * nvarchar -> varchar for status in the audit table * Capitalisation * Capitalisation * Force hibernate to use unicode columns on SQL server * Force hibernate to use unicode columns on SQL server * Schema change to make PostgreSql happy * SQL files to initialise the perfcluster db for SQL server and PostgreSql * Code ordering and extra comment --- .../migration/cash.changelog-init.xml | 8 +-- .../commercial-paper.changelog-init.xml | 10 ++-- .../sample-cash-v1.changelog-init.xml | 8 +-- .../sample-cash-v2.changelog-init.xml | 8 +-- .../sample-cash-v3.changelog-init.xml | 8 +-- .../migration/sample-cp-v1.changelog-init.xml | 10 ++-- .../migration/sample-cp-v2.changelog-init.xml | 10 ++-- .../common/persistence/PersistenceUtils.kt | 2 +- .../network-manager.changelog-init.xml | 28 +++++------ .../internal/persistence/CordaPersistence.kt | 3 +- .../persistence/HibernateConfiguration.kt | 7 ++- .../message-schema.changelog-init.xml | 6 +-- .../net/corda/node/internal/AbstractNode.kt | 2 +- .../migration/common.changelog-init.xml | 6 +-- .../migration/node-info.changelog-init.xml | 10 ++-- .../node-services.changelog-init.xml | 50 +++++++++---------- .../migration/vault-schema.changelog-init.xml | 30 +++++------ .../migration/test.changelog-init.xml | 4 +- .../migration/cash-pt.changelog-init.xml | 8 +-- .../commercial-paper-pt.changelog-init.xml | 10 ++-- .../main/sql/PostgreSql/01-create-logins.sql | 8 +++ .../main/sql/PostgreSql/02-create-schemas.sql | 48 ++++++++++++++++++ .../sql/{ => SqlServer}/01_create_logins.sql | 3 +- .../main/sql/{ => SqlServer}/02_setup_db.sql | 7 ++- .../migration/dummy-deal.changelog-init.xml | 4 +- .../dummy-linear-v1.changelog-init.xml | 6 +-- .../dummy-linear-v2.changelog-init.xml | 6 +-- 27 files changed, 189 insertions(+), 121 deletions(-) create mode 100644 perftestcordapp/src/main/sql/PostgreSql/01-create-logins.sql create mode 100644 perftestcordapp/src/main/sql/PostgreSql/02-create-schemas.sql rename perftestcordapp/src/main/sql/{ => SqlServer}/01_create_logins.sql (82%) rename perftestcordapp/src/main/sql/{ => SqlServer}/02_setup_db.sql (81%) diff --git a/finance/src/main/resources/migration/cash.changelog-init.xml b/finance/src/main/resources/migration/cash.changelog-init.xml index 1142cfa847..33cd6f68ff 100644 --- a/finance/src/main/resources/migration/cash.changelog-init.xml +++ b/finance/src/main/resources/migration/cash.changelog-init.xml @@ -5,13 +5,13 @@ - + - - + + - + diff --git a/finance/src/main/resources/migration/commercial-paper.changelog-init.xml b/finance/src/main/resources/migration/commercial-paper.changelog-init.xml index ff5567281e..85d1387ae1 100644 --- a/finance/src/main/resources/migration/commercial-paper.changelog-init.xml +++ b/finance/src/main/resources/migration/commercial-paper.changelog-init.xml @@ -5,17 +5,17 @@ - + - + - + - + - + diff --git a/finance/src/test/resources/migration/sample-cash-v1.changelog-init.xml b/finance/src/test/resources/migration/sample-cash-v1.changelog-init.xml index bf4dc09eec..e19446d4d9 100644 --- a/finance/src/test/resources/migration/sample-cash-v1.changelog-init.xml +++ b/finance/src/test/resources/migration/sample-cash-v1.changelog-init.xml @@ -8,13 +8,13 @@ - + - - + + - + diff --git a/finance/src/test/resources/migration/sample-cash-v2.changelog-init.xml b/finance/src/test/resources/migration/sample-cash-v2.changelog-init.xml index ea6f138389..db069cff44 100644 --- a/finance/src/test/resources/migration/sample-cash-v2.changelog-init.xml +++ b/finance/src/test/resources/migration/sample-cash-v2.changelog-init.xml @@ -9,14 +9,14 @@ - + - + - + - + - + - - + + - + diff --git a/finance/src/test/resources/migration/sample-cp-v1.changelog-init.xml b/finance/src/test/resources/migration/sample-cp-v1.changelog-init.xml index c43348fd37..21e213b311 100644 --- a/finance/src/test/resources/migration/sample-cp-v1.changelog-init.xml +++ b/finance/src/test/resources/migration/sample-cp-v1.changelog-init.xml @@ -8,17 +8,17 @@ - + - + - + - + - + diff --git a/finance/src/test/resources/migration/sample-cp-v2.changelog-init.xml b/finance/src/test/resources/migration/sample-cp-v2.changelog-init.xml index d441a347af..e6aee3a4f8 100644 --- a/finance/src/test/resources/migration/sample-cp-v2.changelog-init.xml +++ b/finance/src/test/resources/migration/sample-cp-v2.changelog-init.xml @@ -8,15 +8,15 @@ - + - + - + - - + + diff --git a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistenceUtils.kt b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistenceUtils.kt index 8a207aef52..1841350fad 100644 --- a/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistenceUtils.kt +++ b/network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistenceUtils.kt @@ -42,7 +42,7 @@ fun configureDatabase(dataSourceProperties: Properties, SchemaMigration(schemas, dataSource, databaseConfig.schema).runMigration() } - return CordaPersistence(dataSource, databaseConfig, schemas, emptyList()) + return CordaPersistence(dataSource, databaseConfig, schemas, config.dataSourceProperties.getProperty("url", ""), emptyList()) } sealed class NetworkManagementSchemaServices { diff --git a/network-management/src/main/resources/migration/network-manager.changelog-init.xml b/network-management/src/main/resources/migration/network-manager.changelog-init.xml index aac8c50eb2..47324a05a0 100644 --- a/network-management/src/main/resources/migration/network-manager.changelog-init.xml +++ b/network-management/src/main/resources/migration/network-manager.changelog-init.xml @@ -11,10 +11,10 @@ - + - + @@ -22,10 +22,10 @@ - + - + @@ -38,24 +38,24 @@ - - + + - + - + - + @@ -66,7 +66,7 @@ - + @@ -74,7 +74,7 @@ - + @@ -94,16 +94,16 @@ - + - + - + diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index c76778df7e..0aca6022c9 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -42,13 +42,14 @@ class CordaPersistence( val dataSource: DataSource, databaseConfig: DatabaseConfig, schemas: Set, + val jdbcUrl: String, attributeConverters: Collection> = emptySet() ) : Closeable { val defaultIsolationLevel = databaseConfig.transactionIsolationLevel val hibernateConfig: HibernateConfiguration by lazy { transaction { - HibernateConfiguration(schemas, databaseConfig, attributeConverters) + HibernateConfiguration(schemas, databaseConfig, attributeConverters, jdbcUrl) } } val entityManagerFactory get() = hibernateConfig.sessionFactoryForRegisteredSchemas diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index 15b97235cd..8669c79a8e 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -23,7 +23,8 @@ import javax.persistence.AttributeConverter class HibernateConfiguration( schemas: Set, private val databaseConfig: DatabaseConfig, - private val attributeConverters: Collection> + private val attributeConverters: Collection>, + private val jdbcUrl: String ) { companion object { private val logger = contextLogger() @@ -94,6 +95,10 @@ class HibernateConfiguration( // to avoid OOM when large blobs might get logged. applyBasicType(CordaMaterializedBlobType, CordaMaterializedBlobType.name) applyBasicType(CordaWrapperBinaryType, CordaWrapperBinaryType.name) + // When connecting to SqlServer (and only then) do we need to tell hibernate to use + // nationalised (i.e. Unicode) strings by default + val forceUnicodeForSqlServer = jdbcUrl.contains(":sqlserver:", ignoreCase = true); + enableGlobalNationalizedCharacterDataSupport(forceUnicodeForSqlServer) build() } diff --git a/node/src/integration-test/resources/migration/message-schema.changelog-init.xml b/node/src/integration-test/resources/migration/message-schema.changelog-init.xml index 52be6b145e..d108e91729 100644 --- a/node/src/integration-test/resources/migration/message-schema.changelog-init.xml +++ b/node/src/integration-test/resources/migration/message-schema.changelog-init.xml @@ -9,11 +9,11 @@ - + - - + + diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index ed1d3d6539..7c59454012 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -879,5 +879,5 @@ fun configureDatabase(dataSourceProperties: Properties, SchemaMigration(schemaService.schemaOptions.keys, dataSource, databaseConfig.schema).runMigration() } - return CordaPersistence(dataSource, databaseConfig, schemaService.schemaOptions.keys, attributeConverters) + return CordaPersistence(dataSource, databaseConfig, schemaService.schemaOptions.keys, config.dataSourceProperties.getProperty("url", ""), attributeConverters) } diff --git a/node/src/main/resources/migration/common.changelog-init.xml b/node/src/main/resources/migration/common.changelog-init.xml index 513bf9dfa6..948bcf2599 100644 --- a/node/src/main/resources/migration/common.changelog-init.xml +++ b/node/src/main/resources/migration/common.changelog-init.xml @@ -16,13 +16,13 @@ - + - + - + diff --git a/node/src/main/resources/migration/node-info.changelog-init.xml b/node/src/main/resources/migration/node-info.changelog-init.xml index 9a29719c17..ea59bfb268 100644 --- a/node/src/main/resources/migration/node-info.changelog-init.xml +++ b/node/src/main/resources/migration/node-info.changelog-init.xml @@ -8,14 +8,14 @@ - + - + @@ -26,13 +26,13 @@ - + - + @@ -41,7 +41,7 @@ - + diff --git a/node/src/main/resources/migration/node-services.changelog-init.xml b/node/src/main/resources/migration/node-services.changelog-init.xml index 8097adb7dd..e9307295ec 100644 --- a/node/src/main/resources/migration/node-services.changelog-init.xml +++ b/node/src/main/resources/migration/node-services.changelog-init.xml @@ -6,15 +6,15 @@ - + - + - + @@ -22,18 +22,18 @@ - + - - + + - + @@ -41,15 +41,15 @@ - + - + - + @@ -58,7 +58,7 @@ - + @@ -75,10 +75,10 @@ - + - + @@ -86,18 +86,18 @@ - + - - + + - + @@ -106,7 +106,7 @@ - + @@ -118,7 +118,7 @@ - + @@ -128,15 +128,15 @@ - + - + - + @@ -144,11 +144,11 @@ - + - - + + diff --git a/node/src/main/resources/migration/vault-schema.changelog-init.xml b/node/src/main/resources/migration/vault-schema.changelog-init.xml index d4885ec446..d8da8a44ef 100644 --- a/node/src/main/resources/migration/vault-schema.changelog-init.xml +++ b/node/src/main/resources/migration/vault-schema.changelog-init.xml @@ -9,12 +9,12 @@ - + - + - + @@ -23,10 +23,10 @@ - + - + @@ -34,10 +34,10 @@ - + - + @@ -48,10 +48,10 @@ - + - + @@ -59,14 +59,14 @@ - + - - + + - + @@ -76,8 +76,8 @@ - - + + diff --git a/node/src/test/resources/migration/test.changelog-init.xml b/node/src/test/resources/migration/test.changelog-init.xml index fd1f7f199a..aa0fb78de9 100644 --- a/node/src/test/resources/migration/test.changelog-init.xml +++ b/node/src/test/resources/migration/test.changelog-init.xml @@ -6,7 +6,7 @@ - + @@ -20,7 +20,7 @@ - + diff --git a/perftestcordapp/src/main/resources/migration/cash-pt.changelog-init.xml b/perftestcordapp/src/main/resources/migration/cash-pt.changelog-init.xml index 1f7ae34bbd..7eaa21fb47 100644 --- a/perftestcordapp/src/main/resources/migration/cash-pt.changelog-init.xml +++ b/perftestcordapp/src/main/resources/migration/cash-pt.changelog-init.xml @@ -5,13 +5,13 @@ - + - - + + - + diff --git a/perftestcordapp/src/main/resources/migration/commercial-paper-pt.changelog-init.xml b/perftestcordapp/src/main/resources/migration/commercial-paper-pt.changelog-init.xml index eec5e608b3..5d12dc794d 100644 --- a/perftestcordapp/src/main/resources/migration/commercial-paper-pt.changelog-init.xml +++ b/perftestcordapp/src/main/resources/migration/commercial-paper-pt.changelog-init.xml @@ -5,17 +5,17 @@ - + - + - + - + - + diff --git a/perftestcordapp/src/main/sql/PostgreSql/01-create-logins.sql b/perftestcordapp/src/main/sql/PostgreSql/01-create-logins.sql new file mode 100644 index 0000000000..b44a902591 --- /dev/null +++ b/perftestcordapp/src/main/sql/PostgreSql/01-create-logins.sql @@ -0,0 +1,8 @@ +create database perftesting + +CREATE ROLE perfnode1 with LOGIN PASSWORD 'yourStrong(!)Password'; +CREATE ROLE perfnode2 with LOGIN PASSWORD 'yourStrong(!)Password'; +CREATE ROLE perfnode3 with LOGIN PASSWORD 'yourStrong(!)Password'; +CREATE ROLE perfnode4 with LOGIN PASSWORD 'yourStrong(!)Password'; +CREATE ROLE perfnotary with LOGIN PASSWORD 'yourStrong(!)Password'; +CREATE ROLE perfnetworkmap with LOGIN PASSWORD 'yourStrong(!)Password'; diff --git a/perftestcordapp/src/main/sql/PostgreSql/02-create-schemas.sql b/perftestcordapp/src/main/sql/PostgreSql/02-create-schemas.sql new file mode 100644 index 0000000000..85b1cc0ffc --- /dev/null +++ b/perftestcordapp/src/main/sql/PostgreSql/02-create-schemas.sql @@ -0,0 +1,48 @@ + +CREATE SCHEMA perfnode1; +GRANT ALL ON SCHEMA perfnode1 TO perfnode1; +GRANT ALL ON ALL tables IN SCHEMA perfnode1 TO perfnode1; +ALTER DEFAULT privileges IN SCHEMA perfnode1 GRANT ALL ON tables TO perfnode1; +GRANT ALL ON ALL sequences IN SCHEMA perfnode1 TO perfnode1; +ALTER DEFAULT privileges IN SCHEMA perfnode1 GRANT ALL ON sequences TO perfnode1; +CREATE sequence perfnode1.hibernate_sequence start 1 increment 1 + +CREATE SCHEMA perfnode2; +GRANT ALL ON SCHEMA perfnode2 TO perfnode2; +GRANT ALL ON ALL tables IN SCHEMA perfnode2 TO perfnode2; +ALTER DEFAULT privileges IN SCHEMA perfnode2 GRANT ALL ON tables TO perfnode2; +GRANT ALL ON ALL sequences IN SCHEMA perfnode2 TO perfnode2; +ALTER DEFAULT privileges IN SCHEMA perfnode2 GRANT ALL ON sequences TO perfnode2; +CREATE sequence perfnode2.hibernate_sequence start 1 increment 1 + +CREATE SCHEMA perfnode3; +GRANT ALL ON SCHEMA perfnode3 TO perfnode3; +GRANT ALL ON ALL tables IN SCHEMA perfnode3 TO perfnode3; +ALTER DEFAULT privileges IN SCHEMA perfnode3 GRANT ALL ON tables TO perfnode3; +GRANT ALL ON ALL sequences IN SCHEMA perfnode3 TO perfnode3; +ALTER DEFAULT privileges IN SCHEMA perfnode3 GRANT ALL ON sequences TO perfnode3; +CREATE sequence perfnode3.hibernate_sequence start 1 increment 1 + +CREATE SCHEMA perfnode4; +GRANT ALL ON SCHEMA perfnode4 TO perfnode4; +GRANT ALL ON ALL tables IN SCHEMA perfnode4 TO perfnode4; +ALTER DEFAULT privileges IN SCHEMA perfnode4 GRANT ALL ON tables TO perfnode4; +GRANT ALL ON ALL sequences IN SCHEMA perfnode4 TO perfnode4; +ALTER DEFAULT privileges IN SCHEMA perfnode4 GRANT ALL ON sequences TO perfnode4; +CREATE sequence perfnode4.hibernate_sequence start 1 increment 1 + +CREATE SCHEMA perfnotary; +GRANT ALL ON SCHEMA perfnotary TO perfnotary; +GRANT ALL ON ALL tables IN SCHEMA perfnotary TO perfnotary; +ALTER DEFAULT privileges IN SCHEMA perfnotary GRANT ALL ON tables TO perfnotary; +GRANT ALL ON ALL sequences IN SCHEMA perfnotary TO perfnotary; +ALTER DEFAULT privileges IN SCHEMA perfnotary GRANT ALL ON sequences TO perfnotary; +CREATE sequence perfnotary.hibernate_sequence start 1 increment 1 + +CREATE SCHEMA perfnetworkmap; +GRANT ALL ON SCHEMA perfnetworkmap TO perfnetworkmap; +GRANT ALL ON ALL tables IN SCHEMA perfnetworkmap TO perfnetworkmap; +ALTER DEFAULT privileges IN SCHEMA perfnetworkmap GRANT ALL ON tables TO perfnetworkmap; +GRANT ALL ON ALL sequences IN SCHEMA perfnetworkmap TO perfnetworkmap; +ALTER DEFAULT privileges IN SCHEMA perfnetworkmap GRANT ALL ON sequences TO perfnetworkmap; +CREATE sequence perfnetworkmap.hibernate_sequence start 1 increment 1 \ No newline at end of file diff --git a/perftestcordapp/src/main/sql/01_create_logins.sql b/perftestcordapp/src/main/sql/SqlServer/01_create_logins.sql similarity index 82% rename from perftestcordapp/src/main/sql/01_create_logins.sql rename to perftestcordapp/src/main/sql/SqlServer/01_create_logins.sql index ab00a062d0..1c88beb919 100644 --- a/perftestcordapp/src/main/sql/01_create_logins.sql +++ b/perftestcordapp/src/main/sql/SqlServer/01_create_logins.sql @@ -4,4 +4,5 @@ CREATE LOGIN perfnode1 with PASSWORD = 'yourStrong(!)Password'; CREATE LOGIN perfnode2 with PASSWORD = 'yourStrong(!)Password'; CREATE LOGIN perfnode3 with PASSWORD = 'yourStrong(!)Password'; CREATE LOGIN perfnode4 with PASSWORD = 'yourStrong(!)Password'; -CREATE LOGIN perfnotary with PASSWORD = 'yourStrong(!)Password'; \ No newline at end of file +CREATE LOGIN perfnotary with PASSWORD = 'yourStrong(!)Password'; +CREATE LOGIN perfnetworkmap with PASSWORD = 'yourStrong(!)Password'; \ No newline at end of file diff --git a/perftestcordapp/src/main/sql/02_setup_db.sql b/perftestcordapp/src/main/sql/SqlServer/02_setup_db.sql similarity index 81% rename from perftestcordapp/src/main/sql/02_setup_db.sql rename to perftestcordapp/src/main/sql/SqlServer/02_setup_db.sql index 2752287825..d1843633ba 100644 --- a/perftestcordapp/src/main/sql/02_setup_db.sql +++ b/perftestcordapp/src/main/sql/SqlServer/02_setup_db.sql @@ -23,4 +23,9 @@ GRANT CREATE TABLE, CREATE PROCEDURE, CREATE FUNCTION, CREATE VIEW TO perfnode4; IF NOT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'perfnotary') EXEC('CREATE SCHEMA perfnotary'); IF NOT EXISTS (SELECT * FROM sys.sysusers WHERE name='perfnotary') CREATE USER perfnotary FOR LOGIN perfnotary WITH DEFAULT_SCHEMA = perfnotary; GRANT ALTER, DELETE, EXECUTE, INSERT, REFERENCES, SELECT, UPDATE, VIEW DEFINITION ON SCHEMA::perfnotary TO perfnotary; -GRANT CREATE TABLE, CREATE PROCEDURE, CREATE FUNCTION, CREATE VIEW TO perfnotary; \ No newline at end of file +GRANT CREATE TABLE, CREATE PROCEDURE, CREATE FUNCTION, CREATE VIEW TO perfnotary; + +IF NOT EXISTS (SELECT schema_name FROM information_schema.schemata WHERE schema_name = 'perfnetworkmap') EXEC('CREATE SCHEMA perfnetworkmap'); +IF NOT EXISTS (SELECT * FROM sys.sysusers WHERE name='perfnetworkmap') CREATE USER perfnetworkmap FOR LOGIN perfnetworkmap WITH DEFAULT_SCHEMA = perfnetworkmap; +GRANT ALTER, DELETE, EXECUTE, INSERT, REFERENCES, SELECT, UPDATE, VIEW DEFINITION ON SCHEMA::perfnetworkmap TO perfnetworkmap; +GRANT CREATE TABLE, CREATE PROCEDURE, CREATE FUNCTION, CREATE VIEW TO perfnetworkmap; \ No newline at end of file diff --git a/testing/test-utils/src/main/resources/migration/dummy-deal.changelog-init.xml b/testing/test-utils/src/main/resources/migration/dummy-deal.changelog-init.xml index a5f6480277..012f6a7b8d 100644 --- a/testing/test-utils/src/main/resources/migration/dummy-deal.changelog-init.xml +++ b/testing/test-utils/src/main/resources/migration/dummy-deal.changelog-init.xml @@ -9,10 +9,10 @@ - + - + diff --git a/testing/test-utils/src/main/resources/migration/dummy-linear-v1.changelog-init.xml b/testing/test-utils/src/main/resources/migration/dummy-linear-v1.changelog-init.xml index f63061335d..dd259d50ed 100644 --- a/testing/test-utils/src/main/resources/migration/dummy-linear-v1.changelog-init.xml +++ b/testing/test-utils/src/main/resources/migration/dummy-linear-v1.changelog-init.xml @@ -7,13 +7,13 @@ - + - + - + diff --git a/testing/test-utils/src/main/resources/migration/dummy-linear-v2.changelog-init.xml b/testing/test-utils/src/main/resources/migration/dummy-linear-v2.changelog-init.xml index 777fc22329..d4ea2f7afe 100644 --- a/testing/test-utils/src/main/resources/migration/dummy-linear-v2.changelog-init.xml +++ b/testing/test-utils/src/main/resources/migration/dummy-linear-v2.changelog-init.xml @@ -8,16 +8,16 @@ - + - + - +