diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index fec39b13e2..cb36effe06 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -10,6 +10,8 @@ Unreleased * Change type of the `checkpoint_value` column. Please check the upgrade-notes on how to update your database. +* Removed buggy :serverNameTablePrefix: configuration. + * ``freeLocalHostAndPort``, ``freePort``, and ``getFreeLocalPorts`` from ``TestUtils`` have been deprecated as they don't provide any guarantee the returned port will be available which can result in flaky tests. Use ``PortAllocation.Incremental`` instead. diff --git a/docs/source/tutorial-cordapp.rst b/docs/source/tutorial-cordapp.rst index 31ef6befef..a2b7dfe118 100644 --- a/docs/source/tutorial-cordapp.rst +++ b/docs/source/tutorial-cordapp.rst @@ -44,8 +44,8 @@ Let's open the example CorDapp in IntelliJ IDEA: * A splash screen will appear. Click ``open``, select the cloned ``cordapp-example`` folder, and click ``OK`` * Once the project is open, click ``File``, then ``Project Structure``. Under ``Project SDK:``, set the project SDK by - clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` (where ``XXX`` is the - latest minor version number). Click ``OK`` + clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` on Windows or ``Library/Java/JavaVirtualMachines/jdk1.8.XXX`` on MacOSX (where ``XXX`` is the + latest minor version number). Click ``Apply`` followed by ``OK`` * Again under ``File`` then ``Project Structure``, select ``Modules``. Click ``+``, then ``Import Module``, then select the ``cordapp-example`` folder and click ``Open``. Choose to ``Import module from external model``, select diff --git a/docs/source/upgrade-notes.rst b/docs/source/upgrade-notes.rst index e15cb540d3..8091202e88 100644 --- a/docs/source/upgrade-notes.rst +++ b/docs/source/upgrade-notes.rst @@ -22,10 +22,64 @@ Upgrading to |release| from Open Source 3.x requires updating build file propert .. sourcecode:: shell +<<<<<<< HEAD ext.corda_release_distribution = 'com.r3.corda' ext.corda_release_version = '3.1' ext.corda_gradle_plugins_version = '4.0.25' .. +======= + ext.kotlin_version = '1.1.4' + ext.quasar_version = '0.7.9' + +Please consult the relevant release notes of the release in question. If not specified, you may assume the +versions you are currently using are still in force. + +We also strongly recommend cross referencing with the :doc:`changelog` to confirm changes. + +UNRELEASED +---------- + +<<< Fill this in >>> + +* Database upgrade - Change the type of the ``checkpoint_value``. +This will address the issue that the `vacuum` function is unable to clean up deleted checkpoints as they are still referenced from the ``pg_shdepend`` table. + +For Postgres: + + .. sourcecode:: sql + + ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type bytea; + +For H2: + + .. sourcecode:: sql + + ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type VARBINARY(33554432); + + +* API change: ``net.corda.core.schemas.PersistentStateRef`` fields (``index`` and ``txId``) incorrectly marked as nullable are now non-nullable, + :doc:`changelog` contains the explanation. + + H2 database upgrade action: + + For Cordapps persisting custom entities with ``PersistentStateRef`` used as non Primary Key column, the backing table needs to be updated, + In SQL replace ``your_transaction_id``/``your_output_index`` column names with your custom names, if entity didn't used JPA ``@AttributeOverrides`` + then default names are ``transaction_id`` and ``output_index``. + + .. sourcecode:: sql + + SELECT count(*) FROM [YOUR_PersistentState_TABLE_NAME] WHERE your_transaction_id IS NULL OR your_output_index IS NULL; + + In case your table already contains rows with NULL columns, and the logic doesn't distinguish between NULL and an empty string, + all NULL column occurrences can be changed to an empty string: + + .. sourcecode:: sql + + UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_transaction_id="" WHERE your_transaction_id IS NULL; + UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_output_index="" WHERE your_output_index IS NULL; + + If all rows have NON NULL ``transaction_ids`` and ``output_idx`` or you have assigned empty string values, then it's safe to update the table: +>>>>>>> 121dbec87700856679baab3995352448e8214b4e and specifying an additional repository entry to point to the location of the Corda Enterprise distribution. As an example: diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index e1e5720c2c..b0ab949422 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -32,6 +32,7 @@ const val NODE_DATABASE_PREFIX = "node_" // This class forms part of the node config and so any changes to it must be handled with care data class DatabaseConfig( val runMigration: Boolean = false, + val initialiseSchema: Boolean = true, val transactionIsolationLevel: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ, val schema: String? = null, val exportHibernateJMXStatistics: Boolean = false, @@ -99,7 +100,8 @@ class CordaPersistence( // Check not in read-only mode. transaction { check(!connection.metaData.isReadOnly) { "Database should not be readonly." } - + checkCorrectAttachmentsContractsTableName(connection) + checkCorrectCheckpointTypeOnPostgres(connection) } } object DataSourceConfigTag { @@ -303,4 +305,34 @@ private fun Throwable.hasSQLExceptionCause(): Boolean = else -> cause?.hasSQLExceptionCause() ?: false } -class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception() \ No newline at end of file +class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception() + +class DatabaseIncompatibleException(override val message: String?, override val cause: Throwable? = null) : Exception() + +private fun checkCorrectAttachmentsContractsTableName(connection: Connection) { + val correctName = "NODE_ATTACHMENTS_CONTRACTS" + val incorrectV30Name = "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME" + val incorrectV31Name = "NODE_ATTCHMENTS_CONTRACTS" + + fun warning(incorrectName: String, version: String) = "The database contains the older table name $incorrectName instead of $correctName, see upgrade notes to migrate from Corda database version $version https://docs.corda.net/head/upgrade-notes.html." + + if (!connection.metaData.getTables(null, null, correctName, null).next()) { + if (connection.metaData.getTables(null, null, incorrectV30Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV30Name, "3.0")) } + if (connection.metaData.getTables(null, null, incorrectV31Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV31Name, "3.1")) } + } +} + +private fun checkCorrectCheckpointTypeOnPostgres(connection: Connection) { + val metaData = connection.metaData + if (metaData.getDatabaseProductName() != "PostgreSQL") { + return + } + + val result = metaData.getColumns(null, null, "node_checkpoints", "checkpoint_value") + if (result.next()) { + val type = result.getString("TYPE_NAME") + if (type != "bytea") { + throw DatabaseIncompatibleException("The type of the 'checkpoint_value' table must be 'bytea', but 'oid' was found. See upgrade notes to migrate from Corda database version 3.1 https://docs.corda.net/head/upgrade-notes.html.") + } + } +} diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index 2ef8517c41..bf0514303d 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -66,6 +66,7 @@ class HibernateConfiguration( // nationalised (i.e. Unicode) strings by default val forceUnicodeForSqlServer = listOf(":oracle:", ":sqlserver:").any { jdbcUrl.contains(it, ignoreCase = true) } enableGlobalNationalizedCharacterDataSupport(forceUnicodeForSqlServer) + return build() } } @@ -233,3 +234,4 @@ class HibernateConfiguration( /** Allow Oracle database drivers ojdbc7.jar and ojdbc8.jar to deserialize classes from oracle.sql.converter package. */ fun oracleJdbcDriverSerialFilter(clazz: Class<*>): Boolean = clazz.name.startsWith("oracle.sql.converter.") + diff --git a/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt b/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt new file mode 100644 index 0000000000..aabc23e0e2 --- /dev/null +++ b/node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt @@ -0,0 +1,70 @@ +package net.corda.node.persistence + +import net.corda.client.rpc.CordaRPCClient +import net.corda.core.internal.packageName +import net.corda.core.messaging.startFlow +import net.corda.core.utilities.getOrThrow +import net.corda.node.services.Permissions +import net.corda.testMessage.Message +import net.corda.testMessage.MessageState +import net.corda.testing.core.singleIdentity +import net.corda.testing.driver.DriverParameters +import net.corda.testing.driver.driver +import net.corda.testing.node.User +import org.junit.Test +import java.nio.file.Path +import java.sql.DriverManager +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class FailNodeOnNotMigratedAttachmentContractsTableNameTests { + @Test + fun `node fails when detecting table name not migrated from version 3 dot 0`() { + `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME") + } + + @Test + fun `node fails when detecting table name not migrated from version 3 dot 1`() { + `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTCHMENTS_CONTRACTS") + } + + private fun `node fails when not detecting compatible table name`(tableNameFromMapping: String, tableNameInDB: String) { + val user = User("mark", "dadada", setOf(Permissions.startFlow(), Permissions.invokeRpc("vaultQuery"))) + val message = Message("Hello world!") + val baseDir: Path = driver(DriverParameters( + inMemoryDB = false, + startNodesInProcess = isQuasarAgentSpecified(), + extraCordappPackagesToScan = listOf(MessageState::class.packageName) + )) { + val (nodeName, baseDir) = { + val nodeHandle = startNode(rpcUsers = listOf(user)).getOrThrow() + val nodeName = nodeHandle.nodeInfo.singleIdentity().name + CordaRPCClient(nodeHandle.rpcAddress).start(user.username, user.password).use { + it.proxy.startFlow(::SendMessageFlow, message, defaultNotaryIdentity).returnValue.getOrThrow() + } + nodeHandle.stop() + Pair(nodeName, nodeHandle.baseDirectory) + }() + + // replace the correct table name with one from the former release + DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use { + it.createStatement().execute("ALTER TABLE $tableNameFromMapping RENAME TO $tableNameInDB") + it.commit() + } + assertFailsWith(net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException::class) { + val nodeHandle = startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow() + nodeHandle.stop() + } + baseDir + } + + // check that the node didn't recreated the correct table matching it's entity mapping + val (hasTableFromMapping, hasTableFromDB) = DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use { + Pair(it.metaData.getTables(null, null, tableNameFromMapping, null).next(), + it.metaData.getTables(null, null, tableNameInDB, null).next()) + } + assertFalse(hasTableFromMapping) + assertTrue(hasTableFromDB) + } +} \ No newline at end of file diff --git a/node/src/integration-test/kotlin/net/corda/node/services/transactions/RaftTransactionCommitLogTests.kt b/node/src/integration-test/kotlin/net/corda/node/services/transactions/RaftTransactionCommitLogTests.kt index f856aa4c73..f22c967750 100644 --- a/node/src/integration-test/kotlin/net/corda/node/services/transactions/RaftTransactionCommitLogTests.kt +++ b/node/src/integration-test/kotlin/net/corda/node/services/transactions/RaftTransactionCommitLogTests.kt @@ -33,6 +33,7 @@ import net.corda.testing.core.ALICE_NAME import net.corda.testing.core.SerializationEnvironmentRule import net.corda.testing.driver.PortAllocation import net.corda.testing.internal.LogHelper +import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties import net.corda.testing.node.internal.makeInternalTestDataSourceProperties import org.hamcrest.Matchers.instanceOf import org.junit.After @@ -164,7 +165,7 @@ class RaftTransactionCommitLogTests { private fun createReplica(myAddress: NetworkHostAndPort, clusterAddress: NetworkHostAndPort? = null): CompletableFuture { val storage = Storage.builder().withStorageLevel(StorageLevel.MEMORY).build() val address = Address(myAddress.host, myAddress.port) - val database = configureDatabase(makeInternalTestDataSourceProperties( configSupplier = { ConfigFactory.empty() }), DatabaseConfig(runMigration = true), { null }, { null }, NodeSchemaService(includeNotarySchemas = true)) + val database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, NodeSchemaService(includeNotarySchemas = true)) databases.add(database) val stateMachineFactory = { RaftTransactionCommitLog(database, Clock.systemUTC(), RaftUniquenessProvider.Companion::createMap) } diff --git a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt index 03154d5afd..97275c8a78 100644 --- a/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt +++ b/node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt @@ -1054,11 +1054,9 @@ fun configureDatabase(hikariProperties: Properties, databaseConfig: DatabaseConfig, wellKnownPartyFromX500Name: (CordaX500Name) -> Party?, wellKnownPartyFromAnonymous: (AbstractParty) -> Party?, - schemaService: SchemaService = NodeSchemaService()): CordaPersistence { - val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService) - persistence.hikariStart(hikariProperties, databaseConfig, schemaService) - return persistence -} + schemaService: SchemaService = NodeSchemaService()): CordaPersistence = + createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService) + .apply { hikariStart(hikariProperties, databaseConfig, schemaService) } fun createCordaPersistence(databaseConfig: DatabaseConfig, wellKnownPartyFromX500Name: (CordaX500Name) -> Party?, @@ -1089,6 +1087,7 @@ fun CordaPersistence.hikariStart(hikariProperties: Properties, databaseConfig: D when { ex is HikariPool.PoolInitializationException -> throw CouldNotCreateDataSourceException("Could not connect to the database. Please check your JDBC connection URL, or the connectivity to the database.", ex) ex.cause is ClassNotFoundException -> throw CouldNotCreateDataSourceException("Could not find the database driver class. Please add it to the 'drivers' folder. See: https://docs.corda.net/corda-configuration-file.html") + ex is DatabaseIncompatibleException -> throw ex else -> throw CouldNotCreateDataSourceException("Could not create the DataSource: ${ex.message}", ex) } } diff --git a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt index 04aba311d5..27fa2927a0 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt @@ -39,6 +39,7 @@ import net.corda.node.utilities.registration.UnableToRegisterNodeWithDoormanExce import net.corda.node.utilities.saveToKeyStore import net.corda.node.utilities.saveToTrustStore import net.corda.nodeapi.internal.addShutdownHook +import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException import net.corda.nodeapi.internal.config.UnknownConfigurationKeysException import net.corda.nodeapi.internal.persistence.DatabaseMigrationException import net.corda.nodeapi.internal.persistence.oracleJdbcDriverSerialFilter @@ -191,6 +192,10 @@ open class NodeStartup(val args: Array) { } catch (e: NetworkParametersReader.Error) { logger.error(e.message) return false + } catch (e: DatabaseIncompatibleException) { + e.message?.let { Node.printWarning(it) } + logger.error(e.message) + return false } catch (e: Exception) { if (e is Errors.NativeIoException && e.message?.contains("Address already in use") == true) { logger.error("One of the ports required by the Corda node is already in use.")