Merge commit '121dbec87700856679baab3995352448e8214b4e' into df-merge-2018-07-25

This commit is contained in:
Dominic Fox 2018-07-25 18:00:24 +01:00
commit 9c89e3babd
9 changed files with 175 additions and 10 deletions

View File

@ -10,6 +10,8 @@ Unreleased
* Change type of the `checkpoint_value` column. Please check the upgrade-notes on how to update your database.
* Removed buggy :serverNameTablePrefix: configuration.
* ``freeLocalHostAndPort``, ``freePort``, and ``getFreeLocalPorts`` from ``TestUtils`` have been deprecated as they
don't provide any guarantee the returned port will be available which can result in flaky tests. Use ``PortAllocation.Incremental``
instead.

View File

@ -44,8 +44,8 @@ Let's open the example CorDapp in IntelliJ IDEA:
* A splash screen will appear. Click ``open``, select the cloned ``cordapp-example`` folder, and click ``OK``
* Once the project is open, click ``File``, then ``Project Structure``. Under ``Project SDK:``, set the project SDK by
clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` (where ``XXX`` is the
latest minor version number). Click ``OK``
clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` on Windows or ``Library/Java/JavaVirtualMachines/jdk1.8.XXX`` on MacOSX (where ``XXX`` is the
latest minor version number). Click ``Apply`` followed by ``OK``
* Again under ``File`` then ``Project Structure``, select ``Modules``. Click ``+``, then ``Import Module``, then select
the ``cordapp-example`` folder and click ``Open``. Choose to ``Import module from external model``, select

View File

@ -22,10 +22,64 @@ Upgrading to |release| from Open Source 3.x requires updating build file propert
.. sourcecode:: shell
<<<<<<< HEAD
ext.corda_release_distribution = 'com.r3.corda'
ext.corda_release_version = '3.1'
ext.corda_gradle_plugins_version = '4.0.25'
..
=======
ext.kotlin_version = '1.1.4'
ext.quasar_version = '0.7.9'
Please consult the relevant release notes of the release in question. If not specified, you may assume the
versions you are currently using are still in force.
We also strongly recommend cross referencing with the :doc:`changelog` to confirm changes.
UNRELEASED
----------
<<< Fill this in >>>
* Database upgrade - Change the type of the ``checkpoint_value``.
This will address the issue that the `vacuum` function is unable to clean up deleted checkpoints as they are still referenced from the ``pg_shdepend`` table.
For Postgres:
.. sourcecode:: sql
ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type bytea;
For H2:
.. sourcecode:: sql
ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type VARBINARY(33554432);
* API change: ``net.corda.core.schemas.PersistentStateRef`` fields (``index`` and ``txId``) incorrectly marked as nullable are now non-nullable,
:doc:`changelog` contains the explanation.
H2 database upgrade action:
For Cordapps persisting custom entities with ``PersistentStateRef`` used as non Primary Key column, the backing table needs to be updated,
In SQL replace ``your_transaction_id``/``your_output_index`` column names with your custom names, if entity didn't used JPA ``@AttributeOverrides``
then default names are ``transaction_id`` and ``output_index``.
.. sourcecode:: sql
SELECT count(*) FROM [YOUR_PersistentState_TABLE_NAME] WHERE your_transaction_id IS NULL OR your_output_index IS NULL;
In case your table already contains rows with NULL columns, and the logic doesn't distinguish between NULL and an empty string,
all NULL column occurrences can be changed to an empty string:
.. sourcecode:: sql
UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_transaction_id="" WHERE your_transaction_id IS NULL;
UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_output_index="" WHERE your_output_index IS NULL;
If all rows have NON NULL ``transaction_ids`` and ``output_idx`` or you have assigned empty string values, then it's safe to update the table:
>>>>>>> 121dbec87700856679baab3995352448e8214b4e
and specifying an additional repository entry to point to the location of the Corda Enterprise distribution. As an example:

View File

@ -32,6 +32,7 @@ const val NODE_DATABASE_PREFIX = "node_"
// This class forms part of the node config and so any changes to it must be handled with care
data class DatabaseConfig(
val runMigration: Boolean = false,
val initialiseSchema: Boolean = true,
val transactionIsolationLevel: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ,
val schema: String? = null,
val exportHibernateJMXStatistics: Boolean = false,
@ -99,7 +100,8 @@ class CordaPersistence(
// Check not in read-only mode.
transaction {
check(!connection.metaData.isReadOnly) { "Database should not be readonly." }
checkCorrectAttachmentsContractsTableName(connection)
checkCorrectCheckpointTypeOnPostgres(connection)
}
}
object DataSourceConfigTag {
@ -303,4 +305,34 @@ private fun Throwable.hasSQLExceptionCause(): Boolean =
else -> cause?.hasSQLExceptionCause() ?: false
}
class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception()
class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception()
class DatabaseIncompatibleException(override val message: String?, override val cause: Throwable? = null) : Exception()
private fun checkCorrectAttachmentsContractsTableName(connection: Connection) {
val correctName = "NODE_ATTACHMENTS_CONTRACTS"
val incorrectV30Name = "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME"
val incorrectV31Name = "NODE_ATTCHMENTS_CONTRACTS"
fun warning(incorrectName: String, version: String) = "The database contains the older table name $incorrectName instead of $correctName, see upgrade notes to migrate from Corda database version $version https://docs.corda.net/head/upgrade-notes.html."
if (!connection.metaData.getTables(null, null, correctName, null).next()) {
if (connection.metaData.getTables(null, null, incorrectV30Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV30Name, "3.0")) }
if (connection.metaData.getTables(null, null, incorrectV31Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV31Name, "3.1")) }
}
}
private fun checkCorrectCheckpointTypeOnPostgres(connection: Connection) {
val metaData = connection.metaData
if (metaData.getDatabaseProductName() != "PostgreSQL") {
return
}
val result = metaData.getColumns(null, null, "node_checkpoints", "checkpoint_value")
if (result.next()) {
val type = result.getString("TYPE_NAME")
if (type != "bytea") {
throw DatabaseIncompatibleException("The type of the 'checkpoint_value' table must be 'bytea', but 'oid' was found. See upgrade notes to migrate from Corda database version 3.1 https://docs.corda.net/head/upgrade-notes.html.")
}
}
}

View File

@ -66,6 +66,7 @@ class HibernateConfiguration(
// nationalised (i.e. Unicode) strings by default
val forceUnicodeForSqlServer = listOf(":oracle:", ":sqlserver:").any { jdbcUrl.contains(it, ignoreCase = true) }
enableGlobalNationalizedCharacterDataSupport(forceUnicodeForSqlServer)
return build()
}
}
@ -233,3 +234,4 @@ class HibernateConfiguration(
/** Allow Oracle database drivers ojdbc7.jar and ojdbc8.jar to deserialize classes from oracle.sql.converter package. */
fun oracleJdbcDriverSerialFilter(clazz: Class<*>): Boolean = clazz.name.startsWith("oracle.sql.converter.")

View File

@ -0,0 +1,70 @@
package net.corda.node.persistence
import net.corda.client.rpc.CordaRPCClient
import net.corda.core.internal.packageName
import net.corda.core.messaging.startFlow
import net.corda.core.utilities.getOrThrow
import net.corda.node.services.Permissions
import net.corda.testMessage.Message
import net.corda.testMessage.MessageState
import net.corda.testing.core.singleIdentity
import net.corda.testing.driver.DriverParameters
import net.corda.testing.driver.driver
import net.corda.testing.node.User
import org.junit.Test
import java.nio.file.Path
import java.sql.DriverManager
import kotlin.test.assertFailsWith
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class FailNodeOnNotMigratedAttachmentContractsTableNameTests {
@Test
fun `node fails when detecting table name not migrated from version 3 dot 0`() {
`node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME")
}
@Test
fun `node fails when detecting table name not migrated from version 3 dot 1`() {
`node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTCHMENTS_CONTRACTS")
}
private fun `node fails when not detecting compatible table name`(tableNameFromMapping: String, tableNameInDB: String) {
val user = User("mark", "dadada", setOf(Permissions.startFlow<SendMessageFlow>(), Permissions.invokeRpc("vaultQuery")))
val message = Message("Hello world!")
val baseDir: Path = driver(DriverParameters(
inMemoryDB = false,
startNodesInProcess = isQuasarAgentSpecified(),
extraCordappPackagesToScan = listOf(MessageState::class.packageName)
)) {
val (nodeName, baseDir) = {
val nodeHandle = startNode(rpcUsers = listOf(user)).getOrThrow()
val nodeName = nodeHandle.nodeInfo.singleIdentity().name
CordaRPCClient(nodeHandle.rpcAddress).start(user.username, user.password).use {
it.proxy.startFlow(::SendMessageFlow, message, defaultNotaryIdentity).returnValue.getOrThrow()
}
nodeHandle.stop()
Pair(nodeName, nodeHandle.baseDirectory)
}()
// replace the correct table name with one from the former release
DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use {
it.createStatement().execute("ALTER TABLE $tableNameFromMapping RENAME TO $tableNameInDB")
it.commit()
}
assertFailsWith(net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException::class) {
val nodeHandle = startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow()
nodeHandle.stop()
}
baseDir
}
// check that the node didn't recreated the correct table matching it's entity mapping
val (hasTableFromMapping, hasTableFromDB) = DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use {
Pair(it.metaData.getTables(null, null, tableNameFromMapping, null).next(),
it.metaData.getTables(null, null, tableNameInDB, null).next())
}
assertFalse(hasTableFromMapping)
assertTrue(hasTableFromDB)
}
}

View File

@ -33,6 +33,7 @@ import net.corda.testing.core.ALICE_NAME
import net.corda.testing.core.SerializationEnvironmentRule
import net.corda.testing.driver.PortAllocation
import net.corda.testing.internal.LogHelper
import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties
import net.corda.testing.node.internal.makeInternalTestDataSourceProperties
import org.hamcrest.Matchers.instanceOf
import org.junit.After
@ -164,7 +165,7 @@ class RaftTransactionCommitLogTests {
private fun createReplica(myAddress: NetworkHostAndPort, clusterAddress: NetworkHostAndPort? = null): CompletableFuture<Member> {
val storage = Storage.builder().withStorageLevel(StorageLevel.MEMORY).build()
val address = Address(myAddress.host, myAddress.port)
val database = configureDatabase(makeInternalTestDataSourceProperties( configSupplier = { ConfigFactory.empty() }), DatabaseConfig(runMigration = true), { null }, { null }, NodeSchemaService(includeNotarySchemas = true))
val database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, NodeSchemaService(includeNotarySchemas = true))
databases.add(database)
val stateMachineFactory = { RaftTransactionCommitLog(database, Clock.systemUTC(), RaftUniquenessProvider.Companion::createMap) }

View File

@ -1054,11 +1054,9 @@ fun configureDatabase(hikariProperties: Properties,
databaseConfig: DatabaseConfig,
wellKnownPartyFromX500Name: (CordaX500Name) -> Party?,
wellKnownPartyFromAnonymous: (AbstractParty) -> Party?,
schemaService: SchemaService = NodeSchemaService()): CordaPersistence {
val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService)
persistence.hikariStart(hikariProperties, databaseConfig, schemaService)
return persistence
}
schemaService: SchemaService = NodeSchemaService()): CordaPersistence =
createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService)
.apply { hikariStart(hikariProperties, databaseConfig, schemaService) }
fun createCordaPersistence(databaseConfig: DatabaseConfig,
wellKnownPartyFromX500Name: (CordaX500Name) -> Party?,
@ -1089,6 +1087,7 @@ fun CordaPersistence.hikariStart(hikariProperties: Properties, databaseConfig: D
when {
ex is HikariPool.PoolInitializationException -> throw CouldNotCreateDataSourceException("Could not connect to the database. Please check your JDBC connection URL, or the connectivity to the database.", ex)
ex.cause is ClassNotFoundException -> throw CouldNotCreateDataSourceException("Could not find the database driver class. Please add it to the 'drivers' folder. See: https://docs.corda.net/corda-configuration-file.html")
ex is DatabaseIncompatibleException -> throw ex
else -> throw CouldNotCreateDataSourceException("Could not create the DataSource: ${ex.message}", ex)
}
}

View File

@ -39,6 +39,7 @@ import net.corda.node.utilities.registration.UnableToRegisterNodeWithDoormanExce
import net.corda.node.utilities.saveToKeyStore
import net.corda.node.utilities.saveToTrustStore
import net.corda.nodeapi.internal.addShutdownHook
import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException
import net.corda.nodeapi.internal.config.UnknownConfigurationKeysException
import net.corda.nodeapi.internal.persistence.DatabaseMigrationException
import net.corda.nodeapi.internal.persistence.oracleJdbcDriverSerialFilter
@ -191,6 +192,10 @@ open class NodeStartup(val args: Array<String>) {
} catch (e: NetworkParametersReader.Error) {
logger.error(e.message)
return false
} catch (e: DatabaseIncompatibleException) {
e.message?.let { Node.printWarning(it) }
logger.error(e.message)
return false
} catch (e: Exception) {
if (e is Errors.NativeIoException && e.message?.contains("Address already in use") == true) {
logger.error("One of the ports required by the Corda node is already in use.")