Merge commit '121dbec87700856679baab3995352448e8214b4e' into df-merge-2018-07-25

docs/source/changelog.rst

@@ -10,6 +10,8 @@ Unreleased
 
 * Change type of the `checkpoint_value` column. Please check the upgrade-notes on how to update your database.
 
+* Removed buggy :serverNameTablePrefix: configuration.
+
 * ``freeLocalHostAndPort``, ``freePort``, and ``getFreeLocalPorts`` from ``TestUtils`` have been deprecated as they
   don't provide any guarantee the returned port will be available which can result in flaky tests. Use ``PortAllocation.Incremental``
   instead.

docs/source/tutorial-cordapp.rst

@@ -44,8 +44,8 @@ Let's open the example CorDapp in IntelliJ IDEA:
 * A splash screen will appear. Click ``open``, select the cloned ``cordapp-example`` folder, and click ``OK``
 
 * Once the project is open, click ``File``, then ``Project Structure``. Under ``Project SDK:``, set the project SDK by
-  clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` (where ``XXX`` is the
-  latest minor version number). Click ``OK``
+  clicking ``New...``, clicking ``JDK``, and navigating to ``C:\Program Files\Java\jdk1.8.0_XXX`` on Windows or ``Library/Java/JavaVirtualMachines/jdk1.8.XXX`` on MacOSX (where ``XXX`` is the
+  latest minor version number). Click ``Apply`` followed by ``OK``
 
 * Again under ``File`` then ``Project Structure``, select ``Modules``. Click ``+``, then ``Import Module``, then select
   the ``cordapp-example`` folder and click ``Open``. Choose to ``Import module from external model``, select

docs/source/upgrade-notes.rst

@@ -22,10 +22,64 @@ Upgrading to |release| from Open Source 3.x requires updating build file propert
 
 .. sourcecode:: shell
 
+<<<<<<< HEAD
     ext.corda_release_distribution = 'com.r3.corda'
     ext.corda_release_version = '3.1'
     ext.corda_gradle_plugins_version = '4.0.25'
 ..
+=======
+    ext.kotlin_version = '1.1.4'
+    ext.quasar_version = '0.7.9'
+
+Please consult the relevant release notes of the release in question. If not specified, you may assume the
+versions you are currently using are still in force.
+
+We also strongly recommend cross referencing with the :doc:`changelog` to confirm changes.
+
+UNRELEASED
+----------
+
+<<< Fill this in >>>
+
+* Database upgrade - Change the type of the ``checkpoint_value``.
+This will address the issue that the `vacuum` function is unable to clean up deleted checkpoints as they are still referenced from the ``pg_shdepend`` table.
+
+For Postgres:
+
+  .. sourcecode:: sql
+
+    ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type bytea;
+
+For H2:
+
+  .. sourcecode:: sql
+
+    ALTER TABLE node_checkpoints ALTER COLUMN checkpoint_value set data type VARBINARY(33554432);
+
+
+* API change: ``net.corda.core.schemas.PersistentStateRef`` fields (``index`` and ``txId``) incorrectly marked as nullable are now non-nullable,
+  :doc:`changelog` contains the explanation.
+
+  H2 database upgrade action:
+
+  For Cordapps persisting custom entities with ``PersistentStateRef`` used as non Primary Key column, the backing table needs to be updated,
+  In SQL replace ``your_transaction_id``/``your_output_index`` column names with your custom names, if entity didn't used JPA ``@AttributeOverrides``
+  then default names are ``transaction_id`` and ``output_index``.
+
+  .. sourcecode:: sql
+
+       SELECT count(*) FROM [YOUR_PersistentState_TABLE_NAME] WHERE your_transaction_id IS NULL OR your_output_index IS NULL;
+
+  In case your table already contains rows with NULL columns, and the logic doesn't distinguish between NULL and an empty string,
+  all NULL column occurrences can be changed to an empty string:
+
+  .. sourcecode:: sql
+
+       UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_transaction_id="" WHERE your_transaction_id IS NULL;
+       UPDATE [YOUR_PersistentState_TABLE_NAME] SET your_output_index="" WHERE your_output_index IS NULL;
+
+  If all rows have NON NULL ``transaction_ids`` and ``output_idx`` or you have assigned empty string values, then it's safe to update the table:
+>>>>>>> 121dbec87700856679baab3995352448e8214b4e
 
 and specifying an additional repository entry to point to the location of the Corda Enterprise distribution. As an example:
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt

@@ -32,6 +32,7 @@ const val NODE_DATABASE_PREFIX = "node_"
 // This class forms part of the node config and so any changes to it must be handled with care
 data class DatabaseConfig(
         val runMigration: Boolean = false,
+        val initialiseSchema: Boolean = true,
         val transactionIsolationLevel: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ,
         val schema: String? = null,
         val exportHibernateJMXStatistics: Boolean = false,
@@ -99,7 +100,8 @@ class CordaPersistence(
         // Check not in read-only mode.
         transaction {
             check(!connection.metaData.isReadOnly) { "Database should not be readonly." }
-
+            checkCorrectAttachmentsContractsTableName(connection)
+            checkCorrectCheckpointTypeOnPostgres(connection)
         }
     }
     object DataSourceConfigTag {
@@ -303,4 +305,34 @@ private fun Throwable.hasSQLExceptionCause(): Boolean =
             else -> cause?.hasSQLExceptionCause() ?: false
         }
 
-class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception()
+class CouldNotCreateDataSourceException(override val message: String?, override val cause: Throwable? = null) : Exception()
+
+class DatabaseIncompatibleException(override val message: String?, override val cause: Throwable? = null) : Exception()
+
+private fun checkCorrectAttachmentsContractsTableName(connection: Connection) {
+    val correctName = "NODE_ATTACHMENTS_CONTRACTS"
+    val incorrectV30Name = "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME"
+    val incorrectV31Name = "NODE_ATTCHMENTS_CONTRACTS"
+
+    fun warning(incorrectName: String, version: String) = "The database contains the older table name $incorrectName instead of $correctName, see upgrade notes to migrate from Corda database version $version https://docs.corda.net/head/upgrade-notes.html."
+
+    if (!connection.metaData.getTables(null, null, correctName, null).next()) {
+        if (connection.metaData.getTables(null, null, incorrectV30Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV30Name, "3.0")) }
+        if (connection.metaData.getTables(null, null, incorrectV31Name, null).next()) { throw DatabaseIncompatibleException(warning(incorrectV31Name, "3.1")) }
+    }
+}
+
+private fun checkCorrectCheckpointTypeOnPostgres(connection: Connection) {
+    val metaData = connection.metaData
+    if (metaData.getDatabaseProductName() != "PostgreSQL") {
+        return
+    }
+
+    val result = metaData.getColumns(null, null, "node_checkpoints", "checkpoint_value")
+    if (result.next()) {
+        val type = result.getString("TYPE_NAME")
+        if (type != "bytea") {
+            throw DatabaseIncompatibleException("The type of the 'checkpoint_value' table must be 'bytea', but 'oid' was found. See upgrade notes to migrate from Corda database version 3.1 https://docs.corda.net/head/upgrade-notes.html.")
+        }
+    }
+}

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt

@@ -66,6 +66,7 @@ class HibernateConfiguration(
                 // nationalised (i.e. Unicode) strings by default
                 val forceUnicodeForSqlServer = listOf(":oracle:", ":sqlserver:").any { jdbcUrl.contains(it, ignoreCase = true) }
                 enableGlobalNationalizedCharacterDataSupport(forceUnicodeForSqlServer)
+
                 return build()
             }
         }
@@ -233,3 +234,4 @@ class HibernateConfiguration(
 
 /** Allow Oracle database drivers ojdbc7.jar and ojdbc8.jar to deserialize classes from oracle.sql.converter package. */
 fun oracleJdbcDriverSerialFilter(clazz: Class<*>): Boolean = clazz.name.startsWith("oracle.sql.converter.")
+

node/src/integration-test/kotlin/net/corda/node/persistence/FailNodeOnNotMigratedAttachmentContractsTableNameTests.kt

@@ -0,0 +1,70 @@
+package net.corda.node.persistence
+
+import net.corda.client.rpc.CordaRPCClient
+import net.corda.core.internal.packageName
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.getOrThrow
+import net.corda.node.services.Permissions
+import net.corda.testMessage.Message
+import net.corda.testMessage.MessageState
+import net.corda.testing.core.singleIdentity
+import net.corda.testing.driver.DriverParameters
+import net.corda.testing.driver.driver
+import net.corda.testing.node.User
+import org.junit.Test
+import java.nio.file.Path
+import java.sql.DriverManager
+import kotlin.test.assertFailsWith
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+class FailNodeOnNotMigratedAttachmentContractsTableNameTests {
+    @Test
+    fun `node fails when detecting table name not migrated from version 3 dot 0`() {
+        `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTACHMENTS_CONTRACT_CLASS_NAME")
+    }
+
+    @Test
+    fun `node fails when detecting table name not migrated from version 3 dot 1`() {
+        `node fails when not detecting compatible table name`("NODE_ATTACHMENTS_CONTRACTS", "NODE_ATTCHMENTS_CONTRACTS")
+    }
+
+    private fun `node fails when not detecting compatible table name`(tableNameFromMapping: String, tableNameInDB: String) {
+        val user = User("mark", "dadada", setOf(Permissions.startFlow<SendMessageFlow>(), Permissions.invokeRpc("vaultQuery")))
+        val message = Message("Hello world!")
+        val baseDir: Path = driver(DriverParameters(
+                inMemoryDB = false,
+                startNodesInProcess = isQuasarAgentSpecified(),
+                extraCordappPackagesToScan = listOf(MessageState::class.packageName)
+        )) {
+            val (nodeName, baseDir) = {
+                val nodeHandle = startNode(rpcUsers = listOf(user)).getOrThrow()
+                val nodeName = nodeHandle.nodeInfo.singleIdentity().name
+                CordaRPCClient(nodeHandle.rpcAddress).start(user.username, user.password).use {
+                    it.proxy.startFlow(::SendMessageFlow, message, defaultNotaryIdentity).returnValue.getOrThrow()
+                }
+                nodeHandle.stop()
+                Pair(nodeName, nodeHandle.baseDirectory)
+            }()
+
+            // replace the correct table name with one from the former release
+            DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use {
+                it.createStatement().execute("ALTER TABLE $tableNameFromMapping RENAME TO $tableNameInDB")
+                it.commit()
+            }
+            assertFailsWith(net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException::class) {
+                val nodeHandle = startNode(providedName = nodeName, rpcUsers = listOf(user)).getOrThrow()
+                nodeHandle.stop()
+            }
+             baseDir
+        }
+
+        // check that the node didn't recreated the correct table matching it's entity mapping
+        val (hasTableFromMapping, hasTableFromDB) = DriverManager.getConnection("jdbc:h2:file://$baseDir/persistence", "sa", "").use {
+            Pair(it.metaData.getTables(null, null, tableNameFromMapping, null).next(),
+                    it.metaData.getTables(null, null, tableNameInDB, null).next())
+        }
+        assertFalse(hasTableFromMapping)
+        assertTrue(hasTableFromDB)
+    }
+}

node/src/integration-test/kotlin/net/corda/node/services/transactions/RaftTransactionCommitLogTests.kt

@@ -33,6 +33,7 @@ import net.corda.testing.core.ALICE_NAME
 import net.corda.testing.core.SerializationEnvironmentRule
 import net.corda.testing.driver.PortAllocation
 import net.corda.testing.internal.LogHelper
+import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties
 import net.corda.testing.node.internal.makeInternalTestDataSourceProperties
 import org.hamcrest.Matchers.instanceOf
 import org.junit.After
@@ -164,7 +165,7 @@ class RaftTransactionCommitLogTests {
     private fun createReplica(myAddress: NetworkHostAndPort, clusterAddress: NetworkHostAndPort? = null): CompletableFuture<Member> {
         val storage = Storage.builder().withStorageLevel(StorageLevel.MEMORY).build()
         val address = Address(myAddress.host, myAddress.port)
-        val database = configureDatabase(makeInternalTestDataSourceProperties( configSupplier = { ConfigFactory.empty() }), DatabaseConfig(runMigration = true), { null }, { null }, NodeSchemaService(includeNotarySchemas = true))
+        val database = configureDatabase(makeTestDataSourceProperties(), DatabaseConfig(), { null }, { null }, NodeSchemaService(includeNotarySchemas = true))
         databases.add(database)
         val stateMachineFactory = { RaftTransactionCommitLog(database, Clock.systemUTC(), RaftUniquenessProvider.Companion::createMap) }
 

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -1054,11 +1054,9 @@ fun configureDatabase(hikariProperties: Properties,
                       databaseConfig: DatabaseConfig,
                       wellKnownPartyFromX500Name: (CordaX500Name) -> Party?,
                       wellKnownPartyFromAnonymous: (AbstractParty) -> Party?,
-                      schemaService: SchemaService = NodeSchemaService()): CordaPersistence {
-    val persistence = createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService)
-    persistence.hikariStart(hikariProperties, databaseConfig, schemaService)
-    return persistence
-}
+                      schemaService: SchemaService = NodeSchemaService()): CordaPersistence =
+    createCordaPersistence(databaseConfig, wellKnownPartyFromX500Name, wellKnownPartyFromAnonymous, schemaService)
+            .apply { hikariStart(hikariProperties, databaseConfig, schemaService) }
 
 fun createCordaPersistence(databaseConfig: DatabaseConfig,
                            wellKnownPartyFromX500Name: (CordaX500Name) -> Party?,
@@ -1089,6 +1087,7 @@ fun CordaPersistence.hikariStart(hikariProperties: Properties, databaseConfig: D
         when {
             ex is HikariPool.PoolInitializationException -> throw CouldNotCreateDataSourceException("Could not connect to the database. Please check your JDBC connection URL, or the connectivity to the database.", ex)
             ex.cause is ClassNotFoundException -> throw CouldNotCreateDataSourceException("Could not find the database driver class. Please add it to the 'drivers' folder. See: https://docs.corda.net/corda-configuration-file.html")
+            ex is DatabaseIncompatibleException -> throw ex
             else -> throw CouldNotCreateDataSourceException("Could not create the DataSource: ${ex.message}", ex)
         }
     }

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -39,6 +39,7 @@ import net.corda.node.utilities.registration.UnableToRegisterNodeWithDoormanExce
 import net.corda.node.utilities.saveToKeyStore
 import net.corda.node.utilities.saveToTrustStore
 import net.corda.nodeapi.internal.addShutdownHook
+import net.corda.nodeapi.internal.persistence.DatabaseIncompatibleException
 import net.corda.nodeapi.internal.config.UnknownConfigurationKeysException
 import net.corda.nodeapi.internal.persistence.DatabaseMigrationException
 import net.corda.nodeapi.internal.persistence.oracleJdbcDriverSerialFilter
@@ -191,6 +192,10 @@ open class NodeStartup(val args: Array<String>) {
         } catch (e: NetworkParametersReader.Error) {
             logger.error(e.message)
             return false
+        } catch (e: DatabaseIncompatibleException) {
+            e.message?.let { Node.printWarning(it) }
+            logger.error(e.message)
+            return false
         } catch (e: Exception) {
             if (e is Errors.NativeIoException && e.message?.contains("Address already in use") == true) {
                 logger.error("One of the ports required by the Corda node is already in use.")