Remove compile time dependency on JDBC drivers [ENT-1363] (#387)

* Change JDBC drivers dependencies to integrationTestRuntimeOnly
* Added smoke test validating JDBC driver registration located in "./plugins" directory 
* Some docsite tweaks

.idea/compiler.xml

@@ -41,6 +41,7 @@
       <module name="core_integrationTest" target="1.8" />
       <module name="core_main" target="1.8" />
       <module name="core_smokeTest" target="1.8" />
+      <module name="core_smokeTestPlugins" target="1.8" />
       <module name="core_test" target="1.8" />
       <module name="dbmigration_main" target="1.8" />
       <module name="dbmigration_test" target="1.8" />
@@ -148,6 +149,7 @@
       <module name="simm-valuation-demo_test" target="1.8" />
       <module name="smoke-test-utils_main" target="1.8" />
       <module name="smoke-test-utils_test" target="1.8" />
+      <module name="smoke-test-utils_testDriver" target="1.8" />
       <module name="source-example-code_integrationTest" target="1.8" />
       <module name="source-example-code_main" target="1.8" />
       <module name="source-example-code_test" target="1.8" />

client/rpc/src/smoke-test/java/net/corda/java/rpc/StandaloneCordaRPCJavaClientTest.java

@@ -49,7 +49,8 @@ public class StandaloneCordaRPCJavaClientTest {
             port.getAndIncrement(),
             true,
             Collections.singletonList(rpcUser),
-            true
+            true,
+            Collections.emptyList()
     );
 
     @Before

core/build.gradle

@@ -43,6 +43,9 @@ sourceSets {
             runtimeClasspath += main.output
             srcDir file('src/smoke-test/java')
         }
+        resources {
+            srcDir file('src/smoke-test/resources')
+        }
     }
 }
 

core/src/smoke-test/kotlin/net/corda/core/PluginRegistrationTest.kt

@@ -0,0 +1,63 @@
+package net.corda.core
+
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.internal.div
+import net.corda.nodeapi.internal.config.User
+import net.corda.smoketesting.NodeConfig
+import net.corda.smoketesting.NodeProcess
+import org.assertj.core.api.Assertions.assertThatThrownBy
+import org.junit.Rule
+import org.junit.Test
+import org.junit.rules.TemporaryFolder
+import java.io.FileOutputStream
+import java.util.jar.JarOutputStream
+
+
+class PluginRegistrationTest {
+    private companion object {
+        val pluginJarFile = PluginRegistrationTest::class.java.getResource("/dummydriver.jar")!!.path
+    }
+
+    @Rule
+    @JvmField
+    val temporaryFolder = TemporaryFolder()
+
+    @Test
+    fun `test plugin registration` () {
+        // Create node jarDir with an empty jar file in it
+        val jarDir = temporaryFolder.newFolder("jardir").path!!
+        JarOutputStream(FileOutputStream((jarDir / "empty.jar").toFile())).close ()
+        val config = NodeConfig(
+                legalName = CordaX500Name(organisation = "org", locality = "Madrid", country = "ES"),
+                p2pPort = 30100,
+                rpcPort = 30101,
+                webPort = 30102,
+                isNotary = false,
+                users = listOf(User("_", "_", setOf("ALL"))),
+                jarDirs = listOf(jarDir))
+
+        // Check we do not have plugin on classpath
+        assertThatThrownBy({ Class.forName("net.corda.smoketesting.plugins.DummyJDBCDriver") })
+
+        // Install plugin Jars in node directory, then start the node and close it
+        val consoleOutput = temporaryFolder.newFile("node-stdout.txt")
+        val nodeJvmArgs = arrayOf("-logging-level", "DEBUG", "-no-local-shell", "-log-to-console")
+        NodeProcess.Factory(extraJvmArgs = nodeJvmArgs, redirectConsoleTo = consoleOutput)
+                .setupPlugins(config, listOf(pluginJarFile))
+                .create(config)
+                .close()
+        val outputLines = consoleOutput.readLines()
+        val classpath = outputLines.filter { it.contains(" classpath:") }.last()
+
+        // If DummyJdbcDriver has been registered it should have printed a message
+        assert(outputLines.count { it.contains("[DummyJDBCDriver] hello") } > 0) {
+            "Cannot find registration message from installed jdbc driver"
+        }
+
+        // Check the printed classpath contains 'nodeJarDir'
+        assert(classpath.contains(jarDir)) {
+            "Expected to find $jarDir in printed classpath"
+        }
+    }
+}
+

docs/source/changelog.rst

@@ -15,6 +15,9 @@ UNRELEASED
   the table name for the `participants` collection.
   For an example on how the mapping can be done, see: DummyDealStateSchemaV1.PersistentDummyDealState
 
+* JDBC drivers for SQL server and PostgresSQL are no longer bundled as part of Corda releases. If you are running a node
+  on such databases you need to provide the associated driver as described in :doc:`node-database`.
+
 R3 Corda 3.0 Developer Preview
 ------------------------------
 

docs/source/node-database.rst

@@ -36,18 +36,19 @@ Standalone database
 To run a node against a remote database modify node JDBC connection properties in `dataSourceProperties` entry
 and Hibernate properties in `database` entry - see :ref:`Node configuration <database_properties_ref>`.
 
-.. _sql_server_ref:
+Corda will search for valid JDBC drivers either under the ``./plugins`` subdirectory of the node base directory or in one
+of the paths specified by the ``jarDirs`` field of the node configuration. Please make sure a ``jar`` file containing drivers
+supporting the database in use is present in one of these locations.
 
 SQL Azure and SQL Server
 ````````````````````````
-Corda supports SQL Server 2017 (14.0.3006.16) and Azure SQL (12.0.2000.8).
+Corda has been tested with SQL Server 2017 (14.0.3006.16) and Azure SQL (12.0.2000.8), using Microsoft JDBC Driver 6.2.
 The minimum transaction isolation level ``database.transactionIsolationLevel`` is 'READ_COMMITTED'.
 The property ``database.schema`` is optional.
-Corda ships with Microsoft JDBC Driver 6.2 for SQLServer out-of-the-box.
 
 Example node configuration for SQL Azure:
 
-.. sourcecode:: none
+.. sourcecode:: groovy
 
     dataSourceProperties {
         dataSourceClassName = "com.microsoft.sqlserver.jdbc.SQLServerDataSource"
@@ -60,21 +61,17 @@ Example node configuration for SQL Azure:
         transactionIsolationLevel = READ_COMMITTED
         schema = [SCHEMA]
     }
-
+    jarDirs = [PATH_TO_JDBC_DRIVER_DIR]
-.. _postgres_ref:
 
 PostgreSQL
 ````````````````````````
-
+Corda has been tested on PostgreSQL 9.6 database, using PostgreSQL JDBC Driver 42.1.4.
-Corda supports PostgreSQL 9.6 database preliminarily.
+The property ``database.schema`` is optional. The value of ``database.schema`` is automatically wrapped in double quotes
-The property ``database.schema`` is optional. Currently only lowercase value is supported,
+to preserve case-sensitivity (e.g. `AliceCorp` becomes `"AliceCorp"`, without quotes PostgresSQL would treat the value as `alicecorp`).
-please ensure your database setup uses lowercase schema names (Corda sends an unquoted schema name
-and PostgresSQL interprets the value as a lowercase one e.g. `AliceCorp` becomes `alicecorp`).
-Corda ships with PostgreSQL JDBC Driver 42.1.4 out-of-the-box.
 
 Example node configuration for PostgreSQL:
 
-.. sourcecode:: none
+.. sourcecode:: groovy
 
     dataSourceProperties = {
         dataSourceClassName = "org.postgresql.ds.PGSimpleDataSource"
@@ -86,3 +83,4 @@ Example node configuration for PostgreSQL:
         transactionIsolationLevel = READ_COMMITTED
         schema = [SCHEMA]
     }
+    jarDirs = [PATH_TO_JDBC_DRIVER_DIR]

node/build.gradle

@@ -51,6 +51,13 @@ processTestResources {
     from file("$rootDir/config/test/jolokia-access.xml")
 }
 
+// Map DB provider to driver artifact imported as runtime dependency in integration tests.
+def jdbcRuntimeDependency = [
+        'integration-sql-server': "com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre8",
+        'integration-azure-sql' : "com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre8",
+        'integration-postgres'  : "org.postgresql:postgresql:${postgresql_version}"
+]
+
 // To find potential version conflicts, run "gradle htmlDependencyReport" and then look in
 // build/reports/project/dependencies/index.html for green highlighted parts of the tree.
 
@@ -122,12 +129,6 @@ dependencies {
     // For H2 database support in persistence
     compile "com.h2database:h2:$h2_version"
 
-    //TODO remove once we can put driver jar into a predefined directory
-    //JDBC driver can be passed to the Node at startup using setting the jarDirs property in the Node configuration file.
-    //For Postgres database support in persistence
-    compile "org.postgresql:postgresql:$postgresql_version"
-    //For Azure SQL and SQL Server support in persistence
-    compile 'com.microsoft.sqlserver:mssql-jdbc:6.2.1.jre8'
     // For the MySQLUniquenessProvider
     compile group: 'mysql', name: 'mysql-connector-java', version: '6.0.6'
 
@@ -195,6 +196,17 @@ dependencies {
     testCompile "org.glassfish.jersey.containers:jersey-container-servlet-core:${jersey_version}"
     testCompile "org.glassfish.jersey.containers:jersey-container-jetty-http:${jersey_version}"
 
+    // Add runtime-only dependency on the JDBC driver for the specified DB provider
+    def DB_PROVIDER = System.getProperty("databaseProvider")
+    if (DB_PROVIDER != null) {
+        final driverDependency = jdbcRuntimeDependency[DB_PROVIDER]
+        if (driverDependency != null) {
+            integrationTestRuntimeOnly driverDependency
+        } else {
+            throw new GradleException('Unsupported DB provider: ' + DB_PROVIDER)
+        }
+    }
+
     // Jolokia JVM monitoring agent
     runtime "org.jolokia:jolokia-jvm:${jolokia_version}:agent"
 }

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -84,6 +84,7 @@ import java.security.KeyStoreException
 import java.security.PublicKey
 import java.security.cert.X509Certificate
 import java.sql.Connection
+import java.sql.DriverManager
 import java.time.Clock
 import java.time.Duration
 import java.util.*
@@ -616,6 +617,11 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
     class DatabaseConfigurationException(msg: String) : CordaException(msg)
 
     protected open fun <T> initialiseDatabasePersistence(schemaService: SchemaService, identityService: IdentityService, insideTransaction: (CordaPersistence) -> T): T {
+        log.debug {
+            val driverClasses = DriverManager.getDrivers().asSequence().map { it.javaClass.name }
+            "Available JDBC drivers: $driverClasses"
+        }
+
         val props = configuration.dataSourceProperties
         if (props.isNotEmpty()) {
             val database = configureDatabase(props, configuration.database, identityService, schemaService)

testing/smoke-test-utils/src/main/kotlin/net/corda/smoketesting/NodeConfig.kt

@@ -15,7 +15,8 @@ class NodeConfig(
         val webPort: Int,
         val isNotary: Boolean,
         val users: List<User>,
-        val runMigration: Boolean = true
+        val runMigration: Boolean = true,
+        val jarDirs: List<String> = emptyList()
 ) {
     companion object {
         val renderOptions: ConfigRenderOptions = ConfigRenderOptions.defaults().setOriginComments(false)
@@ -37,6 +38,7 @@ class NodeConfig(
                 .withValue("rpcUsers", valueFor(users.map(User::toMap).toList()))
                 .withValue("database", valueFor(mapOf("runMigration" to runMigration)))
                 .withValue("useTestClock", valueFor(true))
+                .withValue("jarDirs", valueFor(jarDirs))
         return if (isNotary) {
             config.withValue("notary", ConfigValueFactory.fromMap(mapOf("validating" to true)))
         } else {

testing/smoke-test-utils/src/main/kotlin/net/corda/smoketesting/NodeProcess.kt

@@ -4,6 +4,7 @@ import net.corda.client.rpc.CordaRPCClient
 import net.corda.client.rpc.CordaRPCConnection
 import net.corda.client.rpc.internal.KryoClientSerializationScheme
 import net.corda.core.internal.copyTo
+import net.corda.core.internal.copyToDirectory
 import net.corda.core.internal.createDirectories
 import net.corda.core.internal.div
 import net.corda.core.utilities.NetworkHostAndPort
@@ -11,6 +12,8 @@ import net.corda.core.utilities.contextLogger
 import net.corda.nodeapi.internal.network.NetworkParametersCopier
 import net.corda.testing.common.internal.testNetworkParameters
 import net.corda.testing.common.internal.asContextEnv
+import java.io.File
+import java.nio.file.Files.createDirectories
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.time.Instant
@@ -51,7 +54,9 @@ class NodeProcess(
     // as a CorDapp for the nodes.
     class Factory(
             val buildDirectory: Path = Paths.get("build"),
-            val cordaJar: Path = Paths.get(this::class.java.getResource("/corda.jar").toURI())
+            val cordaJar: Path = Paths.get(this::class.java.getResource("/corda.jar").toURI()),
+            val extraJvmArgs: Array<out String> = emptyArray(),
+            val redirectConsoleTo: File? = null
     ) {
         private companion object {
             val javaPath: Path = Paths.get(System.getProperty("java.home"), "bin", "java")
@@ -91,6 +96,13 @@ class NodeProcess(
             return NodeProcess(config, nodeDir, process, client)
         }
 
+        fun setupPlugins(config: NodeConfig, jarPaths: List<String>): Factory {
+            (baseDirectory(config) / "plugins").createDirectories().also {
+                jarPaths.forEach { jar -> Paths.get(jar).copyToDirectory(it) }
+            }
+            return this
+        }
+
         private fun waitForNode(process: Process, config: NodeConfig, client: CordaRPCClient) {
             val executor = Executors.newSingleThreadScheduledExecutor()
             try {
@@ -121,11 +133,16 @@ class NodeProcess(
         }
 
         private fun startNode(nodeDir: Path): Process {
+            val redirectTo = redirectConsoleTo?.let {
+                ProcessBuilder.Redirect.appendTo(it)
+            } ?: ProcessBuilder.Redirect.INHERIT
+
             val builder = ProcessBuilder()
-                    .command(javaPath.toString(), "-Dcapsule.log=verbose", "-jar", cordaJar.toString())
+                    .command(javaPath.toString(), "-Dcapsule.log=verbose",
+                            "-jar", cordaJar.toString(), *extraJvmArgs)
                     .directory(nodeDir.toFile())
-                    .redirectError(ProcessBuilder.Redirect.INHERIT)
+                    .redirectError(redirectTo)
-                    .redirectOutput(ProcessBuilder.Redirect.INHERIT)
+                    .redirectOutput(redirectTo)
 
             builder.environment().putAll(mapOf(
                     "CAPSULE_CACHE_DIR" to (buildDirectory / "capsule").toString()