ExternalVendorCode/corda
1
0
Fork 0
mirror of https://github.com/corda/corda.git synced 2025-02-04 18:22:29 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

ENT-979 Enable integration tests to run against another database (#67)

Browse Source 
DatabaseJDBCUrl from system properties
This commit is contained in:
Maksymilian Pawlak 2017-10-20 15:14:14 +01:00 committed by GitHub
parent def9d9d0ea
commit 0ae205ec25
11 changed files with 96 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
build.gradle
View File

@ -138,6 +138,19 @@ allprojects {
// TODO: Remove once we fully switched to AMQP
final AMQP_ENABLE_PROP_NAME = "net.corda.testing.amqp.enable"
systemProperty(AMQP_ENABLE_PROP_NAME, System.getProperty(AMQP_ENABLE_PROP_NAME))
//Allows to pass database-related properties for unit and integration tests
final DATASOURCE_URL = "dataSourceProperties.dataSource.url"
final DATASOURCE_CLASSNAME = "dataSourceProperties.dataSourceClassName"
final DATASOURCE_USER = "dataSourceProperties.dataSource.user"
final DATASOURCE_PASSWORD = "dataSourceProperties.dataSource.password"
[DATASOURCE_URL, DATASOURCE_CLASSNAME, DATASOURCE_USER, DATASOURCE_PASSWORD].forEach {
def property = System.getProperty(it)
if (property != null) {
systemProperty(it, property)
}
}
}
group 'com.r3.corda.enterprise'

17
docs/source/testing.rst
View File

@ -39,3 +39,20 @@ How to manually test each of these areas differs and is currently not fully spec
ensure the program starts, that you can interact with it, and that no exceptions are generated in normal operation.
TODO: Add instructions on manual testing
External Database Testing
-------------------------
By default, all tests which need a database, utilize built-in H2 instances. In case the testing with other database backends
or other database setup (H2 in server mode for example), while running tests extra parameters can be used to specify required
properties. Appropriate changes will then be applied to all tests in a appropriate manner.
- ``dataSourceProperties.dataSource.url`` - JDBC datasource URL. Appropriate drivers must be available in classpath. Also, for
different tests random database name is appended at the end of this string, ie. ``jdbc:h2:tcp://localhost:9092`` will become
full, proper URL - ie.``jdbc:h2:tcp://localhost:9092/34jh543gk243g2`` - mind the last slash missing.
- ``dataSourceProperties.dataSourceClassName`` - JDBC driver classname - defaults to ``org.h2.jdbcx.JdbcDataSource``)
- ``dataSourceProperties.dataSource.user`` - JDBC username - defaults to ``sa``
- ``dataSourceProperties.dataSource.password`` - JDBC password - defaults to ``""`` (Empty string)

2
node/src/integration-test/kotlin/net/corda/node/BootTests.kt
View File

@ -41,7 +41,7 @@ class BootTests {
fun `double node start doesn't write into log file`() {
val logConfigFile = projectRootDir / "config" / "dev" / "log4j2.xml"
assertThat(logConfigFile).isRegularFile()
driver(isDebug = true, systemProperties = mapOf("log4j.configurationFile" to logConfigFile.toString())) {
driver(isDebug = true, extraSystemProperties = mapOf("log4j.configurationFile" to logConfigFile.toString())) {
val alice = startNode(providedName = ALICE.name).get()
val logFolder = alice.configuration.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME
val logFile = logFolder.toFile().listFiles { _, name -> name.endsWith(".log") }.single()

10
node/src/main/kotlin/net/corda/node/services/config/ConfigUtilities.kt
View File

@ -1,9 +1,6 @@
package net.corda.node.services.config
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigParseOptions
import com.typesafe.config.ConfigRenderOptions
import com.typesafe.config.*
import net.corda.core.crypto.Crypto
import net.corda.core.crypto.SignatureScheme
import net.corda.core.identity.CordaX500Name
@ -30,10 +27,9 @@ object ConfigHelper {
val parseOptions = ConfigParseOptions.defaults()
val defaultConfig = ConfigFactory.parseResources("reference.conf", parseOptions.setAllowMissing(false))
val appConfig = ConfigFactory.parseFile(configFile.toFile(), parseOptions.setAllowMissing(allowMissingConfig))
val finalConfig = configOf(
val finalConfig = configOverrides
// Add substitution values here
"baseDirectory" to baseDirectory.toString())
.withFallback(configOverrides)
.withFallback( configOf("baseDirectory" to baseDirectory.toString()))
.withFallback(appConfig)
.withFallback(defaultConfig)
.resolve()

6
node/src/main/resources/reference.conf
View File

@ -5,9 +5,9 @@ keyStorePassword = "cordacadevpass"
trustStorePassword = "trustpass"
dataSourceProperties = {
dataSourceClassName = org.h2.jdbcx.JdbcDataSource
"dataSource.url" = "jdbc:h2:file:"${baseDirectory}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=100;AUTO_SERVER_PORT="${h2port}
"dataSource.user" = sa
"dataSource.password" = ""
dataSource.url = "jdbc:h2:file:"${baseDirectory}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=100;AUTO_SERVER_PORT="${h2port}
dataSource.user = sa
dataSource.password = ""
}
database = {
transactionIsolationLevel = "repeatableRead"

5
samples/attachment-demo/src/integration-test/kotlin/net/corda/attachmentdemo/AttachmentDemoTest.kt
View File

@ -13,14 +13,15 @@ import java.util.concurrent.CompletableFuture.supplyAsync
class AttachmentDemoTest {
// run with a 10,000,000 bytes in-memory zip file. In practice, a slightly bigger file will be used (~10,002,000 bytes).
// Force INFO logging to prevent printing 10MB arrays in logfiles
@Test
fun `attachment demo using a 10MB zip file`() {
val numOfExpectedBytes = 10_000_000
driver(isDebug = true, portAllocation = PortAllocation.Incremental(20000)) {
val demoUser = listOf(User("demo", "demo", setOf(startFlowPermission<AttachmentDemoFlow>())))
val (nodeA, nodeB) = listOf(
startNode(providedName = DUMMY_BANK_A.name, rpcUsers = demoUser, maximumHeapSize = "1g"),
startNode(providedName = DUMMY_BANK_B.name, rpcUsers = demoUser, maximumHeapSize = "1g"),
startNode(providedName = DUMMY_BANK_A.name, rpcUsers = demoUser, maximumHeapSize = "1g", logLevel = "INFO"),
startNode(providedName = DUMMY_BANK_B.name, rpcUsers = demoUser, maximumHeapSize = "1g", logLevel = "INFO"),
startNotaryNode(DUMMY_NOTARY.name, validating = false))
.map { it.getOrThrow() }
startWebserver(nodeB).getOrThrow()

2
testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt
View File

@ -68,7 +68,7 @@ class DriverTests {
// Make sure we're using the log4j2 config which writes to the log file
val logConfigFile = projectRootDir / "config" / "dev" / "log4j2.xml"
assertThat(logConfigFile).isRegularFile()
driver(isDebug = true, systemProperties = mapOf("log4j.configurationFile" to logConfigFile.toString())) {
driver(isDebug = true, extraSystemProperties = mapOf("log4j.configurationFile" to logConfigFile.toString())) {
val baseDirectory = startNode(providedName = DUMMY_BANK_A.name).getOrThrow().configuration.baseDirectory
val logFile = (baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME).list { it.sorted().findFirst().get() }
val debugLinesPresent = logFile.readLines { lines -> lines.anyMatch { line -> line.startsWith("[DEBUG]") } }

4
testing/node-driver/src/main/kotlin/net/corda/testing/RPCDriver.kt
View File

@ -225,7 +225,7 @@ fun <A> rpcDriver(
driverDirectory: Path = Paths.get("build", getTimestampAsDirectoryName()),
portAllocation: PortAllocation = globalPortAllocation,
debugPortAllocation: PortAllocation = globalDebugPortAllocation,
systemProperties: Map<String, String> = emptyMap(),
extraSystemProperties: Map<String, String> = emptyMap(),
useTestClock: Boolean = false,
initialiseSerialization: Boolean = true,
networkMapStartStrategy: NetworkMapStartStrategy = NetworkMapStartStrategy.Dedicated(startAutomatically = false),
@ -237,7 +237,7 @@ fun <A> rpcDriver(
DriverDSL(
portAllocation = portAllocation,
debugPortAllocation = debugPortAllocation,
systemProperties = systemProperties,
extraSystemProperties = extraSystemProperties,
driverDirectory = driverDirectory.toAbsolutePath(),
useTestClock = useTestClock,
networkMapStartStrategy = networkMapStartStrategy,

57
testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt
View File

@ -95,7 +95,8 @@ interface DriverDSLExposedInterface : CordformContext {
verifierType: VerifierType = defaultParameters.verifierType,
customOverrides: Map<String, Any?> = defaultParameters.customOverrides,
startInSameProcess: Boolean? = defaultParameters.startInSameProcess,
maximumHeapSize: String = defaultParameters.maximumHeapSize): CordaFuture<NodeHandle>
maximumHeapSize: String = defaultParameters.maximumHeapSize,
logLevel: String? = defaultParameters.logLevel): CordaFuture<NodeHandle>
// TODO This method has been added temporarily, to be deleted once the set of notaries is defined at the network level.
fun startNotaryNode(providedName: CordaX500Name,
@ -281,7 +282,8 @@ data class NodeParameters(
val verifierType: VerifierType = VerifierType.InMemory,
val customOverrides: Map<String, Any?> = emptyMap(),
val startInSameProcess: Boolean? = null,
val maximumHeapSize: String = "200m"
val maximumHeapSize: String = "200m",
val logLevel: String? = null
) {
fun setProvidedName(providedName: CordaX500Name?) = copy(providedName = providedName)
fun setRpcUsers(rpcUsers: List<User>) = copy(rpcUsers = rpcUsers)
@ -289,6 +291,7 @@ data class NodeParameters(
fun setCustomerOverrides(customOverrides: Map<String, Any?>) = copy(customOverrides = customOverrides)
fun setStartInSameProcess(startInSameProcess: Boolean?) = copy(startInSameProcess = startInSameProcess)
fun setMaximumHeapSize(maximumHeapSize: String) = copy(maximumHeapSize = maximumHeapSize)
fun ssetLogLevel(logLevel: String?) = copy(logLevel = logLevel)
}
/**
@ -314,7 +317,7 @@ data class NodeParameters(
* and may be specified in [DriverDSL.startNode].
* @param portAllocation The port allocation strategy to use for the messaging and the web server addresses. Defaults to incremental.
* @param debugPortAllocation The port allocation strategy to use for jvm debugging. Defaults to incremental.
* @param systemProperties A Map of extra system properties which will be given to each new node. Defaults to empty.
* @param extraSystemProperties A Map of extra system properties which will be given to each new node. Defaults to empty.
* @param useTestClock If true the test clock will be used in Node.
* @param networkMapStartStrategy Determines whether a network map node is started automatically.
* @param startNodesInProcess Provides the default behaviour of whether new nodes should start inside this process or
@ -328,7 +331,7 @@ fun <A> driver(
driverDirectory: Path = defaultParameters.driverDirectory,
portAllocation: PortAllocation = defaultParameters.portAllocation,
debugPortAllocation: PortAllocation = defaultParameters.debugPortAllocation,
systemProperties: Map<String, String> = defaultParameters.systemProperties,
extraSystemProperties: Map<String, String> = defaultParameters.extraSystemProperties,
useTestClock: Boolean = defaultParameters.useTestClock,
initialiseSerialization: Boolean = defaultParameters.initialiseSerialization,
networkMapStartStrategy: NetworkMapStartStrategy = defaultParameters.networkMapStartStrategy,
@ -340,7 +343,7 @@ fun <A> driver(
driverDsl = DriverDSL(
portAllocation = portAllocation,
debugPortAllocation = debugPortAllocation,
systemProperties = systemProperties,
extraSystemProperties = extraSystemProperties,
driverDirectory = driverDirectory.toAbsolutePath(),
useTestClock = useTestClock,
isDebug = isDebug,
@ -376,7 +379,7 @@ data class DriverParameters(
val driverDirectory: Path = Paths.get("build", getTimestampAsDirectoryName()),
val portAllocation: PortAllocation = PortAllocation.Incremental(10000),
val debugPortAllocation: PortAllocation = PortAllocation.Incremental(5005),
val systemProperties: Map<String, String> = emptyMap(),
val extraSystemProperties: Map<String, String> = emptyMap(),
val useTestClock: Boolean = false,
val initialiseSerialization: Boolean = true,
val networkMapStartStrategy: NetworkMapStartStrategy = NetworkMapStartStrategy.Dedicated(startAutomatically = true),
@ -387,7 +390,7 @@ data class DriverParameters(
fun setDriverDirectory(driverDirectory: Path) = copy(driverDirectory = driverDirectory)
fun setPortAllocation(portAllocation: PortAllocation) = copy(portAllocation = portAllocation)
fun setDebugPortAllocation(debugPortAllocation: PortAllocation) = copy(debugPortAllocation = debugPortAllocation)
fun setSystemProperties(systemProperties: Map<String, String>) = copy(systemProperties = systemProperties)
fun setExtraSystemProperties(extraSystemProperties: Map<String, String>) = copy(extraSystemProperties = extraSystemProperties)
fun setUseTestClock(useTestClock: Boolean) = copy(useTestClock = useTestClock)
fun setInitialiseSerialization(initialiseSerialization: Boolean) = copy(initialiseSerialization = initialiseSerialization)
fun setNetworkMapStartStrategy(networkMapStartStrategy: NetworkMapStartStrategy) = copy(networkMapStartStrategy = networkMapStartStrategy)
@ -601,7 +604,7 @@ class ShutdownManager(private val executorService: ExecutorService) {
class DriverDSL(
val portAllocation: PortAllocation,
val debugPortAllocation: PortAllocation,
val systemProperties: Map<String, String>,
val extraSystemProperties: Map<String, String>,
val driverDirectory: Path,
val useTestClock: Boolean,
val isDebug: Boolean,
@ -614,6 +617,8 @@ class DriverDSL(
val executorService get() = _executorService!!
private var _shutdownManager: ShutdownManager? = null
override val shutdownManager get() = _shutdownManager!!
private val databaseNamesByNode = mutableMapOf<CordaX500Name, String>()
val systemProperties by lazy { System.getProperties().toList().map { it.first.toString() to it.second.toString() }.toMap() + extraSystemProperties }
private val cordappPackages = extraCordappPackagesToScan + getCallerPackage()
class State {
@ -697,7 +702,8 @@ class DriverDSL(
verifierType: VerifierType,
customOverrides: Map<String, Any?>,
startInSameProcess: Boolean?,
maximumHeapSize: String
maximumHeapSize: String,
logLevel: String?
): CordaFuture<NodeHandle> {
val p2pAddress = portAllocation.nextHostAndPort()
val rpcAddress = portAllocation.nextHostAndPort()
@ -722,7 +728,7 @@ class DriverDSL(
"verifierType" to verifierType.name
) + customOverrides
)
return startNodeInternal(config, webAddress, startInSameProcess, maximumHeapSize)
return startNodeInternal(name, config, webAddress, startInSameProcess, maximumHeapSize, logLevel)
}
override fun startNotaryNode(providedName: CordaX500Name,
@ -750,7 +756,7 @@ class DriverDSL(
"rpcUsers" to if (rpcUsers.isEmpty()) defaultRpcUserList else rpcUsers
)
)
startNodeInternal(config, webAddress, startInSameProcess, maximumHeapSize)
startNodeInternal(name, config, webAddress, startInSameProcess, maximumHeapSize)
}
}
@ -850,6 +856,7 @@ class DriverDSL(
val webAddress = portAllocation.nextHostAndPort()
val rpcAddress = portAllocation.nextHostAndPort()
val networkMapLegalName = networkMapStartStrategy.legalName
val config = ConfigHelper.loadConfig(
baseDirectory = baseDirectory(networkMapLegalName),
allowMissingConfig = true,
@ -863,13 +870,23 @@ class DriverDSL(
"p2pAddress" to dedicatedNetworkMapAddress.toString(),
"useTestClock" to useTestClock)
)
return startNodeInternal(config, webAddress, startInProcess, maximumHeapSize)
return startNodeInternal(networkMapLegalName, config, webAddress, startInProcess, maximumHeapSize)
}
private fun startNodeInternal(config: Config, webAddress: NetworkHostAndPort, startInProcess: Boolean?, maximumHeapSize: String): CordaFuture<NodeHandle> {
val nodeConfiguration = config.parseAs<FullNodeConfiguration>()
private fun startNodeInternal(name: CordaX500Name, config: Config, webAddress: NetworkHostAndPort, startInProcess: Boolean?, maximumHeapSize: String, logLevel: String? = null): CordaFuture<NodeHandle> {
val globalDataSourceProperties = mutableMapOf<String, Any?>()
val overriddenDatasourceUrl = systemProperties["dataSourceProperties.dataSource.url"]
overriddenDatasourceUrl?.let {
val connectionString = overriddenDatasourceUrl + "/" + databaseNamesByNode.computeIfAbsent(name, { UUID.randomUUID().toString() })
globalDataSourceProperties["dataSourceProperties.dataSource.url"] = connectionString
}
val enhancedConfig = config + globalDataSourceProperties
val nodeConfiguration = (enhancedConfig).parseAs<FullNodeConfiguration>()
if (startInProcess ?: startNodesInProcess) {
val nodeAndThreadFuture = startInProcessNode(executorService, nodeConfiguration, config, cordappPackages)
val nodeAndThreadFuture = startInProcessNode(executorService, nodeConfiguration, enhancedConfig, cordappPackages)
shutdownManager.registerShutdown(
nodeAndThreadFuture.map { (node, thread) ->
{
@ -887,7 +904,7 @@ class DriverDSL(
}
} else {
val debugPort = if (isDebug) debugPortAllocation.nextPort() else null
val processFuture = startOutOfProcessNode(executorService, nodeConfiguration, config, quasarJarPath, debugPort, systemProperties, cordappPackages, maximumHeapSize)
val processFuture = startOutOfProcessNode(executorService, nodeConfiguration, enhancedConfig, quasarJarPath, debugPort, systemProperties, cordappPackages, maximumHeapSize, logLevel)
registerProcess(processFuture)
return processFuture.flatMap { process ->
val processDeathFuture = poll(executorService, "process death") {
@ -956,7 +973,8 @@ class DriverDSL(
debugPort: Int?,
overriddenSystemProperties: Map<String, String>,
cordappPackages: List<String>,
maximumHeapSize: String
maximumHeapSize: String,
logLevel: String? = null
): CordaFuture<Process> {
val processFuture = executorService.fork {
log.info("Starting out-of-process Node ${nodeConf.myLegalName.organisation}")
@ -967,13 +985,14 @@ class DriverDSL(
"name" to nodeConf.myLegalName,
"visualvm.display.name" to "corda-${nodeConf.myLegalName}",
Node.scanPackagesSystemProperty to cordappPackages.joinToString(Node.scanPackagesSeparator),
"java.io.tmpdir" to System.getProperty("java.io.tmpdir") // Inherit from parent process
"java.io.tmpdir" to System.getProperty("java.io.tmpdir"), // Inherit from parent process
"user.dir" to nodeConf.baseDirectory
)
// See experimental/quasar-hook/README.md for how to generate.
val excludePattern = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**)"
val extraJvmArguments = systemProperties.map { "-D${it.key}=${it.value}" } +
"-javaagent:$quasarJarPath=$excludePattern"
val loggingLevel = if (debugPort == null) "INFO" else "DEBUG"
val loggingLevel = logLevel ?: if (debugPort == null) "INFO" else "DEBUG"
ProcessUtilities.startCordaProcess(
className = "net.corda.node.Corda", // cannot directly get class for this, so just use string

16
testing/node-driver/src/main/kotlin/net/corda/testing/node/MockServices.kt
View File

@ -60,6 +60,10 @@ open class MockServices(
@JvmStatic
val MOCK_VERSION_INFO = VersionInfo(1, "Mock release", "Mock revision", "Mock Vendor")
private val systemProperties = System.getProperties().toList().map { it.first.toString() to it.second.toString() }.toMap()
private val dbNames = mutableMapOf<String, String>()
/**
* Make properties appropriate for creating a DataSource for unit tests.
*
@ -68,11 +72,15 @@ open class MockServices(
// TODO: Can we use an X509 principal generator here?
@JvmStatic
fun makeTestDataSourceProperties(nodeName: String = SecureHash.randomSHA256().toString()): Properties {
val overriddenDatasourceUrl = systemProperties["dataSourceProperties.dataSource.url"]?.let { property ->
dbNames.computeIfAbsent(nodeName, { property + "/" + UUID.randomUUID().toString()})
}
val props = Properties()
props.setProperty("dataSourceClassName", "org.h2.jdbcx.JdbcDataSource")
props.setProperty("dataSource.url", "jdbc:h2:mem:${nodeName}_persistence;LOCK_TIMEOUT=10000;DB_CLOSE_ON_EXIT=FALSE")
props.setProperty("dataSource.user", "sa")
props.setProperty("dataSource.password", "")
props.setProperty("dataSourceClassName", systemProperties["dataSourceProperties.dataSourceClassName"] ?: "org.h2.jdbcx.JdbcDataSource")
props.setProperty("dataSource.url", overriddenDatasourceUrl ?: "jdbc:h2:mem:${nodeName}_persistence;LOCK_TIMEOUT=10000;DB_CLOSE_ON_EXIT=FALSE")
props.setProperty("dataSource.user", systemProperties["dataSourceProperties.dataSource.user"] ?: "sa")
props.setProperty("dataSource.password", systemProperties["dataSourceProperties.dataSource.password"] ?: "")
return props
}

4
verifier/src/integration-test/kotlin/net/corda/verifier/VerifierDriver.kt
View File

@ -72,7 +72,7 @@ fun <A> verifierDriver(
driverDirectory: Path = Paths.get("build", getTimestampAsDirectoryName()),
portAllocation: PortAllocation = PortAllocation.Incremental(10000),
debugPortAllocation: PortAllocation = PortAllocation.Incremental(5005),
systemProperties: Map<String, String> = emptyMap(),
extraSystemProperties: Map<String, String> = emptyMap(),
useTestClock: Boolean = false,
networkMapStartStrategy: NetworkMapStartStrategy = NetworkMapStartStrategy.Dedicated(startAutomatically = false),
startNodesInProcess: Boolean = false,
@ -83,7 +83,7 @@ fun <A> verifierDriver(
DriverDSL(
portAllocation = portAllocation,
debugPortAllocation = debugPortAllocation,
systemProperties = systemProperties,
extraSystemProperties = extraSystemProperties,
driverDirectory = driverDirectory.toAbsolutePath(),
useTestClock = useTestClock,
networkMapStartStrategy = networkMapStartStrategy,