diff --git a/.idea/compiler.xml b/.idea/compiler.xml
index 708384f2c6..cd2a14e59a 100644
--- a/.idea/compiler.xml
+++ b/.idea/compiler.xml
@@ -84,6 +84,8 @@
+
+
diff --git a/.idea/runConfigurations/HA_Testing.xml b/.idea/runConfigurations/HA_Testing.xml
new file mode 100644
index 0000000000..64aeb17dcc
--- /dev/null
+++ b/.idea/runConfigurations/HA_Testing.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/experimental/ha-testing/build.gradle b/experimental/ha-testing/build.gradle
new file mode 100644
index 0000000000..6f6a312fa8
--- /dev/null
+++ b/experimental/ha-testing/build.gradle
@@ -0,0 +1,72 @@
+/*
+ * R3 Proprietary and Confidential
+ *
+ * Copyright (c) 2018 R3 Limited. All rights reserved.
+ *
+ * The intellectual and technical concepts contained herein are proprietary to R3 and its suppliers and are protected by trade secret law.
+ *
+ * Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited.
+ */
+
+buildscript {
+ // For sharing constants between builds
+ Properties constants = new Properties()
+ file("$projectDir/../../constants.properties").withInputStream { constants.load(it) }
+
+ ext.kotlin_version = constants.getProperty("kotlinVersion")
+ ext.byteman_version = "4.0.2"
+
+ repositories {
+ mavenLocal()
+ mavenCentral()
+ jcenter()
+ }
+
+ dependencies {
+ classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+ }
+}
+
+repositories {
+ mavenLocal()
+ mavenCentral()
+ jcenter()
+}
+
+apply plugin: 'kotlin'
+apply plugin: 'kotlin-kapt'
+apply plugin: 'idea'
+apply plugin: 'net.corda.plugins.cordapp'
+
+description 'A set of tools to perform Nodes High Availability testing'
+
+dependencies {
+ compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+ compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+
+ cordaCompile project(":client:rpc")
+ cordaCompile project(":finance")
+
+ // Logging
+ compile "org.slf4j:log4j-over-slf4j:$slf4j_version"
+ compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
+ compile "org.apache.logging.log4j:log4j-core:$log4j_version"
+
+ // JOptSimple: command line option parsing
+ compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
+
+ // Byteman for runtime (termination) rules injection on the running node
+ //compile "org.jboss.byteman:byteman:$byteman_version"
+}
+
+jar {
+ archiveName = "${project.name}.jar"
+ manifest {
+ attributes(
+ 'Main-Class': 'net.corda.haTesting.Main',
+ 'Implementation-Title': "HA Testing",
+ 'Implementation-Version': rootProject.version
+ )
+ }
+ from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+}
\ No newline at end of file
diff --git a/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/Main.kt b/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/Main.kt
new file mode 100644
index 0000000000..bf0064ca8d
--- /dev/null
+++ b/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/Main.kt
@@ -0,0 +1,54 @@
+package net.corda.haTesting
+
+import joptsimple.OptionParser
+import joptsimple.ValueConverter
+import net.corda.core.utilities.NetworkHostAndPort
+import org.slf4j.LoggerFactory
+
+fun main(args: Array) {
+
+ val logger = LoggerFactory.getLogger("Main")
+
+ val parser = OptionParser()
+ MandatoryCommandLineArguments.values().forEach { argSpec -> parser.accepts(argSpec.name).withRequiredArg().withValuesConvertedBy(argSpec.valueConverter).describedAs(argSpec.description) }
+
+ val options = parser.parse(*args)
+ try {
+ MandatoryCommandLineArguments.values().forEach { require(options.has(it.name)) { "$it is a mandatory option. Please provide it." } }
+ } catch (th: Throwable) {
+ parser.printHelpOn(System.err)
+ throw th
+ }
+ try {
+ require(ScenarioRunner(options).call()) { "Scenario should pass" }
+ System.exit(0)
+ } catch (th: Throwable) {
+ logger.error("Exception in main()", th)
+ System.exit(1)
+ }
+}
+
+enum class MandatoryCommandLineArguments(val valueConverter: ValueConverter, val description: String) {
+ haNodeRpcAddress(NetworkHostAndPortValueConverter, "High Available Node RPC address"),
+ haNodeRpcUserName(StringValueConverter, "High Available Node RPC user name"),
+ haNodeRpcPassword(StringValueConverter, "High Available Node RPC password"),
+ normalNodeRpcAddress(NetworkHostAndPortValueConverter, "Normal Node RPC address"),
+ normalNodeRpcUserName(StringValueConverter, "Normal Node RPC user name"),
+ normalNodeRpcPassword(StringValueConverter, "Normal Node RPC password"),
+}
+
+private object StringValueConverter : ValueConverter {
+ override fun convert(value: String) = value
+
+ override fun valueType(): Class = String::class.java
+
+ override fun valuePattern(): String = ""
+}
+
+private object NetworkHostAndPortValueConverter : ValueConverter {
+ override fun convert(value: String): NetworkHostAndPort = NetworkHostAndPort.parse(value)
+
+ override fun valueType(): Class = NetworkHostAndPort::class.java
+
+ override fun valuePattern(): String = ":"
+}
\ No newline at end of file
diff --git a/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/ScenarioRunner.kt b/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/ScenarioRunner.kt
new file mode 100644
index 0000000000..34ab20da69
--- /dev/null
+++ b/experimental/ha-testing/src/main/kotlin/net/corda/haTesting/ScenarioRunner.kt
@@ -0,0 +1,96 @@
+package net.corda.haTesting
+
+import joptsimple.OptionSet
+import net.corda.client.rpc.CordaRPCClient
+import net.corda.client.rpc.CordaRPCClientConfiguration
+import net.corda.core.contracts.Amount
+import net.corda.core.crypto.SecureHash
+import net.corda.core.messaging.CordaRPCOps
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.*
+import net.corda.finance.GBP
+import net.corda.finance.flows.CashIssueFlow
+import net.corda.finance.flows.CashPaymentFlow
+import java.util.concurrent.Callable
+
+// Responsible for executing test scenario for 2 nodes and verifying the outcome
+class ScenarioRunner(private val options: OptionSet) : Callable {
+
+ companion object {
+ private val logger = contextLogger()
+
+ private fun establishRpcConnection(endpoint: NetworkHostAndPort, user: String, password: String,
+ onError: (Throwable) -> CordaRPCOps =
+ {
+ logger.error("establishRpcConnection", it)
+ throw it
+ }): CordaRPCOps {
+ try {
+ val retryInterval = 5.seconds
+
+ val client = CordaRPCClient(endpoint,
+ object : CordaRPCClientConfiguration {
+ override val connectionMaxRetryInterval = retryInterval
+ }
+ )
+ val connection = client.start(user, password)
+ return connection.proxy
+ } catch (th: Throwable) {
+ return onError(th)
+ }
+ }
+ }
+
+ override fun call(): Boolean {
+ val haNodeRpcOps = establishRpcConnection(
+ options.valueOf(MandatoryCommandLineArguments.haNodeRpcAddress.name) as NetworkHostAndPort,
+ options.valueOf(MandatoryCommandLineArguments.haNodeRpcUserName.name) as String,
+ options.valueOf(MandatoryCommandLineArguments.haNodeRpcPassword.name) as String
+ )
+ val haNodeParty = haNodeRpcOps.nodeInfo().legalIdentities.first()
+
+ val normalNodeRpcOps = establishRpcConnection(
+ options.valueOf(MandatoryCommandLineArguments.normalNodeRpcAddress.name) as NetworkHostAndPort,
+ options.valueOf(MandatoryCommandLineArguments.normalNodeRpcUserName.name) as String,
+ options.valueOf(MandatoryCommandLineArguments.normalNodeRpcPassword.name) as String
+ )
+ val normalNodeParty = normalNodeRpcOps.nodeInfo().legalIdentities.first()
+
+ val notary = normalNodeRpcOps.notaryIdentities().first()
+
+ // It is assumed that normal Node is capable of issuing.
+ // Create a unique tag for this issuance round
+ val issuerBankPartyRef = SecureHash.randomSHA256().bytes
+ val currency = GBP
+ val amount = Amount(1_000_000, currency)
+ logger.info("Trying: issue to normal, amount: $amount")
+ val issueOutcome = normalNodeRpcOps.startFlow(::CashIssueFlow, amount, OpaqueBytes(issuerBankPartyRef), notary).returnValue.getOrThrow()
+ logger.info("Success: issue to normal, amount: $amount, TX ID: ${issueOutcome.stx.tx.id}")
+
+ // TODO start a daemon thread which will talk to HA Node and installs termination schedule to it
+ // The daemon will monitor availability of HA Node and as soon as it is down and then back-up it will install
+ // the next termination schedule.
+
+ val iterCount = 10
+ val initialAmount: Long = 1000
+ require(initialAmount > iterCount)
+
+ for(iterNo in 0 until iterCount) {
+ val transferQuantity = initialAmount - iterNo
+ logger.info("Trying: normal -> ha, amount: ${transferQuantity}p")
+ val firstPayment = normalNodeRpcOps.startFlow(::CashPaymentFlow, Amount(transferQuantity, currency), haNodeParty, true).returnValue.getOrThrow()
+ logger.info("Success: normal -> ha, amount: ${transferQuantity}p, TX ID: ${firstPayment.stx.tx.id}")
+
+ logger.info("Trying: ha -> normal, amount: ${transferQuantity - 1}p")
+ // TODO: HA node may well have a period of instability, therefore the following RPC posting has to be done in re-try fashion.
+ val secondPayment = haNodeRpcOps.startFlow(::CashPaymentFlow, Amount(transferQuantity - 1, currency), normalNodeParty, true).returnValue.getOrThrow()
+ logger.info("Success: ha -> normal, amount: ${transferQuantity - 1}p, TX ID: ${secondPayment.stx.tx.id}")
+ }
+
+ // TODO: Verify
+
+ // Only then we confirm all the checks have passed.
+ return true
+ }
+
+}
\ No newline at end of file
diff --git a/experimental/ha-testing/src/main/resources/log4j2.xml b/experimental/ha-testing/src/main/resources/log4j2.xml
new file mode 100644
index 0000000000..510368639b
--- /dev/null
+++ b/experimental/ha-testing/src/main/resources/log4j2.xml
@@ -0,0 +1,28 @@
+
+
+
+
+
+ info
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/settings.gradle b/settings.gradle
index 9b83f51f66..344b1be33c 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -35,6 +35,7 @@ include 'experimental:kryo-hook'
include 'experimental:intellij-plugin'
include 'experimental:flow-hook'
include 'experimental:blobinspector'
+include 'experimental:ha-testing'
include 'test-common'
include 'test-utils'
include 'smoke-test-utils'