ENT-1396: First stub on Node HA testing facility. (#856)

* ENT-1396: Skeleton of HA testing project

* ENT-1396: Tidy-up arguments parsing and introduce ScenarioRunner.

* ENT-1396: More changes for ScenarioRunner.

* ENT-1396: Further changes.

* ENT-1396: Further changes.

* ENT-1396: Improve logging.

* ENT-1396: Add TODO comments to indicate what is going to be done in the future.

.idea/compiler.xml

@@ -84,6 +84,8 @@
       <module name="gradle-plugins-cordapp_test" target="1.8" />
       <module name="graphs_main" target="1.8" />
       <module name="graphs_test" target="1.8" />
+      <module name="ha-testing_main" target="1.8" />
+      <module name="ha-testing_test" target="1.8" />
       <module name="irs-demo-cordapp_integrationTest" target="1.8" />
       <module name="irs-demo-cordapp_main" target="1.8" />
       <module name="irs-demo-cordapp_main~1" target="1.8" />

.idea/runConfigurations/HA_Testing.xml

@@ -0,0 +1,13 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="HA Testing" type="JetRunConfigurationType" factoryName="Kotlin">
+    <module name="ha-testing_main" />
+    <option name="VM_PARAMETERS" value="-Dlog4j2.debug=true" />
+    <option name="PROGRAM_PARAMETERS" value="--haNodeRpcAddress 52.174.253.60:10003 --haNodeRpcUserName corda --haNodeRpcPassword corda_is_awesome --normalNodeRpcAddress ha-testing-vm-d.westeurope.cloudapp.azure.com:10013 --normalNodeRpcUserName corda --normalNodeRpcPassword corda_is_awesome" />
+    <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
+    <option name="ALTERNATIVE_JRE_PATH" />
+    <option name="PASS_PARENT_ENVS" value="true" />
+    <option name="MAIN_CLASS_NAME" value="net.corda.haTesting.MainKt" />
+    <option name="WORKING_DIRECTORY" value="" />
+    <method />
+  </configuration>
+</component>

experimental/ha-testing/build.gradle

@@ -0,0 +1,72 @@
+/*
+ * R3 Proprietary and Confidential
+ *
+ * Copyright (c) 2018 R3 Limited.  All rights reserved.
+ *
+ * The intellectual and technical concepts contained herein are proprietary to R3 and its suppliers and are protected by trade secret law.
+ *
+ * Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited.
+ */
+
+buildscript {
+    // For sharing constants between builds
+    Properties constants = new Properties()
+    file("$projectDir/../../constants.properties").withInputStream { constants.load(it) }
+
+    ext.kotlin_version = constants.getProperty("kotlinVersion")
+    ext.byteman_version = "4.0.2"
+
+    repositories {
+        mavenLocal()
+        mavenCentral()
+        jcenter()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+    jcenter()
+}
+
+apply plugin: 'kotlin'
+apply plugin: 'kotlin-kapt'
+apply plugin: 'idea'
+apply plugin: 'net.corda.plugins.cordapp'
+
+description 'A set of tools to perform Nodes High Availability testing'
+
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+
+    cordaCompile project(":client:rpc")
+    cordaCompile project(":finance")
+
+    // Logging
+    compile "org.slf4j:log4j-over-slf4j:$slf4j_version"
+    compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
+    compile "org.apache.logging.log4j:log4j-core:$log4j_version"
+
+    // JOptSimple: command line option parsing
+    compile "net.sf.jopt-simple:jopt-simple:$jopt_simple_version"
+
+    // Byteman for runtime (termination) rules injection on the running node
+    //compile "org.jboss.byteman:byteman:$byteman_version"
+}
+
+jar {
+    archiveName = "${project.name}.jar"
+    manifest {
+        attributes(
+                'Main-Class': 'net.corda.haTesting.Main',
+                'Implementation-Title': "HA Testing",
+                'Implementation-Version': rootProject.version
+        )
+    }
+    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+}

experimental/ha-testing/src/main/kotlin/net/corda/haTesting/Main.kt

@@ -0,0 +1,54 @@
+package net.corda.haTesting
+
+import joptsimple.OptionParser
+import joptsimple.ValueConverter
+import net.corda.core.utilities.NetworkHostAndPort
+import org.slf4j.LoggerFactory
+
+fun main(args: Array<String>) {
+
+    val logger = LoggerFactory.getLogger("Main")
+
+    val parser = OptionParser()
+    MandatoryCommandLineArguments.values().forEach { argSpec -> parser.accepts(argSpec.name).withRequiredArg().withValuesConvertedBy(argSpec.valueConverter).describedAs(argSpec.description) }
+
+    val options = parser.parse(*args)
+    try {
+        MandatoryCommandLineArguments.values().forEach { require(options.has(it.name)) { "$it is a mandatory option. Please provide it." } }
+    } catch (th: Throwable) {
+        parser.printHelpOn(System.err)
+        throw th
+    }
+    try {
+        require(ScenarioRunner(options).call()) { "Scenario should pass" }
+        System.exit(0)
+    } catch (th: Throwable) {
+        logger.error("Exception in main()", th)
+        System.exit(1)
+    }
+}
+
+enum class MandatoryCommandLineArguments(val valueConverter: ValueConverter<out Any>, val description: String) {
+    haNodeRpcAddress(NetworkHostAndPortValueConverter, "High Available Node RPC address"),
+    haNodeRpcUserName(StringValueConverter, "High Available Node RPC user name"),
+    haNodeRpcPassword(StringValueConverter, "High Available Node RPC password"),
+    normalNodeRpcAddress(NetworkHostAndPortValueConverter, "Normal Node RPC address"),
+    normalNodeRpcUserName(StringValueConverter, "Normal Node RPC user name"),
+    normalNodeRpcPassword(StringValueConverter, "Normal Node RPC password"),
+}
+
+private object StringValueConverter : ValueConverter<String> {
+    override fun convert(value: String) = value
+
+    override fun valueType(): Class<out String> = String::class.java
+
+    override fun valuePattern(): String = "<free_form_text>"
+}
+
+private object NetworkHostAndPortValueConverter : ValueConverter<NetworkHostAndPort> {
+    override fun convert(value: String): NetworkHostAndPort = NetworkHostAndPort.parse(value)
+
+    override fun valueType(): Class<out NetworkHostAndPort> = NetworkHostAndPort::class.java
+
+    override fun valuePattern(): String = "<host>:<port>"
+}

experimental/ha-testing/src/main/kotlin/net/corda/haTesting/ScenarioRunner.kt

@@ -0,0 +1,96 @@
+package net.corda.haTesting
+
+import joptsimple.OptionSet
+import net.corda.client.rpc.CordaRPCClient
+import net.corda.client.rpc.CordaRPCClientConfiguration
+import net.corda.core.contracts.Amount
+import net.corda.core.crypto.SecureHash
+import net.corda.core.messaging.CordaRPCOps
+import net.corda.core.messaging.startFlow
+import net.corda.core.utilities.*
+import net.corda.finance.GBP
+import net.corda.finance.flows.CashIssueFlow
+import net.corda.finance.flows.CashPaymentFlow
+import java.util.concurrent.Callable
+
+// Responsible for executing test scenario for 2 nodes and verifying the outcome
+class ScenarioRunner(private val options: OptionSet) : Callable<Boolean> {
+
+    companion object {
+        private val logger = contextLogger()
+
+        private fun establishRpcConnection(endpoint: NetworkHostAndPort, user: String, password: String,
+                                           onError: (Throwable) -> CordaRPCOps =
+                                                   {
+                                                       logger.error("establishRpcConnection", it)
+                                                       throw it
+                                                   }): CordaRPCOps {
+            try {
+                val retryInterval = 5.seconds
+
+                val client = CordaRPCClient(endpoint,
+                    object : CordaRPCClientConfiguration {
+                        override val connectionMaxRetryInterval = retryInterval
+                    }
+                )
+                val connection = client.start(user, password)
+                return connection.proxy
+            } catch (th: Throwable) {
+                return onError(th)
+            }
+        }
+    }
+
+    override fun call(): Boolean {
+        val haNodeRpcOps = establishRpcConnection(
+                options.valueOf(MandatoryCommandLineArguments.haNodeRpcAddress.name) as NetworkHostAndPort,
+                options.valueOf(MandatoryCommandLineArguments.haNodeRpcUserName.name) as String,
+                options.valueOf(MandatoryCommandLineArguments.haNodeRpcPassword.name) as String
+        )
+        val haNodeParty = haNodeRpcOps.nodeInfo().legalIdentities.first()
+
+        val normalNodeRpcOps = establishRpcConnection(
+                options.valueOf(MandatoryCommandLineArguments.normalNodeRpcAddress.name) as NetworkHostAndPort,
+                options.valueOf(MandatoryCommandLineArguments.normalNodeRpcUserName.name) as String,
+                options.valueOf(MandatoryCommandLineArguments.normalNodeRpcPassword.name) as String
+        )
+        val normalNodeParty = normalNodeRpcOps.nodeInfo().legalIdentities.first()
+
+        val notary = normalNodeRpcOps.notaryIdentities().first()
+
+        // It is assumed that normal Node is capable of issuing.
+        // Create a unique tag for this issuance round
+        val issuerBankPartyRef = SecureHash.randomSHA256().bytes
+        val currency = GBP
+        val amount = Amount(1_000_000, currency)
+        logger.info("Trying: issue to normal, amount: $amount")
+        val issueOutcome = normalNodeRpcOps.startFlow(::CashIssueFlow, amount, OpaqueBytes(issuerBankPartyRef), notary).returnValue.getOrThrow()
+        logger.info("Success: issue to normal, amount: $amount, TX ID: ${issueOutcome.stx.tx.id}")
+
+        // TODO start a daemon thread which will talk to HA Node and installs termination schedule to it
+        // The daemon will monitor availability of HA Node and as soon as it is down and then back-up it will install
+        // the next termination schedule.
+
+        val iterCount = 10
+        val initialAmount: Long = 1000
+        require(initialAmount > iterCount)
+
+        for(iterNo in 0 until iterCount) {
+            val transferQuantity = initialAmount - iterNo
+            logger.info("Trying: normal -> ha, amount: ${transferQuantity}p")
+            val firstPayment = normalNodeRpcOps.startFlow(::CashPaymentFlow, Amount(transferQuantity, currency), haNodeParty, true).returnValue.getOrThrow()
+            logger.info("Success: normal -> ha, amount: ${transferQuantity}p, TX ID: ${firstPayment.stx.tx.id}")
+
+            logger.info("Trying: ha -> normal, amount: ${transferQuantity - 1}p")
+            // TODO: HA node may well have a period of instability, therefore the following RPC posting has to be done in re-try fashion.
+            val secondPayment = haNodeRpcOps.startFlow(::CashPaymentFlow, Amount(transferQuantity - 1, currency), normalNodeParty, true).returnValue.getOrThrow()
+            logger.info("Success: ha -> normal, amount: ${transferQuantity - 1}p, TX ID: ${secondPayment.stx.tx.id}")
+        }
+
+        // TODO: Verify
+
+        // Only then we confirm all the checks have passed.
+        return true
+    }
+
+}

experimental/ha-testing/src/main/resources/log4j2.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ R3 Proprietary and Confidential
+  ~
+  ~ Copyright (c) 2018 R3 Limited.  All rights reserved.
+  ~
+  ~ The intellectual and technical concepts contained herein are proprietary to R3 and its suppliers and are protected by trade secret law.
+  ~
+  ~ Distribution of this file or any portion thereof via any medium without the express permission of R3 is strictly prohibited.
+  -->
+
+<Configuration status="info">
+    <Properties>
+        <Property name="defaultLogLevel">info</Property>
+    </Properties>
+
+    <Appenders>
+        <Console name="Console-Appender" target="SYSTEM_OUT">
+            <PatternLayout pattern="[%-5level] %date{HH:mm:ss,SSS} [%t] (%F:%L) %c{2}.%method - %msg %X%n" />
+        </Console>
+    </Appenders>
+
+    <Loggers>
+        <Root level="${sys:defaultLogLevel}">
+            <AppenderRef ref="Console-Appender" level="${sys:defaultLogLevel}"/>
+        </Root>
+    </Loggers>
+</Configuration>

settings.gradle

@@ -35,6 +35,7 @@ include 'experimental:kryo-hook'
 include 'experimental:intellij-plugin'
 include 'experimental:flow-hook'
 include 'experimental:blobinspector'
+include 'experimental:ha-testing'
 include 'test-common'
 include 'test-utils'
 include 'smoke-test-utils'