SSH tunnelling utility. Property files per remote host.

tools/jmeter/build.gradle

@@ -6,7 +6,8 @@ mainClassName = 'net.corda.jmeter.Launcher'
 
 dependencies {
     compile project(':client:rpc')
-    compile project(":finance")
+    compile project(':finance')
+    compile project(':tools:loadtest')
 
     // JMeter
     ext.jmVersion = "3.3"
@@ -42,8 +43,18 @@ task(runServer, dependsOn: 'classes', type: JavaExec) {
              "-s" ]
 }
 
+task(runSsh, dependsOn: 'classes', type: JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    main = 'net.corda.jmeter.Ssh'
+    if ( project.hasProperty("jmeterHosts") ) {
+        args Eval.me(jmeterHosts)
+    }
+    standardInput = System.in
+}
+
 run {
     systemProperty "search_paths", project(':tools:jmeter').configurations.runtime.files.join(";")
+    systemProperty "java.rmi.server.hostname", "localhost"
     systemProperty "jmeter.home", sourceSets.main.resources.getSrcDirs().first().getPath()
     // If you want to debug: jvmArgs += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
     args+= [ "-p", sourceSets.main.resources.getSrcDirs().first().getPath()+"/jmeter.properties",
@@ -70,12 +81,14 @@ task buildJMeterJAR(type: FatCapsule, dependsOn: 'jar') {
             project(':tools:jmeter').jar
     )
     from 'NOTICE' // Copy CDDL notice
-    from { "$rootDir/tools/jmeter/build/resources/main/jmeter.properties" }
+    from("$rootDir/tools/jmeter/build/resources/main") {
-    from { "$rootDir/tools/jmeter/build/resources/main/log4j2.xml" }
+        include "log4j2.xml"
+        include "*.properties"
+    }
 
     capsuleManifest {
         applicationVersion = corda_release_version
-        systemProperties['java.rmi.server.hostname'] = '0.0.0.0'
+        systemProperties['java.rmi.server.hostname'] = 'localhost'
         minJavaVersion = '1.8.0'
         minUpdateVersion['1.8'] = java8_minUpdateVersion
 

tools/jmeter/src/main/kotlin/net/corda/jmeter/Launcher.kt

@@ -3,6 +3,7 @@ package net.corda.jmeter
 import net.corda.core.internal.div
 import org.apache.jmeter.JMeter
 import org.slf4j.LoggerFactory
+import java.net.InetAddress
 import java.nio.file.Files
 import java.nio.file.Paths
 import kotlin.streams.asSequence
@@ -18,8 +19,9 @@ class Launcher {
                 // We are running under Capsule, so assume we want a JMeter slave server to be controlled from
                 // elsewhere.
                 logger.info("Starting JMeter in server mode from $capsuleDir")
+                val capsuleDirPath = Paths.get(capsuleDir)
                 // Add all JMeter and Corda jars onto the JMeter search_paths
-                val searchPath = Files.list(Paths.get(capsuleDir)).asSequence().filter {
+                val searchPath = Files.list(capsuleDirPath).asSequence().filter {
                     val filename = it.fileName.toString()
                     filename.endsWith(".jar") && (filename.contains("corda") || filename.contains("jmeter", true))
                 }.joinToString(";")
@@ -28,9 +30,19 @@ class Launcher {
                 // Set the JMeter home as a property rather than command line arg, due to inconsistent code in JMeter.
                 System.setProperty("jmeter.home", capsuleDir)
                 // Create two dirs that JMeter expects, if they don't already exist.
-                Files.createDirectories(Paths.get(capsuleDir) / "lib" / "ext")
+                Files.createDirectories(capsuleDirPath / "lib" / "ext")
-                Files.createDirectories(Paths.get(capsuleDir) / "lib" / "junit")
+                Files.createDirectories(capsuleDirPath / "lib" / "junit")
-                jmeter.start(arrayOf("-s", "-p", "$capsuleDir/jmeter.properties") + args)
+                // Now see if we have a hostname specific property file, and if so, add it.
+                val hostName = InetAddress.getLocalHost().hostName
+                val hostSpecificConfigFile = capsuleDirPath / "$hostName.properties"
+                logger.info("Attempting to use host-specific properties file $hostSpecificConfigFile")
+                val extraArgs = if (Files.exists(hostSpecificConfigFile)) {
+                    logger.info("Found host-specific properties file")
+                    arrayOf("-q", hostSpecificConfigFile.toString())
+                } else {
+                    emptyArray()
+                }
+                jmeter.start(arrayOf("-s", "-p", (capsuleDirPath / "jmeter.properties").toString()) + extraArgs + args)
             } else {
                 jmeter.start(args)
             }

tools/jmeter/src/main/kotlin/net/corda/jmeter/Ssh.kt

@@ -0,0 +1,102 @@
+package net.corda.jmeter
+
+import com.jcraft.jsch.JSch
+import com.jcraft.jsch.Session
+import net.corda.core.utilities.NetworkHostAndPort
+import net.corda.loadtest.setupJSchWithSshAgent
+import org.slf4j.LoggerFactory
+import java.io.BufferedReader
+import java.io.InputStreamReader
+import java.util.*
+
+
+class Ssh {
+    companion object {
+        val log = LoggerFactory.getLogger(this::class.java)
+
+        @JvmStatic
+        fun main(args: Array<String>) {
+            val userName = System.getProperty("user.name")
+            val jsch = setupJSchWithSshAgent()
+            val sessions = mutableListOf<Session>()
+
+            // Read jmeter.properties
+            // For each host:port combo, map them to hosts from command line
+
+            val jmeterProps = loadProps("/jmeter.properties")
+            // The port the JMeter remote agents call back to on this client host.
+            val clientRmiLocalPort = jmeterProps.getProperty("client.rmi.localport").toInt()
+            // TODO: Where is this value used?  Just on the remote agent to set up the RMI registry?
+            val serverRmiPort = jmeterProps.getProperty("server.rmi.port", "1099").toInt()
+
+            // Where JMeter driver will try to connect for remote agents (should all be localhost so can ssh tunnel).
+            val localHostsAndPorts = jmeterProps.getProperty("remote_hosts", "").split(',').map { it.trim() }
+            args.zip(localHostsAndPorts) { remoteHost, localHostAndPortString ->
+                // Actual remote host and port we will tunnel to.
+                log.info("Creating tunnels for $remoteHost")
+                val localHostAndPort = NetworkHostAndPort.parse(localHostAndPortString)
+
+                // For the remote host, load their specific property file, since it specifies remote RMI server port
+                val unqualifiedHostName = remoteHost.substringBefore('.')
+                val hostProps = loadProps("/$unqualifiedHostName.properties")
+
+                val serverRmiLocalPort = hostProps.getProperty("server.rmi.localport", jmeterProps.getProperty("server.rmi.localport")).toInt()
+
+                val session = connectToHost(jsch, remoteHost, userName)
+                sessions += session
+
+                // TODO: maybe check the local host is actually "localhost"?
+                // For tunnelling the RMI registry on the remote agent
+                // ssh ${remoteHostAndPort.host} -L 0.0.0.0:${localHostAndPort.port}:localhost:$serverRmiPort -N
+                createOutboundTunnel(session, NetworkHostAndPort("0.0.0.0", localHostAndPort.port), NetworkHostAndPort("localhost", serverRmiPort))
+
+                // For tunnelling the actual connection to the remote agent
+                // ssh ${remoteHostAndPort.host} -L 0.0.0.0:$serverRmiLocalPort:localhost:$serverRmiLocalPort -N
+                createOutboundTunnel(session, NetworkHostAndPort("0.0.0.0", serverRmiLocalPort), NetworkHostAndPort("localhost", serverRmiLocalPort))
+
+                // For returning results to the client
+                // ssh ${remoteHostAndPort.host} -R 0.0.0.0:clientRmiLocalPort:localhost:clientRmiLocalPort -N
+                createInboundTunnel(session, NetworkHostAndPort("0.0.0.0", clientRmiLocalPort), NetworkHostAndPort("localhost", clientRmiLocalPort))
+            }
+            val input = BufferedReader(InputStreamReader(System.`in`))
+
+            do {
+                log.info("Type 'quit' to exit cleanly.")
+            } while (input.readLine() != "quit")
+            sessions.forEach {
+                log.info("Closing tunnels for ${it.host}")
+                it.disconnect()
+            }
+        }
+
+        private fun loadProps(filename: String): Properties {
+            val props = Properties()
+            this::class.java.getResourceAsStream(filename).use {
+                props.load(it)
+            }
+            return props
+        }
+
+        fun connectToHost(jSch: JSch, remoteHost: String, remoteUserName: String): Session {
+            val session = jSch.getSession(remoteUserName, remoteHost, 22)
+            // We don't check the host fingerprints because they may change often
+            session.setConfig("StrictHostKeyChecking", "no")
+            log.info("Connecting to $remoteHost...")
+            session.connect()
+            log.info("Connected to $remoteHost!")
+            return session
+        }
+
+        fun createOutboundTunnel(session: Session, local: NetworkHostAndPort, remote: NetworkHostAndPort) {
+            log.info("Creating outbound tunnel from $local to $remote with ${session.host}...")
+            session.setPortForwardingL(local.host, local.port, remote.host, remote.port)
+            log.info("Tunnel created!")
+        }
+
+        fun createInboundTunnel(session: Session, local: NetworkHostAndPort, remote: NetworkHostAndPort) {
+            log.info("Creating inbound tunnel from $remote to $local on ${session.host}...")
+            session.setPortForwardingR(remote.host, remote.port, local.host, local.port)
+            log.info("Tunnel created!")
+        }
+    }
+}

tools/jmeter/src/main/resources/Java Request.jmx

@@ -15,9 +15,9 @@
         <stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
         <elementProp name="ThreadGroup.main_controller" elementType="LoopController" guiclass="LoopControlPanel" testclass="LoopController" testname="Loop Controller" enabled="true">
           <boolProp name="LoopController.continue_forever">false</boolProp>
-          <stringProp name="LoopController.loops">1000</stringProp>
+          <stringProp name="LoopController.loops">10</stringProp>
         </elementProp>
-        <stringProp name="ThreadGroup.num_threads">2</stringProp>
+        <stringProp name="ThreadGroup.num_threads">3</stringProp>
         <stringProp name="ThreadGroup.ramp_time"></stringProp>
         <longProp name="ThreadGroup.start_time">1509455820000</longProp>
         <longProp name="ThreadGroup.end_time">1509455820000</longProp>

tools/jmeter/src/main/resources/jmeter.properties

@@ -238,7 +238,7 @@ gui.quick_9=ViewResultsFullVisualizer
 #---------------------------------------------------------------------------
 
 # Remote Hosts - comma delimited
-remote_hosts=Rick-Parker.local
+remote_hosts=127.0.0.1:20099
 #remote_hosts=localhost:1099,localhost:2010
 
 # RMI port to be used by the server (must start rmiregistry with same port)
@@ -261,7 +261,7 @@ remote_hosts=Rick-Parker.local
 # Parameter that controls the RMI port used by the RemoteSampleListenerImpl (The Controler)
 # Default value is 0 which means port is randomly assigned
 # You may need to open Firewall port on the Controller machine
-#client.rmi.localport=0
+client.rmi.localport=4001
 
 # When distributed test is starting, there may be several attempts to initialize
 # remote engines. By default, only single try is made. Increase following property
@@ -280,7 +280,7 @@ remote_hosts=Rick-Parker.local
 
 # To use a specific port for the JMeter server engine, define
 # the following property before starting the server:
-#server.rmi.localport=4000
+server.rmi.localport=5000
 
 # From JMeter 2.3.1, the jmeter server creates the RMI registry as part of the server process.
 # To stop the server creating the RMI registry:

tools/jmeter/src/main/resources/perf-node-1.properties

@@ -0,0 +1 @@
+server.rmi.localport=10101

tools/jmeter/src/main/resources/perf-node-2.properties

@@ -0,0 +1 @@
+server.rmi.localport=10102

tools/jmeter/src/main/resources/perf-node-3.properties

@@ -0,0 +1 @@
+server.rmi.localport=10103

tools/jmeter/src/main/resources/perf-node-4.properties

@@ -0,0 +1 @@
+server.rmi.localport=10104

tools/jmeter/src/main/resources/perf-notary.properties

@@ -0,0 +1 @@
+server.rmi.localport=10100