From 501f6c4a1ae0903203bf9f03216a2ba28f734af2 Mon Sep 17 00:00:00 2001 From: "rick.parker" Date: Fri, 10 Nov 2017 08:33:55 +0000 Subject: [PATCH] SSH tunnelling utility. Property files per remote host. --- tools/jmeter/build.gradle | 21 +++- .../main/kotlin/net/corda/jmeter/Launcher.kt | 20 +++- .../src/main/kotlin/net/corda/jmeter/Ssh.kt | 102 ++++++++++++++++++ .../src/main/resources/Java Request.jmx | 4 +- .../src/main/resources/jmeter.properties | 6 +- .../src/main/resources/perf-node-1.properties | 1 + .../src/main/resources/perf-node-2.properties | 1 + .../src/main/resources/perf-node-3.properties | 1 + .../src/main/resources/perf-node-4.properties | 1 + .../src/main/resources/perf-notary.properties | 1 + 10 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 tools/jmeter/src/main/kotlin/net/corda/jmeter/Ssh.kt create mode 100644 tools/jmeter/src/main/resources/perf-node-1.properties create mode 100644 tools/jmeter/src/main/resources/perf-node-2.properties create mode 100644 tools/jmeter/src/main/resources/perf-node-3.properties create mode 100644 tools/jmeter/src/main/resources/perf-node-4.properties create mode 100644 tools/jmeter/src/main/resources/perf-notary.properties diff --git a/tools/jmeter/build.gradle b/tools/jmeter/build.gradle index 7e7abbd9f8..e74d78026c 100644 --- a/tools/jmeter/build.gradle +++ b/tools/jmeter/build.gradle @@ -6,7 +6,8 @@ mainClassName = 'net.corda.jmeter.Launcher' dependencies { compile project(':client:rpc') - compile project(":finance") + compile project(':finance') + compile project(':tools:loadtest') // JMeter ext.jmVersion = "3.3" @@ -42,8 +43,18 @@ task(runServer, dependsOn: 'classes', type: JavaExec) { "-s" ] } +task(runSsh, dependsOn: 'classes', type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + main = 'net.corda.jmeter.Ssh' + if ( project.hasProperty("jmeterHosts") ) { + args Eval.me(jmeterHosts) + } + standardInput = System.in +} + run { systemProperty "search_paths", project(':tools:jmeter').configurations.runtime.files.join(";") + systemProperty "java.rmi.server.hostname", "localhost" systemProperty "jmeter.home", sourceSets.main.resources.getSrcDirs().first().getPath() // If you want to debug: jvmArgs += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005" args+= [ "-p", sourceSets.main.resources.getSrcDirs().first().getPath()+"/jmeter.properties", @@ -70,12 +81,14 @@ task buildJMeterJAR(type: FatCapsule, dependsOn: 'jar') { project(':tools:jmeter').jar ) from 'NOTICE' // Copy CDDL notice - from { "$rootDir/tools/jmeter/build/resources/main/jmeter.properties" } - from { "$rootDir/tools/jmeter/build/resources/main/log4j2.xml" } + from("$rootDir/tools/jmeter/build/resources/main") { + include "log4j2.xml" + include "*.properties" + } capsuleManifest { applicationVersion = corda_release_version - systemProperties['java.rmi.server.hostname'] = '0.0.0.0' + systemProperties['java.rmi.server.hostname'] = 'localhost' minJavaVersion = '1.8.0' minUpdateVersion['1.8'] = java8_minUpdateVersion diff --git a/tools/jmeter/src/main/kotlin/net/corda/jmeter/Launcher.kt b/tools/jmeter/src/main/kotlin/net/corda/jmeter/Launcher.kt index c26dae07a1..a108da4b89 100644 --- a/tools/jmeter/src/main/kotlin/net/corda/jmeter/Launcher.kt +++ b/tools/jmeter/src/main/kotlin/net/corda/jmeter/Launcher.kt @@ -3,6 +3,7 @@ package net.corda.jmeter import net.corda.core.internal.div import org.apache.jmeter.JMeter import org.slf4j.LoggerFactory +import java.net.InetAddress import java.nio.file.Files import java.nio.file.Paths import kotlin.streams.asSequence @@ -18,8 +19,9 @@ class Launcher { // We are running under Capsule, so assume we want a JMeter slave server to be controlled from // elsewhere. logger.info("Starting JMeter in server mode from $capsuleDir") + val capsuleDirPath = Paths.get(capsuleDir) // Add all JMeter and Corda jars onto the JMeter search_paths - val searchPath = Files.list(Paths.get(capsuleDir)).asSequence().filter { + val searchPath = Files.list(capsuleDirPath).asSequence().filter { val filename = it.fileName.toString() filename.endsWith(".jar") && (filename.contains("corda") || filename.contains("jmeter", true)) }.joinToString(";") @@ -28,9 +30,19 @@ class Launcher { // Set the JMeter home as a property rather than command line arg, due to inconsistent code in JMeter. System.setProperty("jmeter.home", capsuleDir) // Create two dirs that JMeter expects, if they don't already exist. - Files.createDirectories(Paths.get(capsuleDir) / "lib" / "ext") - Files.createDirectories(Paths.get(capsuleDir) / "lib" / "junit") - jmeter.start(arrayOf("-s", "-p", "$capsuleDir/jmeter.properties") + args) + Files.createDirectories(capsuleDirPath / "lib" / "ext") + Files.createDirectories(capsuleDirPath / "lib" / "junit") + // Now see if we have a hostname specific property file, and if so, add it. + val hostName = InetAddress.getLocalHost().hostName + val hostSpecificConfigFile = capsuleDirPath / "$hostName.properties" + logger.info("Attempting to use host-specific properties file $hostSpecificConfigFile") + val extraArgs = if (Files.exists(hostSpecificConfigFile)) { + logger.info("Found host-specific properties file") + arrayOf("-q", hostSpecificConfigFile.toString()) + } else { + emptyArray() + } + jmeter.start(arrayOf("-s", "-p", (capsuleDirPath / "jmeter.properties").toString()) + extraArgs + args) } else { jmeter.start(args) } diff --git a/tools/jmeter/src/main/kotlin/net/corda/jmeter/Ssh.kt b/tools/jmeter/src/main/kotlin/net/corda/jmeter/Ssh.kt new file mode 100644 index 0000000000..74a0fa1036 --- /dev/null +++ b/tools/jmeter/src/main/kotlin/net/corda/jmeter/Ssh.kt @@ -0,0 +1,102 @@ +package net.corda.jmeter + +import com.jcraft.jsch.JSch +import com.jcraft.jsch.Session +import net.corda.core.utilities.NetworkHostAndPort +import net.corda.loadtest.setupJSchWithSshAgent +import org.slf4j.LoggerFactory +import java.io.BufferedReader +import java.io.InputStreamReader +import java.util.* + + +class Ssh { + companion object { + val log = LoggerFactory.getLogger(this::class.java) + + @JvmStatic + fun main(args: Array) { + val userName = System.getProperty("user.name") + val jsch = setupJSchWithSshAgent() + val sessions = mutableListOf() + + // Read jmeter.properties + // For each host:port combo, map them to hosts from command line + + val jmeterProps = loadProps("/jmeter.properties") + // The port the JMeter remote agents call back to on this client host. + val clientRmiLocalPort = jmeterProps.getProperty("client.rmi.localport").toInt() + // TODO: Where is this value used? Just on the remote agent to set up the RMI registry? + val serverRmiPort = jmeterProps.getProperty("server.rmi.port", "1099").toInt() + + // Where JMeter driver will try to connect for remote agents (should all be localhost so can ssh tunnel). + val localHostsAndPorts = jmeterProps.getProperty("remote_hosts", "").split(',').map { it.trim() } + args.zip(localHostsAndPorts) { remoteHost, localHostAndPortString -> + // Actual remote host and port we will tunnel to. + log.info("Creating tunnels for $remoteHost") + val localHostAndPort = NetworkHostAndPort.parse(localHostAndPortString) + + // For the remote host, load their specific property file, since it specifies remote RMI server port + val unqualifiedHostName = remoteHost.substringBefore('.') + val hostProps = loadProps("/$unqualifiedHostName.properties") + + val serverRmiLocalPort = hostProps.getProperty("server.rmi.localport", jmeterProps.getProperty("server.rmi.localport")).toInt() + + val session = connectToHost(jsch, remoteHost, userName) + sessions += session + + // TODO: maybe check the local host is actually "localhost"? + // For tunnelling the RMI registry on the remote agent + // ssh ${remoteHostAndPort.host} -L 0.0.0.0:${localHostAndPort.port}:localhost:$serverRmiPort -N + createOutboundTunnel(session, NetworkHostAndPort("0.0.0.0", localHostAndPort.port), NetworkHostAndPort("localhost", serverRmiPort)) + + // For tunnelling the actual connection to the remote agent + // ssh ${remoteHostAndPort.host} -L 0.0.0.0:$serverRmiLocalPort:localhost:$serverRmiLocalPort -N + createOutboundTunnel(session, NetworkHostAndPort("0.0.0.0", serverRmiLocalPort), NetworkHostAndPort("localhost", serverRmiLocalPort)) + + // For returning results to the client + // ssh ${remoteHostAndPort.host} -R 0.0.0.0:clientRmiLocalPort:localhost:clientRmiLocalPort -N + createInboundTunnel(session, NetworkHostAndPort("0.0.0.0", clientRmiLocalPort), NetworkHostAndPort("localhost", clientRmiLocalPort)) + } + val input = BufferedReader(InputStreamReader(System.`in`)) + + do { + log.info("Type 'quit' to exit cleanly.") + } while (input.readLine() != "quit") + sessions.forEach { + log.info("Closing tunnels for ${it.host}") + it.disconnect() + } + } + + private fun loadProps(filename: String): Properties { + val props = Properties() + this::class.java.getResourceAsStream(filename).use { + props.load(it) + } + return props + } + + fun connectToHost(jSch: JSch, remoteHost: String, remoteUserName: String): Session { + val session = jSch.getSession(remoteUserName, remoteHost, 22) + // We don't check the host fingerprints because they may change often + session.setConfig("StrictHostKeyChecking", "no") + log.info("Connecting to $remoteHost...") + session.connect() + log.info("Connected to $remoteHost!") + return session + } + + fun createOutboundTunnel(session: Session, local: NetworkHostAndPort, remote: NetworkHostAndPort) { + log.info("Creating outbound tunnel from $local to $remote with ${session.host}...") + session.setPortForwardingL(local.host, local.port, remote.host, remote.port) + log.info("Tunnel created!") + } + + fun createInboundTunnel(session: Session, local: NetworkHostAndPort, remote: NetworkHostAndPort) { + log.info("Creating inbound tunnel from $remote to $local on ${session.host}...") + session.setPortForwardingR(remote.host, remote.port, local.host, local.port) + log.info("Tunnel created!") + } + } +} \ No newline at end of file diff --git a/tools/jmeter/src/main/resources/Java Request.jmx b/tools/jmeter/src/main/resources/Java Request.jmx index 974d09bfac..e0718d0211 100644 --- a/tools/jmeter/src/main/resources/Java Request.jmx +++ b/tools/jmeter/src/main/resources/Java Request.jmx @@ -15,9 +15,9 @@ continue false - 1000 + 10 - 2 + 3 1509455820000 1509455820000 diff --git a/tools/jmeter/src/main/resources/jmeter.properties b/tools/jmeter/src/main/resources/jmeter.properties index 5fc50531b6..c4d001f71d 100644 --- a/tools/jmeter/src/main/resources/jmeter.properties +++ b/tools/jmeter/src/main/resources/jmeter.properties @@ -238,7 +238,7 @@ gui.quick_9=ViewResultsFullVisualizer #--------------------------------------------------------------------------- # Remote Hosts - comma delimited -remote_hosts=Rick-Parker.local +remote_hosts=127.0.0.1:20099 #remote_hosts=localhost:1099,localhost:2010 # RMI port to be used by the server (must start rmiregistry with same port) @@ -261,7 +261,7 @@ remote_hosts=Rick-Parker.local # Parameter that controls the RMI port used by the RemoteSampleListenerImpl (The Controler) # Default value is 0 which means port is randomly assigned # You may need to open Firewall port on the Controller machine -#client.rmi.localport=0 +client.rmi.localport=4001 # When distributed test is starting, there may be several attempts to initialize # remote engines. By default, only single try is made. Increase following property @@ -280,7 +280,7 @@ remote_hosts=Rick-Parker.local # To use a specific port for the JMeter server engine, define # the following property before starting the server: -#server.rmi.localport=4000 +server.rmi.localport=5000 # From JMeter 2.3.1, the jmeter server creates the RMI registry as part of the server process. # To stop the server creating the RMI registry: diff --git a/tools/jmeter/src/main/resources/perf-node-1.properties b/tools/jmeter/src/main/resources/perf-node-1.properties new file mode 100644 index 0000000000..3b29059afc --- /dev/null +++ b/tools/jmeter/src/main/resources/perf-node-1.properties @@ -0,0 +1 @@ +server.rmi.localport=10101 diff --git a/tools/jmeter/src/main/resources/perf-node-2.properties b/tools/jmeter/src/main/resources/perf-node-2.properties new file mode 100644 index 0000000000..97a2fed339 --- /dev/null +++ b/tools/jmeter/src/main/resources/perf-node-2.properties @@ -0,0 +1 @@ +server.rmi.localport=10102 diff --git a/tools/jmeter/src/main/resources/perf-node-3.properties b/tools/jmeter/src/main/resources/perf-node-3.properties new file mode 100644 index 0000000000..4381c5a309 --- /dev/null +++ b/tools/jmeter/src/main/resources/perf-node-3.properties @@ -0,0 +1 @@ +server.rmi.localport=10103 diff --git a/tools/jmeter/src/main/resources/perf-node-4.properties b/tools/jmeter/src/main/resources/perf-node-4.properties new file mode 100644 index 0000000000..c1a29df71d --- /dev/null +++ b/tools/jmeter/src/main/resources/perf-node-4.properties @@ -0,0 +1 @@ +server.rmi.localport=10104 diff --git a/tools/jmeter/src/main/resources/perf-notary.properties b/tools/jmeter/src/main/resources/perf-notary.properties new file mode 100644 index 0000000000..b307059e23 --- /dev/null +++ b/tools/jmeter/src/main/resources/perf-notary.properties @@ -0,0 +1 @@ +server.rmi.localport=10100