Start Jolokia agents if configured without modifying JVM options [CORDA-1042] (#2541)

* Jolokia agents are loaded dynamically if configured
* Renamed exportJmxTo (never used) to jmxMonitoringHttpPort and take it from config
* Updated documentation and tests

.idea/compiler.xml

@@ -39,6 +39,7 @@
       <module name="cordformation_main" target="1.8" />
       <module name="cordformation_runnodes" target="1.8" />
       <module name="cordformation_test" target="1.8" />
+      <module name="core_extraResource" target="1.8" />
       <module name="core_integrationTest" target="1.8" />
       <module name="core_main" target="1.8" />
       <module name="core_smokeTest" target="1.8" />

build.gradle

@@ -2,6 +2,7 @@ buildscript {
     // For sharing constants between builds
     Properties constants = new Properties()
     file("$projectDir/constants.properties").withInputStream { constants.load(it) }
+    file("${project(':node').projectDir}/src/main/resources/build.properties").withInputStream { constants.load(it) }
 
     // Our version: bump this on release.
     ext.corda_release_version = "3.0-SNAPSHOT"
@@ -40,7 +41,7 @@ buildscript {
     ext.jackson_version = '2.9.3'
     ext.jetty_version = '9.4.7.v20170914'
     ext.jersey_version = '2.25'
-    ext.jolokia_version = '1.3.7'
+    ext.jolokia_version = constants.getProperty("jolokiaAgentVersion")
     ext.assertj_version = '3.8.0'
     ext.slf4j_version = '1.7.25'
     ext.log4j_version = '2.9.1'
@@ -71,6 +72,7 @@ buildscript {
     ext.docker_compose_rule_version = '0.33.0'
     ext.selenium_version = '3.8.1'
     ext.ghostdriver_version = '2.1.0'
+    ext.eaagentloader_version = '1.0.3'
 
     // Update 121 is required for ObjectInputFilter and at time of writing 131 was latest:
     ext.java8_minUpdateVersion = '131'

docs/source/corda-configuration-file.rst

@@ -185,8 +185,8 @@ path to the node's base directory.
 
     :port: The port to start SSH server on
 
-:exportJMXTo: If set to ``http``, will enable JMX metrics reporting via the Jolokia HTTP/JSON agent.
+:jmxMonitoringHttpPort: If set, will enable JMX metrics reporting via the Jolokia HTTP/JSON agent on the corresponding port.
-    Default Jolokia access url is http://127.0.0.1:7005/jolokia/
+    Default Jolokia access url is http://127.0.0.1:port/jolokia/
 
 :transactionCacheSizeMegaBytes: Optionally specify how much memory should be used for caching of ledger transactions in memory.
             Otherwise defaults to 8MB plus 5% of all heap memory above 300MB.

docs/source/node-administration.rst

@@ -106,7 +106,7 @@ Here are a few ways to build dashboards and extract monitoring data for a node:
   It can bridge any data input to any output using their plugin system, for example, Telegraf can
   be configured to collect data from Jolokia and write to DataDog web api.
 
-The Node configuration parameter `exportJMXTo` should be set to ``http`` to ensure a Jolokia agent is instrumented with
+The Node configuration parameter `jmxMonitoringHttpPort` has to be present in order to ensure a Jolokia agent is instrumented with
 the JVM run-time.
 
 The following JMX statistics are exported:

node/build.gradle

@@ -172,6 +172,9 @@ dependencies {
     // Jsh: Testing SSH server
     integrationTestCompile group: 'com.jcraft', name: 'jsch', version: '0.1.54'
 
+    // AgentLoader: dynamic loading of JVM agents
+    compile group: 'com.ea.agentloader', name: 'ea-agent-loader', version: "${eaagentloader_version}"
+
     // Jetty dependencies for NetworkMapClient test.
     // Web stuff: for HTTP[S] servlets
     testCompile "org.eclipse.jetty:jetty-servlet:${jetty_version}"
@@ -183,7 +186,6 @@ dependencies {
     testCompile "org.glassfish.jersey.containers:jersey-container-servlet-core:${jersey_version}"
     testCompile "org.glassfish.jersey.containers:jersey-container-jetty-http:${jersey_version}"
 
-    // Jolokia JVM monitoring agent
     runtime "org.jolokia:jolokia-jvm:${jolokia_version}:agent"
 }
 

node/src/integration-test/kotlin/net/corda/node/amqp/AMQPBridgeTest.kt

@@ -138,7 +138,7 @@ class AMQPBridgeTest {
             doReturn("trustpass").whenever(it).trustStorePassword
             doReturn("cordacadevpass").whenever(it).keyStorePassword
             doReturn(artemisAddress).whenever(it).p2pAddress
-            doReturn("").whenever(it).exportJMXto
+            doReturn(null).whenever(it).jmxMonitoringHttpPort
             doReturn(emptyList<CertChainPolicyConfig>()).whenever(it).certificateChainCheckPolicies
         }
         artemisConfig.configureWithDevSSLCertificate()

node/src/integration-test/kotlin/net/corda/node/amqp/ProtonWrapperTests.kt

@@ -222,7 +222,7 @@ class ProtonWrapperTests {
             doReturn("trustpass").whenever(it).trustStorePassword
             doReturn("cordacadevpass").whenever(it).keyStorePassword
             doReturn(NetworkHostAndPort("0.0.0.0", artemisPort)).whenever(it).p2pAddress
-            doReturn("").whenever(it).exportJMXto
+            doReturn(null).whenever(it).jmxMonitoringHttpPort
             doReturn(emptyList<CertChainPolicyConfig>()).whenever(it).certificateChainCheckPolicies
         }
         artemisConfig.configureWithDevSSLCertificate()

node/src/main/kotlin/net/corda/node/internal/AbstractNode.kt

@@ -59,6 +59,8 @@ import net.corda.node.services.vault.NodeVaultService
 import net.corda.node.services.vault.VaultSoftLockManager
 import net.corda.node.shell.InteractiveShell
 import net.corda.node.utilities.AffinityExecutor
+import net.corda.node.utilities.NodeBuildProperties
+import net.corda.node.utilities.JVMAgentRegistry
 import net.corda.nodeapi.internal.DevIdentityGenerator
 import net.corda.nodeapi.internal.crypto.X509Utilities
 import net.corda.nodeapi.internal.persistence.CordaPersistence
@@ -73,6 +75,7 @@ import rx.Observable
 import rx.Scheduler
 import java.io.IOException
 import java.lang.reflect.InvocationTargetException
+import java.nio.file.Paths
 import java.security.KeyPair
 import java.security.KeyStoreException
 import java.security.PublicKey
@@ -192,6 +195,7 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
         check(started == null) { "Node has already been started" }
         log.info("Node starting up ...")
         initCertificate()
+        initialiseJVMAgents()
         val schemaService = NodeSchemaService(cordappLoader.cordappSchemas, configuration.notary != null)
         val (identity, identityKeyPair) = obtainIdentity(notaryConfig = null)
         val identityService = makeIdentityService(identity.certificate)
@@ -749,6 +753,22 @@ abstract class AbstractNode(val configuration: NodeConfiguration,
         return NodeVaultService(platformClock, keyManagementService, stateLoader, hibernateConfig)
     }
 
+    /** Load configured JVM agents */
+    private fun initialiseJVMAgents() {
+        configuration.jmxMonitoringHttpPort?.let { port ->
+            requireNotNull(NodeBuildProperties.JOLOKIA_AGENT_VERSION) {
+                "'jolokiaAgentVersion' missing from build properties"
+            }
+            log.info("Starting Jolokia agent on HTTP port: $port")
+            val libDir = Paths.get(configuration.baseDirectory.toString(), "drivers")
+            val jarFilePath = JVMAgentRegistry.resolveAgentJar(
+                    "jolokia-jvm-${NodeBuildProperties.JOLOKIA_AGENT_VERSION}-agent.jar", libDir) ?:
+                    throw Error("Unable to locate agent jar file")
+            log.info("Agent jar file: $jarFilePath")
+            JVMAgentRegistry.attach("jolokia", "port=$port", jarFilePath)
+        }
+    }
+
     private inner class ServiceHubInternalImpl(
             override val identityService: IdentityService,
             // Place the long term identity key in the KMS. Eventually, this is likely going to be separated again because

node/src/main/kotlin/net/corda/node/internal/Node.kt

@@ -192,9 +192,9 @@ open class Node(configuration: NodeConfiguration,
                 val rpcBrokerDirectory: Path = baseDirectory / "brokers" / "rpc"
                 with(rpcOptions) {
                     rpcBroker = if (useSsl) {
-                        ArtemisRpcBroker.withSsl(this.address!!, sslConfig, securityManager, certificateChainCheckPolicies, networkParameters.maxMessageSize, exportJMXto.isNotEmpty(), rpcBrokerDirectory)
+                        ArtemisRpcBroker.withSsl(this.address!!, sslConfig, securityManager, certificateChainCheckPolicies, networkParameters.maxMessageSize, jmxMonitoringHttpPort != null, rpcBrokerDirectory)
                     } else {
-                        ArtemisRpcBroker.withoutSsl(this.address!!, adminAddress!!, sslConfig, securityManager, certificateChainCheckPolicies, networkParameters.maxMessageSize, exportJMXto.isNotEmpty(), rpcBrokerDirectory)
+                        ArtemisRpcBroker.withoutSsl(this.address!!, adminAddress!!, sslConfig, securityManager, certificateChainCheckPolicies, networkParameters.maxMessageSize, jmxMonitoringHttpPort != null, rpcBrokerDirectory)
                     }
                 }
                 return rpcBroker!!.addresses

node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

@@ -25,7 +25,7 @@ val Int.MB: Long get() = this * 1024L * 1024L
 interface NodeConfiguration : NodeSSLConfiguration {
     val myLegalName: CordaX500Name
     val emailAddress: String
-    val exportJMXto: String
+    val jmxMonitoringHttpPort: Int?
     val dataSourceProperties: Properties
     val rpcUsers: List<User>
     val security: SecurityConfiguration?
@@ -118,6 +118,7 @@ data class NodeConfigurationImpl(
         /** This is not retrieved from the config file but rather from a command line argument. */
         override val baseDirectory: Path,
         override val myLegalName: CordaX500Name,
+        override val jmxMonitoringHttpPort: Int? = null,
         override val emailAddress: String,
         override val keyStorePassword: String,
         override val trustStorePassword: String,
@@ -184,7 +185,6 @@ data class NodeConfigurationImpl(
         return errors
     }
 
-    override val exportJMXto: String get() = "http"
     override val transactionCacheSizeBytes: Long
         get() = transactionCacheSizeMegaBytes?.MB ?: super.transactionCacheSizeBytes
     override val attachmentContentCacheSizeBytes: Long

node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt

@@ -144,7 +144,7 @@ class ArtemisMessagingServer(private val config: NodeConfiguration,
         managementNotificationAddress = SimpleString(NOTIFICATIONS_ADDRESS)
 
         // JMX enablement
-        if (config.exportJMXto.isNotEmpty()) {
+        if (config.jmxMonitoringHttpPort != null) {
             isJMXManagementEnabled = true
             isJMXUseBrokerName = true
         }

node/src/main/kotlin/net/corda/node/utilities/JVMAgentRegistry.kt

@@ -0,0 +1,50 @@
+package net.corda.node.utilities
+
+import com.ea.agentloader.AgentLoader
+import net.corda.core.internal.exists
+import net.corda.core.internal.isRegularFile
+import java.net.URLClassLoader
+import java.nio.file.Path
+import java.nio.file.Paths
+import java.util.concurrent.ConcurrentHashMap
+
+/**
+ * Helper class for loading JVM agents dynamically
+ */
+object JVMAgentRegistry {
+
+    /**
+     * Names and options of loaded agents
+     */
+    val loadedAgents = ConcurrentHashMap<String, String>()
+
+    /**
+     * Load and attach agent located at given [jar], unless [loadedAgents]
+     * indicate that one of its instance has been already loaded.
+     */
+    fun attach(agentName: String, options: String, jar: Path) {
+        loadedAgents.computeIfAbsent(agentName.toLowerCase()) {
+            AgentLoader.loadAgent(jar.toString(), options)
+            options
+        }
+    }
+
+    /**
+     * Attempt finding location of jar for given agent by first searching into
+     * "drivers" directory of [nodeBaseDirectory] and then falling back to
+     * classpath. Returns null if no match is found.
+     */
+    fun resolveAgentJar(jarFileName: String, driversDir: Path): Path? {
+        require(jarFileName.endsWith(".jar")) { "jarFileName does not have .jar suffix" }
+
+        val path = Paths.get(driversDir.toString(), jarFileName)
+        return if (path.exists() && path.isRegularFile()) {
+            path
+        } else {
+            (this::class.java.classLoader as? URLClassLoader)
+                ?.urLs
+                ?.map { Paths.get(it.path) }
+                ?.firstOrNull { it.fileName.toString() == jarFileName }
+        }
+    }
+}

node/src/main/kotlin/net/corda/node/utilities/NodeBuildProperties.kt

@@ -0,0 +1,27 @@
+package net.corda.node.utilities
+
+import java.util.*
+
+/**
+ * Expose properties defined in top-level 'constants.properties' file.
+ */
+object NodeBuildProperties {
+
+    // Note: initialization order is important
+    private val data by lazy {
+        Properties().apply {
+            NodeBuildProperties::class.java.getResourceAsStream("/build.properties")
+                    ?.let { load(it) }
+        }
+    }
+
+    /**
+     * Jolokia dependency version
+     */
+    val JOLOKIA_AGENT_VERSION = get("jolokiaAgentVersion")
+
+    /**
+     * Get property value by name
+     */
+    fun get(key: String): String? = data.getProperty(key)
+}

node/src/main/resources/build.properties

@@ -0,0 +1,5 @@
+# Build constants exported as resource file to make them visible in Node program
+# Note: sadly, due to present limitation of IntelliJ-IDEA in processing resource files, these constants cannot be
+# imported from top-level 'constants.properties' file
+
+jolokiaAgentVersion=1.3.7

node/src/test/kotlin/net/corda/node/services/messaging/ArtemisMessagingTest.kt

@@ -67,7 +67,7 @@ class ArtemisMessagingTest {
             doReturn("trustpass").whenever(it).trustStorePassword
             doReturn("cordacadevpass").whenever(it).keyStorePassword
             doReturn(NetworkHostAndPort("0.0.0.0", serverPort)).whenever(it).p2pAddress
-            doReturn("").whenever(it).exportJMXto
+            doReturn(null).whenever(it).jmxMonitoringHttpPort
             doReturn(emptyList<CertChainPolicyConfig>()).whenever(it).certificateChainCheckPolicies
             doReturn(5).whenever(it).messageRedeliveryDelaySeconds
         }

testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt

@@ -91,10 +91,11 @@ class DriverTests {
 
     @Test
     fun `monitoring mode enables jolokia exporting of JMX metrics via HTTP JSON`() {
-        driver(DriverParameters(jmxPolicy = JmxPolicy(true))) {
+        driver(DriverParameters(startNodesInProcess = false)) {
             // start another node so we gain access to node JMX metrics
-            startNode(providedName = DUMMY_REGULATOR_NAME).getOrThrow()
             val webAddress = NetworkHostAndPort("localhost", 7006)
+            startNode(providedName = DUMMY_REGULATOR_NAME,
+                      customOverrides = mapOf("jmxMonitoringHttpPort" to webAddress.port)).getOrThrow()
             // request access to some JMX metrics via Jolokia HTTP/JSON
             val api = HttpApi.fromHostAndPort(webAddress, "/jolokia/")
             val versionAsJson = api.getJson<JSONObject>("/jolokia/version/")

testing/node-driver/src/main/kotlin/net/corda/testing/node/internal/InternalMockNetwork.kt

@@ -469,7 +469,7 @@ private fun mockNodeConfiguration(): NodeConfiguration {
         doReturn(null).whenever(it).notary
         doReturn(DatabaseConfig()).whenever(it).database
         doReturn("").whenever(it).emailAddress
-        doReturn("").whenever(it).exportJMXto
+        doReturn(null).whenever(it).jmxMonitoringHttpPort
         doReturn(true).whenever(it).devMode
         doReturn(null).whenever(it).compatibilityZoneURL
         doReturn(emptyList<CertChainPolicyConfig>()).whenever(it).certificateChainCheckPolicies