diff --git a/build.gradle b/build.gradle index e124945ddf..c927d04987 100644 --- a/build.gradle +++ b/build.gradle @@ -24,7 +24,7 @@ buildscript { ext.jackson_version = '2.9.2' ext.jetty_version = '9.4.7.v20170914' ext.jersey_version = '2.25' - ext.jolokia_version = '2.0.0-M3' + ext.jolokia_version = '1.3.7' ext.assertj_version = '3.8.0' ext.slf4j_version = '1.7.25' ext.log4j_version = '2.9.1' diff --git a/config/dev/jolokia-access.xml b/config/dev/jolokia-access.xml index 9b4acde879..41ff27a44e 100644 --- a/config/dev/jolokia-access.xml +++ b/config/dev/jolokia-access.xml @@ -1,5 +1,5 @@ - + post @@ -8,23 +8,10 @@ read + write + exec list + search + version - - - - - java.lang:type=Memory - gc - - - - - - - com.mchange.v2.c3p0:type=PooledDataSource,* - properties - - - diff --git a/config/prod/jolokia-access.xml b/config/prod/jolokia-access.xml new file mode 100644 index 0000000000..c17fd09320 --- /dev/null +++ b/config/prod/jolokia-access.xml @@ -0,0 +1,24 @@ + + + + + + + + 127.0.0.1 + localhost + + + + version + read + + + + + get + + + \ No newline at end of file diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index d84fd4eff7..01270b15a3 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -6,6 +6,9 @@ from the previous milestone release. UNRELEASED ---------- +* Exporting additional JMX metrics (artemis, hibernate statistics) and loading Jolokia agent at JVM startup when using + DriverDSL and/or cordformation node runner. + * Removed confusing property database.initDatabase, enabling its guarded behaviour with the dev-mode. In devMode Hibernate will try to create or update database schemas, otherwise it will expect relevant schemas to be present in the database (pre configured via DDL scripts or equivalent), and validate these are correct. diff --git a/docs/source/corda-configuration-file.rst b/docs/source/corda-configuration-file.rst index 346d92ff90..b8361203e4 100644 --- a/docs/source/corda-configuration-file.rst +++ b/docs/source/corda-configuration-file.rst @@ -73,6 +73,7 @@ path to the node's base directory. :serverNameTablePrefix: Prefix string to apply to all the database tables. The default is no prefix. :transactionIsolationLevel: Transaction isolation level as defined by the ``TRANSACTION_`` constants in ``java.sql.Connection``, but without the "TRANSACTION_" prefix. Defaults to REPEATABLE_READ. + :exportHibernateJMXStatistics: Whether to export Hibernate JMX statistics (caution: expensive run-time overhead) :dataSourceProperties: This section is used to configure the jdbc connection and database driver used for the nodes persistence. Currently the defaults in ``/node/src/main/resources/reference.conf`` are as shown in the first example. This is currently @@ -163,7 +164,9 @@ path to the node's base directory. Each should be a string. Only the JARs in the directories are added, not the directories themselves. This is useful for including JDBC drivers and the like. e.g. ``jarDirs = [ 'lib' ]`` -:sshd: If provided, node will start internal SSH server which will provide a management shell. It uses the same credentials - and permissions as RPC subsystem. It has one required parameter. +:sshd: If provided, node will start internal SSH server which will provide a management shell. It uses the same credentials and permissions as RPC subsystem. It has one required parameter. :port: The port to start SSH server on + +:exportJMXTo: If set to ``http``, will enable JMX metrics reporting via the Jolokia HTTP/JSON agent. + Default Jolokia access url is http://127.0.0.1:7005/jolokia/ \ No newline at end of file diff --git a/docs/source/node-administration.rst b/docs/source/node-administration.rst index 7a677c34d2..fe9b9231fd 100644 --- a/docs/source/node-administration.rst +++ b/docs/source/node-administration.rst @@ -93,6 +93,8 @@ formats for accessing MBeans, and provides client libraries to work with that pr Here are a few ways to build dashboards and extract monitoring data for a node: +* `hawtio `_ is a web based console that connects directly to JVM's that have been instrumented with a + jolokia agent. This tool provides a nice JMX dashboard very similar to the traditional JVisualVM / JConsole MBbeans original. * `JMX2Graphite `_ is a tool that can be pointed to /monitoring/json and will scrape the statistics found there, then insert them into the Graphite monitoring tool on a regular basis. It runs in Docker and can be started with a single command. @@ -105,6 +107,29 @@ Here are a few ways to build dashboards and extract monitoring data for a node: It can bridge any data input to any output using their plugin system, for example, Telegraf can be configured to collect data from Jolokia and write to DataDog web api. +The Node configuration parameter `exportJMXTo` should be set to ``http`` to ensure a Jolokia agent is instrumented with +the JVM run-time. + +The following JMX statistics are exported: + +* Corda specific metrics: flow information (total started, finished, in-flight; flow duration by flow type), attachments (count) +* Apache Artemis metrics: queue information for P2P and RPC services +* JVM statistics: classloading, garbage collection, memory, runtime, threading, operating system +* Hibernate statistics (only when node is started-up in `devMode` due to to expensive run-time costs) + +When starting Corda nodes using Cordformation runner (see :doc:`running-a-node`), you should see a startup message similar to the following: +**Jolokia: Agent started with URL http://127.0.0.1:7005/jolokia/** + +When starting Corda nodes using the `DriverDSL`, you should see a startup message in the logs similar to the following: +**Starting out-of-process Node USA Bank Corp, debug port is not enabled, jolokia monitoring port is 7005 {}** + +Several Jolokia policy based security configuration files (``jolokia-access.xml``) are available for dev, test, and prod +environments under ``/config/``. + +The following diagram illustrates Corda flow metrics visualized using `hawtio `_ : + +.. image:: resources/hawtio-jmx.png + Memory usage and tuning ----------------------- diff --git a/docs/source/resources/hawtio-jmx.png b/docs/source/resources/hawtio-jmx.png new file mode 100644 index 0000000000..42d4504830 Binary files /dev/null and b/docs/source/resources/hawtio-jmx.png differ diff --git a/gradle-plugins/cordapp/src/main/kotlin/net/corda/plugins/Utils.kt b/gradle-plugins/cordapp/src/main/kotlin/net/corda/plugins/Utils.kt index d7200ba9f8..7572cd9876 100644 --- a/gradle-plugins/cordapp/src/main/kotlin/net/corda/plugins/Utils.kt +++ b/gradle-plugins/cordapp/src/main/kotlin/net/corda/plugins/Utils.kt @@ -23,6 +23,13 @@ class Utils { project.configurations.single { it.name == "compile" }.extendsFrom(configuration) } } + fun createRuntimeConfiguration(name: String, project: Project) { + if(!project.configurations.any { it.name == name }) { + val configuration = project.configurations.create(name) + configuration.isTransitive = false + project.configurations.single { it.name == "runtime" }.extendsFrom(configuration) + } + } } } \ No newline at end of file diff --git a/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordformation.kt b/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordformation.kt index c28c8b645e..4b722a7f03 100644 --- a/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordformation.kt +++ b/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Cordformation.kt @@ -10,6 +10,8 @@ import java.io.File */ class Cordformation : Plugin { internal companion object { + const val CORDFORMATION_TYPE = "cordformationInternal" + /** * Gets a resource file from this plugin's JAR file. * @@ -31,5 +33,8 @@ class Cordformation : Plugin { override fun apply(project: Project) { Utils.createCompileConfiguration("cordapp", project) + Utils.createRuntimeConfiguration(CORDFORMATION_TYPE, project) + val jolokiaVersion = project.rootProject.ext("jolokia_version") + project.dependencies.add(CORDFORMATION_TYPE, "org.jolokia:jolokia-jvm:$jolokiaVersion:agent") } } diff --git a/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Node.kt b/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Node.kt index 90b1364126..c9db5095c7 100644 --- a/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Node.kt +++ b/gradle-plugins/cordformation/src/main/kotlin/net/corda/plugins/Node.kt @@ -1,6 +1,8 @@ package net.corda.plugins -import com.typesafe.config.* +import com.typesafe.config.ConfigFactory +import com.typesafe.config.ConfigRenderOptions +import com.typesafe.config.ConfigValueFactory import net.corda.cordform.CordformNode import org.bouncycastle.asn1.x500.X500Name import org.bouncycastle.asn1.x500.style.BCStyle @@ -90,6 +92,7 @@ class Node(private val project: Project) : CordformNode() { if (config.hasPath("webAddress")) { installWebserverJar() } + installAgentJar() installBuiltCordapp() installCordapps() installConfig() @@ -177,6 +180,29 @@ class Node(private val project: Project) : CordformNode() { } } + /** + * Installs the jolokia monitoring agent JAR to the node/drivers directory + */ + private fun installAgentJar() { + val jolokiaVersion = project.rootProject.ext("jolokia_version") + val agentJar = project.configuration("runtime").files { + (it.group == "org.jolokia") && + (it.name == "jolokia-jvm") && + (it.version == jolokiaVersion) + // TODO: revisit when classifier attribute is added. eg && (it.classifier = "agent") + }.first() // should always be the jolokia agent fat jar: eg. jolokia-jvm-1.3.7-agent.jar + project.logger.info("Jolokia agent jar: $agentJar") + if (agentJar.isFile) { + val driversDir = File(nodeDir, "drivers") + project.copy { + it.apply { + from(agentJar) + into(driversDir) + } + } + } + } + /** * Installs the configuration file to this node's directory and detokenises it. */ diff --git a/gradle-plugins/cordformation/src/noderunner/kotlin/net/corda/plugins/NodeRunner.kt b/gradle-plugins/cordformation/src/noderunner/kotlin/net/corda/plugins/NodeRunner.kt index 04b01654f3..94953584ab 100644 --- a/gradle-plugins/cordformation/src/noderunner/kotlin/net/corda/plugins/NodeRunner.kt +++ b/gradle-plugins/cordformation/src/noderunner/kotlin/net/corda/plugins/NodeRunner.kt @@ -22,6 +22,11 @@ private object debugPortAlloc { internal fun next() = basePort++ } +private object monitoringPortAlloc { + private var basePort = 7005 + internal fun next() = basePort++ +} + fun main(args: Array) { val startedProcesses = mutableListOf() val headless = GraphicsEnvironment.isHeadless() || args.contains(HEADLESS_FLAG) @@ -49,8 +54,9 @@ private abstract class JarType(private val jarName: String) { return null } val debugPort = debugPortAlloc.next() + val monitoringPort = monitoringPortAlloc.next() println("Starting $jarName in $dir on debug port $debugPort") - val process = (if (headless) ::HeadlessJavaCommand else ::TerminalWindowJavaCommand)(jarName, dir, debugPort, javaArgs, jvmArgs).start() + val process = (if (headless) ::HeadlessJavaCommand else ::TerminalWindowJavaCommand)(jarName, dir, debugPort, monitoringPort, javaArgs, jvmArgs).start() if (os == OS.MACOS) Thread.sleep(1000) return process } @@ -69,15 +75,23 @@ private abstract class JavaCommand( jarName: String, internal val dir: File, debugPort: Int?, + monitoringPort: Int?, internal val nodeName: String, init: MutableList.() -> Unit, args: List, jvmArgs: List ) { + private val jolokiaJar by lazy { + File("$dir/drivers").listFiles { _, filename -> + filename.matches("jolokia-jvm-.*-agent\\.jar$".toRegex()) + }.first().name + } + internal val command: List = mutableListOf().apply { add(getJavaPath()) addAll(jvmArgs) add("-Dname=$nodeName") null != debugPort && add("-Dcapsule.jvm.args=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=$debugPort") + null != monitoringPort && add("-Dcapsule.jvm.args=-javaagent:drivers/$jolokiaJar=port=$monitoringPort") add("-jar") add(jarName) init() @@ -89,14 +103,14 @@ private abstract class JavaCommand( internal abstract fun getJavaPath(): String } -private class HeadlessJavaCommand(jarName: String, dir: File, debugPort: Int?, args: List, jvmArgs: List) - : JavaCommand(jarName, dir, debugPort, dir.name, { add("--no-local-shell") }, args, jvmArgs) { +private class HeadlessJavaCommand(jarName: String, dir: File, debugPort: Int?, monitoringPort: Int?, args: List, jvmArgs: List) + : JavaCommand(jarName, dir, debugPort, monitoringPort, dir.name, { add("--no-local-shell") }, args, jvmArgs) { override fun processBuilder() = ProcessBuilder(command).redirectError(File("error.$nodeName.log")).inheritIO() override fun getJavaPath() = File(File(System.getProperty("java.home"), "bin"), "java").path } -private class TerminalWindowJavaCommand(jarName: String, dir: File, debugPort: Int?, args: List, jvmArgs: List) - : JavaCommand(jarName, dir, debugPort, "${dir.name}-$jarName", {}, args, jvmArgs) { +private class TerminalWindowJavaCommand(jarName: String, dir: File, debugPort: Int?, monitoringPort: Int?, args: List, jvmArgs: List) + : JavaCommand(jarName, dir, debugPort, monitoringPort, "${dir.name}-$jarName", {}, args, jvmArgs) { override fun processBuilder() = ProcessBuilder(when (os) { OS.MACOS -> { listOf("osascript", "-e", """tell app "Terminal" diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt index 30c2817fad..146bd4308d 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/CordaPersistence.kt @@ -20,7 +20,8 @@ const val NODE_DATABASE_PREFIX = "node_" data class DatabaseConfig( val initialiseSchema: Boolean = true, val serverNameTablePrefix: String = "", - val transactionIsolationLevel: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ + val transactionIsolationLevel: TransactionIsolationLevel = TransactionIsolationLevel.REPEATABLE_READ, + val exportHibernateJMXStatistics: Boolean = false ) // This class forms part of the node config and so any changes to it must be handled with care diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt index 22a664022d..4daf490c92 100644 --- a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateConfiguration.kt @@ -17,8 +17,10 @@ import org.hibernate.type.AbstractSingleColumnStandardBasicType import org.hibernate.type.descriptor.java.PrimitiveByteArrayTypeDescriptor import org.hibernate.type.descriptor.sql.BlobTypeDescriptor import org.hibernate.type.descriptor.sql.VarbinaryTypeDescriptor +import java.lang.management.ManagementFactory import java.sql.Connection import java.util.concurrent.ConcurrentHashMap +import javax.management.ObjectName import javax.persistence.AttributeConverter class HibernateConfiguration( @@ -60,9 +62,31 @@ class HibernateConfiguration( val sessionFactory = buildSessionFactory(config, metadataSources, databaseConfig.serverNameTablePrefix) logger.info("Created session factory for schemas: $schemas") + + // export Hibernate JMX statistics + if (databaseConfig.exportHibernateJMXStatistics) + initStatistics(sessionFactory) + return sessionFactory } + // NOTE: workaround suggested to overcome deprecation of StatisticsService (since Hibernate v4.0) + // https://stackoverflow.com/questions/23606092/hibernate-upgrade-statisticsservice + fun initStatistics(sessionFactory: SessionFactory) { + val statsName = ObjectName("org.hibernate:type=statistics") + val mbeanServer = ManagementFactory.getPlatformMBeanServer() + + val statisticsMBean = DelegatingStatisticsService(sessionFactory.statistics) + statisticsMBean.isStatisticsEnabled = true + + try { + mbeanServer.registerMBean(statisticsMBean, statsName) + } + catch (e: Exception) { + logger.warn(e.message) + } + } + private fun buildSessionFactory(config: Configuration, metadataSources: MetadataSources, tablePrefix: String): SessionFactory { config.standardServiceRegistryBuilder.applySettings(config.properties) val metadata = metadataSources.getMetadataBuilder(config.standardServiceRegistryBuilder.build()).run { diff --git a/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateStatistics.kt b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateStatistics.kt new file mode 100644 index 0000000000..2c08d3e77f --- /dev/null +++ b/node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/HibernateStatistics.kt @@ -0,0 +1,227 @@ +package net.corda.nodeapi.internal.persistence + +import javax.management.MXBean + +import org.hibernate.stat.Statistics +import org.hibernate.stat.SecondLevelCacheStatistics +import org.hibernate.stat.QueryStatistics +import org.hibernate.stat.NaturalIdCacheStatistics +import org.hibernate.stat.EntityStatistics +import org.hibernate.stat.CollectionStatistics + +/** + * Exposes Hibernate [Statistics] contract as JMX resource. + */ +@MXBean +interface StatisticsService : Statistics + +/** + * Implements the MXBean interface by delegating through the actual [Statistics] implementation retrieved from the + * session factory. + */ +class DelegatingStatisticsService(private val delegate: Statistics) : StatisticsService { + + override fun clear() { + delegate.clear() + } + + override fun getCloseStatementCount(): Long { + return delegate.closeStatementCount + } + + override fun getCollectionFetchCount(): Long { + return delegate.collectionFetchCount + } + + override fun getCollectionLoadCount(): Long { + return delegate.collectionLoadCount + } + + override fun getCollectionRecreateCount(): Long { + return delegate.collectionRecreateCount + } + + override fun getCollectionRemoveCount(): Long { + return delegate.collectionRemoveCount + } + + override fun getCollectionRoleNames(): Array { + return delegate.collectionRoleNames + } + + override fun getCollectionStatistics(arg0: String): CollectionStatistics { + return delegate.getCollectionStatistics(arg0) + } + + override fun getCollectionUpdateCount(): Long { + return delegate.collectionUpdateCount + } + + override fun getConnectCount(): Long { + return delegate.connectCount + } + + override fun getEntityDeleteCount(): Long { + return delegate.entityDeleteCount + } + + override fun getEntityFetchCount(): Long { + return delegate.entityFetchCount + } + + override fun getEntityInsertCount(): Long { + return delegate.entityInsertCount + } + + override fun getEntityLoadCount(): Long { + return delegate.entityLoadCount + } + + override fun getEntityNames(): Array { + return delegate.entityNames + } + + override fun getEntityStatistics(arg0: String): EntityStatistics { + return delegate.getEntityStatistics(arg0) + } + + override fun getEntityUpdateCount(): Long { + return delegate.entityUpdateCount + } + + override fun getFlushCount(): Long { + return delegate.flushCount + } + + override fun getNaturalIdCacheHitCount(): Long { + return delegate.naturalIdCacheHitCount + } + + override fun getNaturalIdCacheMissCount(): Long { + return delegate.naturalIdCacheMissCount + } + + override fun getNaturalIdCachePutCount(): Long { + return delegate.naturalIdCachePutCount + } + + override fun getNaturalIdCacheStatistics(arg0: String): NaturalIdCacheStatistics { + return delegate.getNaturalIdCacheStatistics(arg0) + } + + override fun getNaturalIdQueryExecutionCount(): Long { + return delegate.naturalIdQueryExecutionCount + } + + override fun getNaturalIdQueryExecutionMaxTime(): Long { + return delegate.naturalIdQueryExecutionMaxTime + } + + override fun getNaturalIdQueryExecutionMaxTimeRegion(): String { + return delegate.naturalIdQueryExecutionMaxTimeRegion + } + + override fun getOptimisticFailureCount(): Long { + return delegate.optimisticFailureCount + } + + override fun getPrepareStatementCount(): Long { + return delegate.prepareStatementCount + } + + override fun getQueries(): Array { + return delegate.queries + } + + override fun getQueryCacheHitCount(): Long { + return delegate.queryCacheHitCount + } + + override fun getQueryCacheMissCount(): Long { + return delegate.queryCacheMissCount + } + + override fun getQueryCachePutCount(): Long { + return delegate.queryCachePutCount + } + + override fun getQueryExecutionCount(): Long { + return delegate.queryExecutionCount + } + + override fun getQueryExecutionMaxTime(): Long { + return delegate.queryExecutionMaxTime + } + + override fun getQueryExecutionMaxTimeQueryString(): String { + return delegate.queryExecutionMaxTimeQueryString + } + + override fun getQueryStatistics(arg0: String): QueryStatistics { + return delegate.getQueryStatistics(arg0) + } + + override fun getSecondLevelCacheHitCount(): Long { + return delegate.secondLevelCacheHitCount + } + + override fun getSecondLevelCacheMissCount(): Long { + return delegate.secondLevelCacheMissCount + } + + override fun getSecondLevelCachePutCount(): Long { + return delegate.secondLevelCachePutCount + } + + override fun getSecondLevelCacheRegionNames(): Array { + return delegate.secondLevelCacheRegionNames + } + + override fun getSecondLevelCacheStatistics(arg0: String): SecondLevelCacheStatistics { + return delegate.getSecondLevelCacheStatistics(arg0) + } + + override fun getSessionCloseCount(): Long { + return delegate.sessionCloseCount + } + + override fun getSessionOpenCount(): Long { + return delegate.sessionOpenCount + } + + override fun getStartTime(): Long { + return delegate.startTime + } + + override fun getSuccessfulTransactionCount(): Long { + return delegate.successfulTransactionCount + } + + override fun getTransactionCount(): Long { + return delegate.transactionCount + } + + override fun getUpdateTimestampsCacheHitCount(): Long { + return delegate.updateTimestampsCacheHitCount + } + + override fun getUpdateTimestampsCacheMissCount(): Long { + return delegate.updateTimestampsCacheMissCount + } + + override fun getUpdateTimestampsCachePutCount(): Long { + return delegate.updateTimestampsCachePutCount + } + + override fun isStatisticsEnabled(): Boolean { + return delegate.isStatisticsEnabled + } + + override fun logSummary() { + delegate.logSummary() + } + + override fun setStatisticsEnabled(arg0: Boolean) { + delegate.isStatisticsEnabled = arg0 + } +} \ No newline at end of file diff --git a/node/build.gradle b/node/build.gradle index 07a4248964..91a5000f3d 100644 --- a/node/build.gradle +++ b/node/build.gradle @@ -43,6 +43,11 @@ sourceSets { // This prevents problems in IntelliJ with regard to duplicate source roots. processResources { from file("$rootDir/config/dev/log4j2.xml") + from file("$rootDir/config/dev/jolokia-access.xml") +} + +processTestResources { + from file("$rootDir/config/test/jolokia-access.xml") } // To find potential version conflicts, run "gradle htmlDependencyReport" and then look in @@ -174,6 +179,9 @@ dependencies { testCompile "org.glassfish.jersey.core:jersey-server:${jersey_version}" testCompile "org.glassfish.jersey.containers:jersey-container-servlet-core:${jersey_version}" testCompile "org.glassfish.jersey.containers:jersey-container-jetty-http:${jersey_version}" + + // Jolokia JVM monitoring agent + runtime "org.jolokia:jolokia-jvm:${jolokia_version}:agent" } task integrationTest(type: Test) { diff --git a/node/capsule/build.gradle b/node/capsule/build.gradle index a0f2f2f7d1..a9bb2bbbea 100644 --- a/node/capsule/build.gradle +++ b/node/capsule/build.gradle @@ -42,7 +42,7 @@ task buildCordaJAR(type: FatCapsule, dependsOn: project(':node').compileJava) { capsuleManifest { applicationVersion = corda_release_version - appClassPath = ["jolokia-agent-war-${project.rootProject.ext.jolokia_version}.war"] + appClassPath = ["jolokia-war-${project.rootProject.ext.jolokia_version}.war"] // See experimental/quasar-hook/README.md for how to generate. def quasarExcludeExpression = "x(antlr**;bftsmart**;ch**;co.paralleluniverse**;com.codahale**;com.esotericsoftware**;com.fasterxml**;com.google**;com.ibm**;com.intellij**;com.jcabi**;com.nhaarman**;com.opengamma**;com.typesafe**;com.zaxxer**;de.javakaffee**;groovy**;groovyjarjarantlr**;groovyjarjarasm**;io.atomix**;io.github**;io.netty**;jdk**;joptsimple**;junit**;kotlin**;net.bytebuddy**;net.i2p**;org.apache**;org.assertj**;org.bouncycastle**;org.codehaus**;org.crsh**;org.dom4j**;org.fusesource**;org.h2**;org.hamcrest**;org.hibernate**;org.jboss**;org.jcp**;org.joda**;org.junit**;org.mockito**;org.objectweb**;org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**)" javaAgents = ["quasar-core-${quasar_version}-jdk8.jar=${quasarExcludeExpression}"] diff --git a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt index ebb245c955..14803be925 100644 --- a/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt +++ b/node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt @@ -81,13 +81,13 @@ open class NodeStartup(val args: Array) { conf0 } - banJavaSerialisation(conf) - preNetworkRegistration(conf) - if (shouldRegisterWithNetwork(cmdlineOptions, conf)) { + banJavaSerialisation(conf) + preNetworkRegistration(conf) + if (shouldRegisterWithNetwork(cmdlineOptions, conf)) { registerWithNetwork(cmdlineOptions, conf) return true } - logStartupInfo(versionInfo, cmdlineOptions, conf) + logStartupInfo(versionInfo, cmdlineOptions, conf) try { cmdlineOptions.baseDirectory.createDirectories() diff --git a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt index cb104321c0..33ccd2bf0f 100644 --- a/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt +++ b/node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt @@ -113,7 +113,7 @@ data class NodeConfigurationImpl( // TODO See TODO above. Rename this to nodeInfoPollingFrequency and make it of type Duration override val additionalNodeInfoPollingFrequencyMsec: Long = 5.seconds.toMillis(), override val sshd: SSHDConfiguration? = null, - override val database: DatabaseConfig = DatabaseConfig(initialiseSchema = devMode) + override val database: DatabaseConfig = DatabaseConfig(initialiseSchema = devMode, exportHibernateJMXStatistics = devMode) ) : NodeConfiguration { override val exportJMXto: String get() = "http" diff --git a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt index 2b7e496b32..e5c79352f0 100644 --- a/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt +++ b/node/src/main/kotlin/net/corda/node/services/messaging/ArtemisMessagingServer.kt @@ -211,7 +211,12 @@ class ArtemisMessagingServer(private val config: NodeConfiguration, addressFullMessagePolicy = AddressFullMessagePolicy.FAIL } ) - }.configureAddressSecurity() + // JMX enablement + if (config.exportJMXto.isNotEmpty()) {isJMXManagementEnabled = true + isJMXUseBrokerName = true} + + }.configureAddressSecurity() + private fun queueConfig(name: String, address: String = name, filter: String? = null, durable: Boolean): CoreQueueConfiguration { return CoreQueueConfiguration().apply { diff --git a/node/src/main/resources/reference.conf b/node/src/main/resources/reference.conf index dc91c3d1c0..0eb49880ea 100644 --- a/node/src/main/resources/reference.conf +++ b/node/src/main/resources/reference.conf @@ -11,6 +11,7 @@ dataSourceProperties = { } database = { transactionIsolationLevel = "REPEATABLE_READ" + exportHibernateJMXStatistics = "false" } devMode = true useHTTPS = false diff --git a/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt b/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt index 37bb84424e..7c5e390960 100644 --- a/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt +++ b/testing/node-driver/src/integration-test/kotlin/net/corda/testing/driver/DriverTests.kt @@ -4,16 +4,19 @@ import net.corda.core.concurrent.CordaFuture import net.corda.core.internal.div import net.corda.core.internal.list import net.corda.core.internal.readLines +import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.getOrThrow import net.corda.node.internal.NodeStartup import net.corda.testing.DUMMY_BANK_A import net.corda.testing.DUMMY_NOTARY import net.corda.testing.DUMMY_REGULATOR import net.corda.testing.common.internal.ProjectStructure.projectRootDir +import net.corda.testing.http.HttpApi import net.corda.testing.internal.addressMustBeBound import net.corda.testing.internal.addressMustNotBeBound import net.corda.testing.node.NotarySpec import org.assertj.core.api.Assertions.assertThat +import org.json.simple.JSONObject import org.junit.Test import java.util.concurrent.Executors import java.util.concurrent.ScheduledExecutorService @@ -66,6 +69,20 @@ class DriverTests { } } + @Test + fun `monitoring mode enables jolokia exporting of JMX metrics via HTTP JSON`() { + driver(jmxPolicy = JmxPolicy(true)) { + // start another node so we gain access to node JMX metrics + startNode(providedName = DUMMY_REGULATOR.name).getOrThrow() + + val webAddress = NetworkHostAndPort("localhost", 7006) + // request access to some JMX metrics via Jolokia HTTP/JSON + val api = HttpApi.fromHostAndPort(webAddress, "/jolokia/") + val versionAsJson = api.getJson("/jolokia/version/") + assertThat(versionAsJson.getValue("status")).isEqualTo(200) + } + } + @Test fun `started node, which is not waited for in the driver, is shutdown when the driver exits`() { // First check that the process-id file is created by the node on startup, so that we can be sure our check that diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt index 5e5cf90e9d..e81696dc2f 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/driver/Driver.kt @@ -128,6 +128,10 @@ data class NodeParameters( fun setMaximumHeapSize(maximumHeapSize: String) = copy(maximumHeapSize = maximumHeapSize) } +data class JmxPolicy(val startJmxHttpServer: Boolean = false, + val jmxHttpServerPortAllocation: PortAllocation? = + if (startJmxHttpServer) PortAllocation.Incremental(7005) else null) + /** * [driver] allows one to start up nodes like this: * driver { @@ -155,6 +159,9 @@ data class NodeParameters( * not. Note that this may be overridden in [DriverDSL.startNode]. * @param notarySpecs The notaries advertised for this network. These nodes will be started automatically and will be * available from [DriverDSL.notaryHandles]. Defaults to a simple validating notary. + * @param jmxPolicy Used to specify whether to expose JMX metrics via Jolokia HHTP/JSON. Defines two attributes: + * startJmxHttpServer: indicates whether the spawned nodes should start with a Jolokia JMX agent to enable remote JMX monitoring using HTTP/JSON. + * jmxHttpServerPortAllocation: the port allocation strategy to use for remote Jolokia/JMX monitoring over HTTP. Defaults to incremental. * @param dsl The dsl itself. * @return The value returned in the [dsl] closure. */ @@ -171,6 +178,7 @@ fun driver( waitForAllNodesToFinish: Boolean = defaultParameters.waitForAllNodesToFinish, notarySpecs: List = defaultParameters.notarySpecs, extraCordappPackagesToScan: List = defaultParameters.extraCordappPackagesToScan, + jmxPolicy: JmxPolicy = JmxPolicy(), dsl: DriverDSL.() -> A ): A { return genericDriver( @@ -184,7 +192,8 @@ fun driver( startNodesInProcess = startNodesInProcess, waitForNodesToFinish = waitForAllNodesToFinish, notarySpecs = notarySpecs, - extraCordappPackagesToScan = extraCordappPackagesToScan + extraCordappPackagesToScan = extraCordappPackagesToScan, + jmxPolicy = jmxPolicy ), coerce = { it }, dsl = dsl, @@ -219,7 +228,9 @@ data class DriverParameters( val startNodesInProcess: Boolean = false, val waitForAllNodesToFinish: Boolean = false, val notarySpecs: List = listOf(NotarySpec(DUMMY_NOTARY.name)), - val extraCordappPackagesToScan: List = emptyList() + val extraCordappPackagesToScan: List = emptyList(), + val jmxPolicy: JmxPolicy = JmxPolicy() + ) { fun setIsDebug(isDebug: Boolean) = copy(isDebug = isDebug) fun setDriverDirectory(driverDirectory: Path) = copy(driverDirectory = driverDirectory) @@ -232,4 +243,5 @@ data class DriverParameters( fun setWaitForAllNodesToFinish(waitForAllNodesToFinish: Boolean) = copy(waitForAllNodesToFinish = waitForAllNodesToFinish) fun setExtraCordappPackagesToScan(extraCordappPackagesToScan: List) = copy(extraCordappPackagesToScan = extraCordappPackagesToScan) fun setNotarySpecs(notarySpecs: List) = copy(notarySpecs = notarySpecs) + fun setJmxPolicy(jmxPolicy: JmxPolicy) = copy(jmxPolicy = jmxPolicy) } diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/DriverDSLImpl.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/DriverDSLImpl.kt index b2fe9a1070..a4046764d6 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/DriverDSLImpl.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/DriverDSLImpl.kt @@ -72,6 +72,7 @@ class DriverDSLImpl( val startNodesInProcess: Boolean, val waitForNodesToFinish: Boolean, extraCordappPackagesToScan: List, + val jmxPolicy: JmxPolicy, val notarySpecs: List ) : InternalDriverDSL { private var _executorService: ScheduledExecutorService? = null @@ -96,11 +97,25 @@ class DriverDSLImpl( //TODO: remove this once we can bundle quasar properly. private val quasarJarPath: String by lazy { - val cl = ClassLoader.getSystemClassLoader() - val urls = (cl as URLClassLoader).urLs - val quasarPattern = ".*quasar.*\\.jar$".toRegex() - val quasarFileUrl = urls.first { quasarPattern.matches(it.path) } - Paths.get(quasarFileUrl.toURI()).toString() + resolveJar(".*quasar.*\\.jar$") + } + + private val jolokiaJarPath: String by lazy { + resolveJar(".*jolokia-jvm-.*-agent\\.jar$") + } + + private fun resolveJar(jarNamePattern: String): String { + return try { + val cl = ClassLoader.getSystemClassLoader() + val urls = (cl as URLClassLoader).urLs + val jarPattern = jarNamePattern.toRegex() + val jarFileUrl = urls.first { jarPattern.matches(it.path) } + Paths.get(jarFileUrl.toURI()).toString() + } + catch(e: Exception) { + log.warn("Unable to locate JAR `$jarNamePattern` on classpath: ${e.message}", e) + throw e + } } override fun shutdown() { @@ -380,7 +395,8 @@ class DriverDSLImpl( } } else { val debugPort = if (isDebug) debugPortAllocation.nextPort() else null - val process = startOutOfProcessNode(configuration, config, quasarJarPath, debugPort, systemProperties, cordappPackages, maximumHeapSize) + val monitorPort = if (jmxPolicy.startJmxHttpServer) jmxPolicy.jmxHttpServerPortAllocation?.nextPort() else null + val process = startOutOfProcessNode(configuration, config, quasarJarPath, debugPort, jolokiaJarPath, monitorPort, systemProperties, cordappPackages, maximumHeapSize) if (waitForNodesToFinish) { state.locked { processes += process @@ -473,11 +489,13 @@ class DriverDSLImpl( config: Config, quasarJarPath: String, debugPort: Int?, + jolokiaJarPath: String, + monitorPort: Int?, overriddenSystemProperties: Map, cordappPackages: List, maximumHeapSize: String ): Process { - log.info("Starting out-of-process Node ${nodeConf.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled")) + log.info("Starting out-of-process Node ${nodeConf.myLegalName.organisation}, debug port is " + (debugPort ?: "not enabled") + ", jolokia monitoring port is " + (monitorPort ?: "not enabled")) // Write node.conf writeConfig(nodeConf.baseDirectory, "node.conf", config) @@ -498,6 +516,7 @@ class DriverDSLImpl( "org.objenesis**;org.slf4j**;org.w3c**;org.xml**;org.yaml**;reflectasm**;rx**)" val extraJvmArguments = systemProperties.removeResolvedClasspath().map { "-D${it.key}=${it.value}" } + "-javaagent:$quasarJarPath=$excludePattern" + val jolokiaAgent = monitorPort?.let { "-javaagent:$jolokiaJarPath=port=$monitorPort,host=localhost" } val loggingLevel = if (debugPort == null) "INFO" else "DEBUG" return ProcessUtilities.startCordaProcess( @@ -508,7 +527,7 @@ class DriverDSLImpl( "--no-local-shell" ), jdwpPort = debugPort, - extraJvmArguments = extraJvmArguments, + extraJvmArguments = extraJvmArguments + listOfNotNull(jolokiaAgent), errorLogPath = nodeConf.baseDirectory / NodeStartup.LOGS_DIRECTORY_NAME / "error.log", workingDirectory = nodeConf.baseDirectory, maximumHeapSize = maximumHeapSize @@ -641,6 +660,7 @@ fun genericDriver( startNodesInProcess: Boolean = defaultParameters.startNodesInProcess, notarySpecs: List, extraCordappPackagesToScan: List = defaultParameters.extraCordappPackagesToScan, + jmxPolicy: JmxPolicy = JmxPolicy(), driverDslWrapper: (DriverDSLImpl) -> D, coerce: (D) -> DI, dsl: DI.() -> A ): A { @@ -656,6 +676,7 @@ fun genericDriver( startNodesInProcess = startNodesInProcess, waitForNodesToFinish = waitForNodesToFinish, extraCordappPackagesToScan = extraCordappPackagesToScan, + jmxPolicy = jmxPolicy, notarySpecs = notarySpecs ) ) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/RPCDriver.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/RPCDriver.kt index 8efac75115..086651a72e 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/RPCDriver.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/RPCDriver.kt @@ -89,6 +89,7 @@ val fakeNodeLegalName = CordaX500Name(organisation = "Not:a:real:name", locality // Use a global pool so that we can run RPC tests in parallel private val globalPortAllocation = PortAllocation.Incremental(10000) private val globalDebugPortAllocation = PortAllocation.Incremental(5005) +private val globalMonitorPortAllocation = PortAllocation.Incremental(7005) fun rpcDriver( isDebug: Boolean = false, driverDirectory: Path = Paths.get("build", getTimestampAsDirectoryName()), @@ -101,28 +102,29 @@ fun rpcDriver( extraCordappPackagesToScan: List = emptyList(), notarySpecs: List = emptyList(), externalTrace: Trace? = null, + jmxPolicy: JmxPolicy = JmxPolicy(), dsl: RPCDriverDSL.() -> A -): A { +) : A { return genericDriver( - driverDsl = RPCDriverDSL( - DriverDSLImpl( - portAllocation = portAllocation, - debugPortAllocation = debugPortAllocation, - systemProperties = systemProperties, - driverDirectory = driverDirectory.toAbsolutePath(), - useTestClock = useTestClock, - isDebug = isDebug, - startNodesInProcess = startNodesInProcess, - waitForNodesToFinish = waitForNodesToFinish, - extraCordappPackagesToScan = extraCordappPackagesToScan, - notarySpecs = notarySpecs - ), externalTrace - ), - coerce = { it }, - dsl = dsl, - initialiseSerialization = false - ) -} + driverDsl = RPCDriverDSL( + DriverDSLImpl( + portAllocation = portAllocation, + debugPortAllocation = debugPortAllocation, + systemProperties = systemProperties, + driverDirectory = driverDirectory.toAbsolutePath(), + useTestClock = useTestClock, + isDebug = isDebug, + startNodesInProcess = startNodesInProcess, + waitForNodesToFinish = waitForNodesToFinish, + extraCordappPackagesToScan = extraCordappPackagesToScan, + notarySpecs = notarySpecs, + jmxPolicy = jmxPolicy + ), externalTrace + ), + coerce = { it }, + dsl = dsl, + initialiseSerialization = false +)} private class SingleUserSecurityManager(val rpcUser: User) : ActiveMQSecurityManager3 { override fun validateUser(user: String?, password: String?) = isValid(user, password) diff --git a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/demorun/DemoRunner.kt b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/demorun/DemoRunner.kt index fb7396ec1d..60dfe3262d 100644 --- a/testing/node-driver/src/main/kotlin/net/corda/testing/internal/demorun/DemoRunner.kt +++ b/testing/node-driver/src/main/kotlin/net/corda/testing/internal/demorun/DemoRunner.kt @@ -8,6 +8,7 @@ import net.corda.core.internal.concurrent.flatMap import net.corda.core.internal.concurrent.transpose import net.corda.core.utilities.NetworkHostAndPort import net.corda.core.utilities.getOrThrow +import net.corda.testing.driver.JmxPolicy import net.corda.testing.internal.DriverDSLImpl import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver @@ -41,6 +42,7 @@ private fun CordformDefinition.runNodes(waitForAllNodesToFinish: Boolean, block: .max()!! driver( isDebug = true, + jmxPolicy = JmxPolicy(true), driverDirectory = nodesDirectory, extraCordappPackagesToScan = cordappPackages, // Notaries are manually specified in Cordform so we don't want the driver automatically starting any diff --git a/tools/explorer/src/main/kotlin/net/corda/explorer/ExplorerSimulation.kt b/tools/explorer/src/main/kotlin/net/corda/explorer/ExplorerSimulation.kt index b92dfd144b..ff22278612 100644 --- a/tools/explorer/src/main/kotlin/net/corda/explorer/ExplorerSimulation.kt +++ b/tools/explorer/src/main/kotlin/net/corda/explorer/ExplorerSimulation.kt @@ -24,6 +24,7 @@ import net.corda.node.services.Permissions.Companion.startFlow import net.corda.nodeapi.internal.config.User import net.corda.testing.ALICE import net.corda.testing.BOB +import net.corda.testing.driver.JmxPolicy import net.corda.testing.driver.NodeHandle import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver @@ -64,14 +65,14 @@ class ExplorerSimulation(private val options: OptionSet) { private fun startDemoNodes() { val portAllocation = PortAllocation.Incremental(20000) - driver(portAllocation = portAllocation, extraCordappPackagesToScan = listOf("net.corda.finance"), waitForAllNodesToFinish = true) { + driver(portAllocation = portAllocation, extraCordappPackagesToScan = listOf("net.corda.finance"), waitForAllNodesToFinish = true, jmxPolicy = JmxPolicy(true)) { // TODO : Supported flow should be exposed somehow from the node instead of set of ServiceInfo. - val alice = startNode(providedName = ALICE.name, rpcUsers = listOf(user)) + val alice = startNode(providedName = ALICE.name, rpcUsers = listOf(user), customOverrides = mapOf("devMode" to "true")) val bob = startNode(providedName = BOB.name, rpcUsers = listOf(user)) val ukBankName = CordaX500Name(organisation = "UK Bank Plc", locality = "London", country = "GB") val usaBankName = CordaX500Name(organisation = "USA Bank Corp", locality = "New York", country = "US") val issuerGBP = startNode(providedName = ukBankName, rpcUsers = listOf(manager), - customOverrides = mapOf("issuableCurrencies" to listOf("GBP"))) + customOverrides = mapOf("issuableCurrencies" to listOf("GBP"), "" to "true")) val issuerUSD = startNode(providedName = usaBankName, rpcUsers = listOf(manager), customOverrides = mapOf("issuableCurrencies" to listOf("USD"))) diff --git a/verifier/src/integration-test/kotlin/net/corda/verifier/VerifierDriver.kt b/verifier/src/integration-test/kotlin/net/corda/verifier/VerifierDriver.kt index 8ce77720f5..b5096b46fa 100644 --- a/verifier/src/integration-test/kotlin/net/corda/verifier/VerifierDriver.kt +++ b/verifier/src/integration-test/kotlin/net/corda/verifier/VerifierDriver.kt @@ -22,6 +22,7 @@ import net.corda.nodeapi.VerifierApi import net.corda.nodeapi.internal.config.NodeSSLConfiguration import net.corda.nodeapi.internal.config.SSLConfiguration import net.corda.nodeapi.internal.ArtemisMessagingComponent.Companion.NODE_USER +import net.corda.testing.driver.JmxPolicy import net.corda.testing.driver.NodeHandle import net.corda.testing.driver.PortAllocation import net.corda.testing.driver.driver @@ -59,6 +60,7 @@ fun verifierDriver( waitForNodesToFinish: Boolean = false, extraCordappPackagesToScan: List = emptyList(), notarySpecs: List = emptyList(), + jmxPolicy: JmxPolicy = JmxPolicy(), dsl: VerifierDriverDSL.() -> A ) = genericDriver( driverDsl = VerifierDriverDSL( @@ -72,7 +74,8 @@ fun verifierDriver( startNodesInProcess = startNodesInProcess, waitForNodesToFinish = waitForNodesToFinish, extraCordappPackagesToScan = extraCordappPackagesToScan, - notarySpecs = notarySpecs + notarySpecs = notarySpecs, + jmxPolicy = jmxPolicy ) ), coerce = { it }, diff --git a/webserver/build.gradle b/webserver/build.gradle index 166a8fe3aa..e0f66a360c 100644 --- a/webserver/build.gradle +++ b/webserver/build.gradle @@ -38,7 +38,7 @@ dependencies { compile "org.eclipse.jetty:jetty-servlet:$jetty_version" compile "org.eclipse.jetty:jetty-webapp:$jetty_version" compile "javax.servlet:javax.servlet-api:3.1.0" - compile "org.jolokia:jolokia-agent-war:$jolokia_version" + compile "org.jolokia:jolokia-war:$jolokia_version" compile "commons-fileupload:commons-fileupload:$fileupload_version" // Log4J: logging framework (with SLF4J bindings) diff --git a/webserver/src/main/kotlin/net/corda/webserver/internal/NodeWebServer.kt b/webserver/src/main/kotlin/net/corda/webserver/internal/NodeWebServer.kt index 46bdf5a7ef..e5a2ce254f 100644 --- a/webserver/src/main/kotlin/net/corda/webserver/internal/NodeWebServer.kt +++ b/webserver/src/main/kotlin/net/corda/webserver/internal/NodeWebServer.kt @@ -58,7 +58,7 @@ class NodeWebServer(val config: WebServerConfig) { // Export JMX monitoring statistics and data over REST/JSON. if (config.exportJMXto.split(',').contains("http")) { val classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator")) - val warpath = classpath.firstOrNull { it.contains("jolokia-agent-war-2") && it.endsWith(".war") } + val warpath = classpath.firstOrNull { it.contains("jolokia-war") && it.endsWith(".war") } if (warpath != null) { handlerCollection.addHandler(WebAppContext().apply { // Find the jolokia WAR file on the classpath.