Export JMX statistics via JSON REST endpoints using Jolokia.

build.gradle

@@ -94,7 +94,9 @@ dependencies {
 
     // Web stuff: for HTTP[S] servlets
     compile "org.eclipse.jetty:jetty-servlet:${jetty_version}"
+    compile "org.eclipse.jetty:jetty-webapp:${jetty_version}"
     compile "javax.servlet:javax.servlet-api:3.1.0"
+    compile "org.jolokia:jolokia-agent-war:2.0.0-M1"
     compile "commons-fileupload:commons-fileupload:1.3.1"
 
     // Unit testing helpers.

docs/source/node-administration.rst

@@ -4,6 +4,34 @@ Node administration
 When a node is running, it exposes an embedded web server that lets you monitor it, upload and download attachments,
 access a REST API and so on.
 
+Monitoring your node
+--------------------
+
+Like most Java servers, the node exports various useful metrics and management operations via the industry-standard
+`JMX infrastructure <https://en.wikipedia.org/wiki/Java_Management_Extensions>`_. JMX is a standard _in-process_ API
+for registering so-called _MBeans_ ... objects whose properties and methods are intended for server management. It does
+not require any particular network protocol for export. So this data can be exported from the node in various ways:
+some monitoring systems provide a "Java Agent", which is essentially a JVM plugin that finds all the MBeans and sends
+them out to a statistics collector over the network. For those systems, follow the instructions provided by the vendor.
+
+Sometimes though, you just want raw access to the data and operations itself. So nodes export them over HTTP on the
+`/monitoring/json` HTTP endpoint, using a program called `Jolokia <https://jolokia.org/>`_. Jolokia defines the JSON
+and REST formats for accessing MBeans, and provides client libraries to work with that protocol as well.
+
+Here are a few ways to build dashboards and extract monitoring data for a node:
+
+* `JMX2Graphite <https://github.com/logzio/jmx2graphite>`_ is a tool that can be pointed to /monitoring/json and will
+  scrape the statistics found there, then insert them into the Graphite monitoring tool on a regular basis. It runs
+  in Docker and can be started with a single command.
+* `JMXTrans <https://github.com/jmxtrans/jmxtrans>`_ is another tool for Graphite, this time, it's got its own agent
+  (JVM plugin) which reads a custom config file and exports only the named data. It's more configurable than
+  JMX2Graphite and doesn't require a separate process, as the JVM will write directly to Graphite.
+* *Java Mission Control* is a desktop app that can connect to a target JVM that has the right command line flags set
+  (or always, if running locally). You can explore what data is available, create graphs of those metrics, and invoke
+  management operations like forcing a garbage collection.
+* Cloud metrics services like New Relic also understand JMX, typically, by providing their own agent that uploads the
+  data to their service on a regular schedule.
+
 Uploading and downloading attachments
 -------------------------------------
 

src/main/kotlin/core/node/Node.kt

@@ -16,7 +16,9 @@ import core.node.servlets.AttachmentDownloadServlet
 import core.node.servlets.DataUploadServlet
 import core.utilities.loggerFor
 import org.eclipse.jetty.server.Server
+import org.eclipse.jetty.server.handler.HandlerCollection
 import org.eclipse.jetty.servlet.ServletContextHandler
+import org.eclipse.jetty.webapp.WebAppContext
 import java.io.RandomAccessFile
 import java.lang.management.ManagementFactory
 import java.nio.channels.FileLock
@@ -58,11 +60,28 @@ class Node(dir: Path, val p2pAddr: HostAndPort, configuration: NodeConfiguration
     private fun initWebServer(): Server {
         val port = p2pAddr.port + 1   // TODO: Move this into the node config file.
         val server = Server(port)
-        val handler = ServletContextHandler()
+
-        handler.setAttribute("node", this)
+        val handlerCollection = HandlerCollection()
-        handler.addServlet(DataUploadServlet::class.java, "/upload/*")
+
-        handler.addServlet(AttachmentDownloadServlet::class.java, "/attachments/*")
+        // Export JMX monitoring statistics and data over REST/JSON.
-        server.handler = handler
+        if (configuration.exportJMXto.split(',').contains("http")) {
+            handlerCollection.addHandler(WebAppContext().apply {
+                // Find the jolokia WAR file on the classpath.
+                contextPath = "/monitoring/json"
+                val classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator"))
+                war = classpath.first { it.contains("jolokia-agent-war-2") && it.endsWith(".war") }
+            })
+        }
+
+        // Data upload and download to services (attachments, rates oracles etc)
+        handlerCollection.addHandler(ServletContextHandler().apply {
+            contextPath = "/"
+            setAttribute("storage", storage)
+            addServlet(DataUploadServlet::class.java, "/upload/*")
+            addServlet(AttachmentDownloadServlet::class.java, "/attachments/*")
+        })
+
+        server.handler = handlerCollection
         server.start()
         return server
     }

src/main/kotlin/core/node/NodeConfiguration.kt

@@ -9,9 +9,26 @@
 package core.node
 
 import java.util.*
+import kotlin.reflect.declaredMemberProperties
 
 interface NodeConfiguration {
     val myLegalName: String
+    val exportJMXto: String
+}
+
+object DefaultConfiguration : NodeConfiguration {
+    override val myLegalName: String = "Vast Global MegaCorp"
+    override val exportJMXto: String = ""   // can be "http" or empty
+
+    fun toProperties(): Properties {
+        val settings = DefaultConfiguration::class.declaredMemberProperties.map { it.name to it.get(this@DefaultConfiguration).toString() }
+        val p = Properties().apply {
+            for (setting in settings) {
+                setProperty(setting.first, setting.second)
+            }
+        }
+        return p
+    }
 }
 
 /**
@@ -22,5 +39,6 @@ interface NodeConfiguration {
  * editing the file is a must-have.
  */
 class NodeConfigurationFromProperties(private val properties: Properties) : NodeConfiguration {
-    override val myLegalName: String by properties
+    override val myLegalName: String get() = properties.getProperty("myLegalName")
-}
+    override val exportJMXto: String get() = properties.getProperty("exportJMXto")
+}

src/main/kotlin/demos/RateFixDemo.kt

@@ -70,6 +70,7 @@ fun main(args: Array<String>) {
     val myNetAddr = ArtemisMessagingService.toHostAndPort(options.valueOf(networkAddressArg))
     val config = object : NodeConfiguration {
         override val myLegalName: String = "Rate fix demo node"
+        override val exportJMXto: String = "http"
     }
     val node = logElapsedTime("Node startup") { Node(dir, myNetAddr, config, null).start() }
 

src/main/kotlin/demos/TraderDemo.kt

@@ -17,6 +17,7 @@ import core.crypto.SecureHash
 import core.crypto.generateKeyPair
 import core.messaging.LegallyIdentifiableNode
 import core.messaging.SingleMessageRecipient
+import core.node.DefaultConfiguration
 import core.node.Node
 import core.node.NodeConfiguration
 import core.node.NodeConfigurationFromProperties
@@ -299,12 +300,12 @@ private fun loadConfigFile(configFile: Path): NodeConfiguration {
         askAdminToEditConfig(configFile)
     }
 
-    val configProps = configFile.toFile().reader().use {
+    val config = configFile.toFile().reader().use {
-        Properties().apply { load(it) }
+        NodeConfigurationFromProperties(
+            Properties(DefaultConfiguration.toProperties()).apply { load(it) }
+        )
     }
 
-    val config = NodeConfigurationFromProperties(configProps)
-
     // Make sure admin did actually edit at least the legal name.
     if (config.myLegalName == defaultLegalName)
         askAdminToEditConfig(configFile)

src/test/kotlin/core/node/MockNode.kt

@@ -95,6 +95,7 @@ class MockNetwork(private val threadPerNode: Boolean = false) {
             Files.createDirectories(path.resolve("attachments"))
         val config = object : NodeConfiguration {
             override val myLegalName: String = "Mock Company $id"
+            override val exportJMXto: String = ""
         }
         val fac = factory ?: { p, n, n2, l -> MockNode(p, n, n2, l, id) }
         val node = fac(path, config, this, withTimestamper).start()