diff --git a/build.gradle b/build.gradle index 9ddf96d7f9..8a823b44a9 100644 --- a/build.gradle +++ b/build.gradle @@ -94,7 +94,9 @@ dependencies { // Web stuff: for HTTP[S] servlets compile "org.eclipse.jetty:jetty-servlet:${jetty_version}" + compile "org.eclipse.jetty:jetty-webapp:${jetty_version}" compile "javax.servlet:javax.servlet-api:3.1.0" + compile "org.jolokia:jolokia-agent-war:2.0.0-M1" compile "commons-fileupload:commons-fileupload:1.3.1" // Unit testing helpers. diff --git a/docs/source/node-administration.rst b/docs/source/node-administration.rst index bdefa3e6e8..e9ea9264d8 100644 --- a/docs/source/node-administration.rst +++ b/docs/source/node-administration.rst @@ -4,6 +4,34 @@ Node administration When a node is running, it exposes an embedded web server that lets you monitor it, upload and download attachments, access a REST API and so on. +Monitoring your node +-------------------- + +Like most Java servers, the node exports various useful metrics and management operations via the industry-standard +`JMX infrastructure `_. JMX is a standard _in-process_ API +for registering so-called _MBeans_ ... objects whose properties and methods are intended for server management. It does +not require any particular network protocol for export. So this data can be exported from the node in various ways: +some monitoring systems provide a "Java Agent", which is essentially a JVM plugin that finds all the MBeans and sends +them out to a statistics collector over the network. For those systems, follow the instructions provided by the vendor. + +Sometimes though, you just want raw access to the data and operations itself. So nodes export them over HTTP on the +`/monitoring/json` HTTP endpoint, using a program called `Jolokia `_. Jolokia defines the JSON +and REST formats for accessing MBeans, and provides client libraries to work with that protocol as well. + +Here are a few ways to build dashboards and extract monitoring data for a node: + +* `JMX2Graphite `_ is a tool that can be pointed to /monitoring/json and will + scrape the statistics found there, then insert them into the Graphite monitoring tool on a regular basis. It runs + in Docker and can be started with a single command. +* `JMXTrans `_ is another tool for Graphite, this time, it's got its own agent + (JVM plugin) which reads a custom config file and exports only the named data. It's more configurable than + JMX2Graphite and doesn't require a separate process, as the JVM will write directly to Graphite. +* *Java Mission Control* is a desktop app that can connect to a target JVM that has the right command line flags set + (or always, if running locally). You can explore what data is available, create graphs of those metrics, and invoke + management operations like forcing a garbage collection. +* Cloud metrics services like New Relic also understand JMX, typically, by providing their own agent that uploads the + data to their service on a regular schedule. + Uploading and downloading attachments ------------------------------------- diff --git a/src/main/kotlin/core/node/Node.kt b/src/main/kotlin/core/node/Node.kt index c3b7322b0e..59df6cff02 100644 --- a/src/main/kotlin/core/node/Node.kt +++ b/src/main/kotlin/core/node/Node.kt @@ -16,7 +16,9 @@ import core.node.servlets.AttachmentDownloadServlet import core.node.servlets.DataUploadServlet import core.utilities.loggerFor import org.eclipse.jetty.server.Server +import org.eclipse.jetty.server.handler.HandlerCollection import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.webapp.WebAppContext import java.io.RandomAccessFile import java.lang.management.ManagementFactory import java.nio.channels.FileLock @@ -58,11 +60,28 @@ class Node(dir: Path, val p2pAddr: HostAndPort, configuration: NodeConfiguration private fun initWebServer(): Server { val port = p2pAddr.port + 1 // TODO: Move this into the node config file. val server = Server(port) - val handler = ServletContextHandler() - handler.setAttribute("node", this) - handler.addServlet(DataUploadServlet::class.java, "/upload/*") - handler.addServlet(AttachmentDownloadServlet::class.java, "/attachments/*") - server.handler = handler + + val handlerCollection = HandlerCollection() + + // Export JMX monitoring statistics and data over REST/JSON. + if (configuration.exportJMXto.split(',').contains("http")) { + handlerCollection.addHandler(WebAppContext().apply { + // Find the jolokia WAR file on the classpath. + contextPath = "/monitoring/json" + val classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator")) + war = classpath.first { it.contains("jolokia-agent-war-2") && it.endsWith(".war") } + }) + } + + // Data upload and download to services (attachments, rates oracles etc) + handlerCollection.addHandler(ServletContextHandler().apply { + contextPath = "/" + setAttribute("storage", storage) + addServlet(DataUploadServlet::class.java, "/upload/*") + addServlet(AttachmentDownloadServlet::class.java, "/attachments/*") + }) + + server.handler = handlerCollection server.start() return server } diff --git a/src/main/kotlin/core/node/NodeConfiguration.kt b/src/main/kotlin/core/node/NodeConfiguration.kt index 917a3d019d..d4233c17ef 100644 --- a/src/main/kotlin/core/node/NodeConfiguration.kt +++ b/src/main/kotlin/core/node/NodeConfiguration.kt @@ -9,9 +9,26 @@ package core.node import java.util.* +import kotlin.reflect.declaredMemberProperties interface NodeConfiguration { val myLegalName: String + val exportJMXto: String +} + +object DefaultConfiguration : NodeConfiguration { + override val myLegalName: String = "Vast Global MegaCorp" + override val exportJMXto: String = "" // can be "http" or empty + + fun toProperties(): Properties { + val settings = DefaultConfiguration::class.declaredMemberProperties.map { it.name to it.get(this@DefaultConfiguration).toString() } + val p = Properties().apply { + for (setting in settings) { + setProperty(setting.first, setting.second) + } + } + return p + } } /** @@ -22,5 +39,6 @@ interface NodeConfiguration { * editing the file is a must-have. */ class NodeConfigurationFromProperties(private val properties: Properties) : NodeConfiguration { - override val myLegalName: String by properties -} + override val myLegalName: String get() = properties.getProperty("myLegalName") + override val exportJMXto: String get() = properties.getProperty("exportJMXto") +} \ No newline at end of file diff --git a/src/main/kotlin/demos/RateFixDemo.kt b/src/main/kotlin/demos/RateFixDemo.kt index 8a5c3ccbae..07199d1750 100644 --- a/src/main/kotlin/demos/RateFixDemo.kt +++ b/src/main/kotlin/demos/RateFixDemo.kt @@ -70,6 +70,7 @@ fun main(args: Array) { val myNetAddr = ArtemisMessagingService.toHostAndPort(options.valueOf(networkAddressArg)) val config = object : NodeConfiguration { override val myLegalName: String = "Rate fix demo node" + override val exportJMXto: String = "http" } val node = logElapsedTime("Node startup") { Node(dir, myNetAddr, config, null).start() } diff --git a/src/main/kotlin/demos/TraderDemo.kt b/src/main/kotlin/demos/TraderDemo.kt index 8d7ddfa4ef..f16327d052 100644 --- a/src/main/kotlin/demos/TraderDemo.kt +++ b/src/main/kotlin/demos/TraderDemo.kt @@ -17,6 +17,7 @@ import core.crypto.SecureHash import core.crypto.generateKeyPair import core.messaging.LegallyIdentifiableNode import core.messaging.SingleMessageRecipient +import core.node.DefaultConfiguration import core.node.Node import core.node.NodeConfiguration import core.node.NodeConfigurationFromProperties @@ -299,12 +300,12 @@ private fun loadConfigFile(configFile: Path): NodeConfiguration { askAdminToEditConfig(configFile) } - val configProps = configFile.toFile().reader().use { - Properties().apply { load(it) } + val config = configFile.toFile().reader().use { + NodeConfigurationFromProperties( + Properties(DefaultConfiguration.toProperties()).apply { load(it) } + ) } - val config = NodeConfigurationFromProperties(configProps) - // Make sure admin did actually edit at least the legal name. if (config.myLegalName == defaultLegalName) askAdminToEditConfig(configFile) diff --git a/src/test/kotlin/core/node/MockNode.kt b/src/test/kotlin/core/node/MockNode.kt index ab6e7ffbfa..52b4a6943e 100644 --- a/src/test/kotlin/core/node/MockNode.kt +++ b/src/test/kotlin/core/node/MockNode.kt @@ -95,6 +95,7 @@ class MockNetwork(private val threadPerNode: Boolean = false) { Files.createDirectories(path.resolve("attachments")) val config = object : NodeConfiguration { override val myLegalName: String = "Mock Company $id" + override val exportJMXto: String = "" } val fac = factory ?: { p, n, n2, l -> MockNode(p, n, n2, l, id) } val node = fac(path, config, this, withTimestamper).start()