Jmx Reporter Addition to allow for New Relic over Jolokia alternative (#3739)

* gradle.build - Added metrics-new-relic dependency
Node.kt - Refactored start(): NodeInfo function extracting the code that was creating the Jolokia JMX reporter configuration and placing it into its own registerJolokiaReporter private function, added a registerJmxReporter function that is now called from start(), the registerJxmReporter function checks the NodeConfiguration's JmxReporterType value for either JOLOKIA or NEW_RELIC to derive whether to execute the registerJolokiaReporter vs. registerNewRelic reporter.
NodeConfiguration - enhanced to encapsulate a JmxReporterType (JOLOKIA is the default config) configuration options for Jolokia or NewRelic reporters.
Enhanced NodeTest.kt, NodeConfigurationImpleTest.kt and added test-working-config-newrelic.conf to ensure that tests still work as expected.

* Added configuration details concerning JmxReporterType ...

* Updated files with style suggestions made by @tlil

* Updated markdown of the external url

* Changed grammer on "See `Introduction to New Relic for Java`_ for details on getting started and how to install the New Relic Java Agent."
to "See `Introduction to New Relic for Java`_ for details on how to get started and how to install the New Relic Java agent."

docs/source/corda-configuration-file.rst

@@ -245,6 +245,13 @@ absolute path to the node's base directory.
 
 :flowMonitorSuspensionLoggingThresholdMillis: Threshold ``Duration`` suspended flows waiting for IO need to exceed before they are logged. Default value is ``60 seconds``.
 
+:jmxReporterType:  Provides an option for registering an alternative JMX reporter. Available options are ``JOLOKIA`` and ``NEW_RELIC``. If no value is provided, ``JOLOKIA`` will be used.
+
+                    .. note:: The Jolokia configuration is provided by default.  The New Relic configuration leverages the Dropwizard_ NewRelicReporter solution. See `Introduction to New Relic for Java`_ for details on how to get started and how to install the New Relic Java agent.
+
+                        .. _Dropwizard: https://metrics.dropwizard.io/3.2.3/manual/third-party.html
+                        .. _Introduction to New Relic for Java: https://docs.newrelic.com/docs/agents/java-agent/getting-started/introduction-new-relic-java
+
 Examples
 --------
 

node/build.gradle

@@ -189,6 +189,8 @@ dependencies {
 
     // Jolokia JVM monitoring agent, required to push logs through slf4j
     compile "org.jolokia:jolokia-jvm:${jolokia_version}:agent"
+    // Optional New Relic JVM reporter, used to push metrics to the configured account associated with a newrelic.yml configuration. See https://mvnrepository.com/artifact/com.palominolabs.metrics/metrics-new-relic
+    compile group: 'com.palominolabs.metrics', name: 'metrics-new-relic', version: '1.1.1'
 }
 
 task integrationTest(type: Test) {

node/src/main/kotlin/net/corda/node/internal/Node.kt

@@ -1,6 +1,10 @@
 package net.corda.node.internal
 
 import com.codahale.metrics.JmxReporter
+import com.codahale.metrics.MetricFilter
+import com.codahale.metrics.MetricRegistry
+import com.palominolabs.metrics.newrelic.AllEnabledMetricAttributeFilter
+import com.palominolabs.metrics.newrelic.NewRelicReporter
 import net.corda.client.rpc.internal.serialization.amqp.AMQPClientSerializationScheme
 import net.corda.core.concurrent.CordaFuture
 import net.corda.core.flows.FlowLogic
@@ -42,6 +46,7 @@ import net.corda.node.services.config.NodeConfiguration
 import net.corda.node.services.config.SecurityConfiguration
 import net.corda.node.services.config.shouldInitCrashShell
 import net.corda.node.services.config.shouldStartLocalShell
+import net.corda.node.services.config.JmxReporterType
 import net.corda.node.services.messaging.*
 import net.corda.node.services.rpc.ArtemisRpcBroker
 import net.corda.node.utilities.AddressUtils
@@ -64,11 +69,12 @@ import rx.schedulers.Schedulers
 import java.net.BindException
 import java.net.InetAddress
 import java.nio.file.Path
+import java.nio.file.Paths
 import java.time.Clock
 import java.util.concurrent.atomic.AtomicInteger
 import javax.management.ObjectName
 import kotlin.system.exitProcess
-import java.nio.file.Paths
+import java.util.concurrent.TimeUnit
 
 class NodeWithInfo(val node: Node, val info: NodeInfo) {
     val services: StartedNodeServices = object : StartedNodeServices, ServiceHubInternal by node.services, FlowStarter by node.flowStarter {}
@@ -382,18 +388,8 @@ open class Node(configuration: NodeConfiguration,
         val nodeInfo: NodeInfo = super.start()
         nodeReadyFuture.thenMatch({
             serverThread.execute {
-                // Begin exporting our own metrics via JMX. These can be monitored using any agent, e.g. Jolokia:
+
-                //
+                registerJmxReporter(services.monitoringService.metrics)
-                // https://jolokia.org/agent/jvm.html
-                JmxReporter.forRegistry(services.monitoringService.metrics).inDomain("net.corda").createsObjectNamesWith { _, domain, name ->
-                    // Make the JMX hierarchy a bit better organised.
-                    val category = name.substringBefore('.')
-                    val subName = name.substringAfter('.', "")
-                    if (subName == "")
-                        ObjectName("$domain:name=$category")
-                    else
-                        ObjectName("$domain:type=$category,name=$subName")
-                }.build().start()
 
                 _startupComplete.set(Unit)
             }
@@ -406,6 +402,47 @@ open class Node(configuration: NodeConfiguration,
         return nodeInfo
     }
 
+    /**
+     * A hook to allow configuration override of the JmxReporter being used.
+     */
+    fun registerJmxReporter(metrics: MetricRegistry) {
+        log.info("Registering JMX reporter:")
+        when (configuration.jmxReporterType) {
+            JmxReporterType.JOLOKIA -> registerJolokiaReporter(metrics)
+            JmxReporterType.NEW_RELIC -> registerNewRelicReporter(metrics)
+        }
+    }
+
+    private fun registerJolokiaReporter(registry: MetricRegistry) {
+        log.info("Registering Jolokia JMX reporter:")
+        // Begin exporting our own metrics via JMX. These can be monitored using any agent, e.g. Jolokia:
+        //
+        // https://jolokia.org/agent/jvm.html
+        JmxReporter.forRegistry(registry).inDomain("net.corda").createsObjectNamesWith { _, domain, name ->
+            // Make the JMX hierarchy a bit better organised.
+            val category = name.substringBefore('.')
+            val subName = name.substringAfter('.', "")
+            if (subName == "")
+                ObjectName("$domain:name=$category")
+            else
+                ObjectName("$domain:type=$category,name=$subName")
+        }.build().start()
+    }
+
+    private fun registerNewRelicReporter (registry: MetricRegistry) {
+        log.info("Registering New Relic JMX Reporter:")
+        val reporter = NewRelicReporter.forRegistry(registry)
+                .name("New Relic Reporter")
+                .filter(MetricFilter.ALL)
+                .attributeFilter(AllEnabledMetricAttributeFilter())
+                .rateUnit(TimeUnit.SECONDS)
+                .durationUnit(TimeUnit.MILLISECONDS)
+                .metricNamePrefix("corda/")
+                .build()
+
+        reporter.start(1, TimeUnit.MINUTES)
+    }
+
     override val rxIoScheduler: Scheduler get() = Schedulers.io()
 
     private fun initialiseSerialization() {

node/src/main/kotlin/net/corda/node/services/config/NodeConfiguration.kt

@@ -70,6 +70,7 @@ interface NodeConfiguration : NodeSSLConfiguration {
     val flowMonitorPeriodMillis: Duration get() = DEFAULT_FLOW_MONITOR_PERIOD_MILLIS
     val flowMonitorSuspensionLoggingThresholdMillis: Duration get() = DEFAULT_FLOW_MONITOR_SUSPENSION_LOGGING_THRESHOLD_MILLIS
     val cordappDirectories: List<Path> get() = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT)
+    val jmxReporterType : JmxReporterType? get() = defaultJmxReporterType
 
     fun validate(): List<String>
 
@@ -86,9 +87,18 @@ interface NodeConfiguration : NodeSSLConfiguration {
         const val defaultAttachmentCacheBound = 1024L
 
         const val cordappDirectoriesKey = "cordappDirectories"
+
+        val defaultJmxReporterType = JmxReporterType.JOLOKIA
     }
 }
 
+/**
+ * Currently registered JMX Reporters
+ */
+enum class JmxReporterType {
+    JOLOKIA, NEW_RELIC
+}
+
 data class DevModeOptions(val disableCheckpointChecker: Boolean = false, val allowCompatibilityZone: Boolean = false)
 
 fun NodeConfiguration.shouldCheckCheckpoints(): Boolean {
@@ -208,7 +218,8 @@ data class NodeConfigurationImpl(
         private val jarDirs: List<String> = emptyList(),
         override val flowMonitorPeriodMillis: Duration = DEFAULT_FLOW_MONITOR_PERIOD_MILLIS,
         override val flowMonitorSuspensionLoggingThresholdMillis: Duration = DEFAULT_FLOW_MONITOR_SUSPENSION_LOGGING_THRESHOLD_MILLIS,
-        override val cordappDirectories: List<Path> = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT)
+        override val cordappDirectories: List<Path> = listOf(baseDirectory / CORDAPPS_DIR_NAME_DEFAULT),
+        override val jmxReporterType: JmxReporterType? = JmxReporterType.JOLOKIA
 ) : NodeConfiguration {
     companion object {
         private val logger = loggerFor<NodeConfigurationImpl>()

node/src/main/resources/reference.conf

@@ -25,3 +25,5 @@ flowTimeout {
     maxRestartCount = 6
     backoffBase = 1.8
 }
+jmxReporterType = JOLOKIA
+

node/src/test/kotlin/net/corda/node/internal/NodeTest.kt

@@ -169,6 +169,7 @@ class NodeTest {
                 rpcSettings = NodeRpcSettings(address = fakeAddress, adminAddress = null, ssl = null),
                 messagingServerAddress = null,
                 notary = null
+
         )
     }
 }

node/src/test/kotlin/net/corda/node/services/config/NodeConfigurationImplTest.kt

@@ -208,6 +208,29 @@ class NodeConfigurationImplTest {
         assertEquals(compatibilityZoneURL, configuration.networkServices!!.networkMapURL)
     }
 
+    @Test
+    fun `jmxReporterType is null and defaults to Jokolia`() {
+        var rawConfig = getConfig("working-config.conf", ConfigFactory.parseMap(mapOf("devMode" to true)))
+        val nodeConfig = rawConfig.parseAsNodeConfiguration()
+        assertTrue(JmxReporterType.JOLOKIA.toString() == nodeConfig.jmxReporterType.toString())
+    }
+
+    @Test
+    fun `jmxReporterType is not null and is set to New Relic`() {
+        var rawConfig = getConfig("working-config.conf", ConfigFactory.parseMap(mapOf("devMode" to true)))
+        rawConfig = rawConfig.withValue("jmxReporterType", ConfigValueFactory.fromAnyRef("NEW_RELIC"))
+        val nodeConfig = rawConfig.parseAsNodeConfiguration()
+        assertTrue(JmxReporterType.NEW_RELIC.toString() == nodeConfig.jmxReporterType.toString())
+    }
+
+    @Test
+    fun `jmxReporterType is not null and set to Jokolia`() {
+        var rawConfig = getConfig("working-config.conf", ConfigFactory.parseMap(mapOf("devMode" to true)))
+        rawConfig = rawConfig.withValue("jmxReporterType", ConfigValueFactory.fromAnyRef("JOLOKIA"))
+        val nodeConfig = rawConfig.parseAsNodeConfiguration()
+        assertTrue(JmxReporterType.JOLOKIA.toString() == nodeConfig.jmxReporterType.toString())
+    }
+
     private fun configDebugOptions(devMode: Boolean, devModeOptions: DevModeOptions?): NodeConfiguration {
         return testConfiguration.copy(devMode = devMode, devModeOptions = devModeOptions)
     }