From f628f3009801a63b97f442b0653b586198a98fd8 Mon Sep 17 00:00:00 2001 From: Rick Parker Date: Tue, 23 Jan 2018 14:00:36 +0000 Subject: [PATCH] ENT-1382 JMeter changes to allow easy HTML report generation (#382) * Fixes to allow the HTML reports to be run on JMeter CSV output files, with custom label. Also adjust each test plan and examples to include the label. (cherry picked from commit 35fd309) * Added some documentation to the README. --- tools/jmeter/README.md | 10 +- .../com/r3/corda/jmeter/BaseFlowSampler.kt | 11 +- .../main/resources/Example Flow Request.jmx | 5 + .../main/resources/IssueAndPay Request.jmx | 10 ++ .../resources/LocalBatchNotarise Request.jmx | 36 ++++- .../resources/LocalIssueAndPay Request.jmx | 9 +- .../resources/Testplans/CashIssuance_40k.jmx | 5 + .../resources/bin/reportgenerator.properties | 153 ++++++++++++++++++ 8 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 tools/jmeter/src/main/resources/bin/reportgenerator.properties diff --git a/tools/jmeter/README.md b/tools/jmeter/README.md index 86ef7a95a7..ed4bdc314e 100644 --- a/tools/jmeter/README.md +++ b/tools/jmeter/README.md @@ -85,4 +85,12 @@ The interesting bit here are the `jmeterArgs`: - `-t ` loads the testplan to run - `-l ` specifies the output to write to (if it exists, it will be appended) - `-R ` specifies the host to run against - note this is localhost in this case as we are using ssh -tunnels to reach the test nodes. \ No newline at end of file +tunnels to reach the test nodes. + +### Generating an HTML report from a recorded CSV file + +It's possible to generate a simple but effective HTML report for a JMeter test run that already produced a CSV file. + +```./gradlew tools:jmeter:run -PjmeterArgs="['-g', '.csv', '-o', '', '-Jjmeter.reportgenerator.report_title=']"``` + +The report output directory must be empty or not exist (in which case JMeter attempts to create it). \ No newline at end of file diff --git a/tools/jmeter/src/main/kotlin/com/r3/corda/jmeter/BaseFlowSampler.kt b/tools/jmeter/src/main/kotlin/com/r3/corda/jmeter/BaseFlowSampler.kt index 04f8b46e97..6b801f9987 100644 --- a/tools/jmeter/src/main/kotlin/com/r3/corda/jmeter/BaseFlowSampler.kt +++ b/tools/jmeter/src/main/kotlin/com/r3/corda/jmeter/BaseFlowSampler.kt @@ -16,17 +16,19 @@ import org.apache.jmeter.samplers.SampleResult */ abstract class BaseFlowSampler() : AbstractJavaSamplerClient() { companion object { + val label = Argument("label", "\${__samplerName}", "", "The value in the label column in the resulting CSV file to dissambiguate this test run from others.") val host = Argument("host", "localhost", "", "The remote network address (hostname or IP address) to connect to for RPC.") val port = Argument("port", "10000", "", "The remote port to connect to for RPC.") val username = Argument("username", "corda", "", "The RPC user to connect to connect as.") val password = Argument("password", "corda_is_awesome", "", "The password for the RPC user.") - val allArgs = setOf(host, port, username, password) + val allArgs = setOf(label, host, port, username, password) } var rpcClient: CordaRPCClient? = null var rpcConnection: CordaRPCConnection? = null var rpcProxy: CordaRPCOps? = null + var labelValue: String? = null override fun getDefaultParameters(): Arguments { // Add copies of all args, since they seem to be mutable. @@ -45,6 +47,10 @@ abstract class BaseFlowSampler() : AbstractJavaSamplerClient() { rpcClient = CordaRPCClient(NetworkHostAndPort(context.getParameter(host.name), context.getIntParameter(port.name))) rpcConnection = rpcClient!!.start(context.getParameter(username.name), context.getParameter(password.name)) rpcProxy = rpcConnection!!.proxy + labelValue = context.getParameter(label.name) + if (labelValue.isNullOrBlank()) { + labelValue = null + } setupTest(rpcProxy!!, context) } @@ -57,7 +63,7 @@ abstract class BaseFlowSampler() : AbstractJavaSamplerClient() { val result = SampleResult() result.sampleStart() val handle = rpcProxy!!.startFlowDynamic(flowInvoke.flowLogicClass, *(flowInvoke.args)) - result.sampleLabel = handle.id.toString() + result.sampleLabel = labelValue ?: flowInvoke.flowLogicClass.simpleName result.latencyEnd() try { val flowResult = handle.returnValue.get() @@ -82,6 +88,7 @@ abstract class BaseFlowSampler() : AbstractJavaSamplerClient() { rpcConnection!!.close() rpcConnection = null rpcClient = null + labelValue = null super.teardownTest(context) } diff --git a/tools/jmeter/src/main/resources/Example Flow Request.jmx b/tools/jmeter/src/main/resources/Example Flow Request.jmx index d244b74e8e..31c0f26355 100644 --- a/tools/jmeter/src/main/resources/Example Flow Request.jmx +++ b/tools/jmeter/src/main/resources/Example Flow Request.jmx @@ -29,6 +29,11 @@ + + label + ${___samplerName} + = + host localhost diff --git a/tools/jmeter/src/main/resources/IssueAndPay Request.jmx b/tools/jmeter/src/main/resources/IssueAndPay Request.jmx index 66b60c5cfa..3984b7d506 100644 --- a/tools/jmeter/src/main/resources/IssueAndPay Request.jmx +++ b/tools/jmeter/src/main/resources/IssueAndPay Request.jmx @@ -29,6 +29,11 @@ + + label + ${___samplerName} + = + host localhost @@ -59,6 +64,11 @@ O=Perf-10.155.0.7, OU=Corda, L=London, C=GB = + + useCoinSelection + true + = + com.r3.corda.jmeter.CashIssueAndPaySampler diff --git a/tools/jmeter/src/main/resources/LocalBatchNotarise Request.jmx b/tools/jmeter/src/main/resources/LocalBatchNotarise Request.jmx index ba7995e318..6e02d29e53 100644 --- a/tools/jmeter/src/main/resources/LocalBatchNotarise Request.jmx +++ b/tools/jmeter/src/main/resources/LocalBatchNotarise Request.jmx @@ -63,9 +63,14 @@ - + + + label + ${___samplerName} + = + host localhost @@ -106,9 +111,9 @@ true = - - transactionsPerMinute - 10 + + transactionsPerSecond + 1.0 = @@ -124,6 +129,11 @@ + + label + + = + host localhost @@ -180,9 +190,14 @@ - + + + label + ${___samplerName} + = + host localhost @@ -223,9 +238,9 @@ true = - - transactionsPerMinute - 10 + + transactionsPerSecond + 1.0 = @@ -241,6 +256,11 @@ + + label + + = + host localhost diff --git a/tools/jmeter/src/main/resources/LocalIssueAndPay Request.jmx b/tools/jmeter/src/main/resources/LocalIssueAndPay Request.jmx index d36d667bca..ba4c291564 100644 --- a/tools/jmeter/src/main/resources/LocalIssueAndPay Request.jmx +++ b/tools/jmeter/src/main/resources/LocalIssueAndPay Request.jmx @@ -26,9 +26,14 @@ - + + + label + ${___samplerName} + = + host localhost @@ -61,7 +66,7 @@ useCoinSelection - true + false = diff --git a/tools/jmeter/src/main/resources/Testplans/CashIssuance_40k.jmx b/tools/jmeter/src/main/resources/Testplans/CashIssuance_40k.jmx index e19436d028..31b7ba9fbe 100644 --- a/tools/jmeter/src/main/resources/Testplans/CashIssuance_40k.jmx +++ b/tools/jmeter/src/main/resources/Testplans/CashIssuance_40k.jmx @@ -29,6 +29,11 @@ + + label + ${___samplerName} + = + host localhost diff --git a/tools/jmeter/src/main/resources/bin/reportgenerator.properties b/tools/jmeter/src/main/resources/bin/reportgenerator.properties new file mode 100644 index 0000000000..ed3b3b7e3f --- /dev/null +++ b/tools/jmeter/src/main/resources/bin/reportgenerator.properties @@ -0,0 +1,153 @@ +################################################################################ +# Apache JMeter Property file for Report Generator +################################################################################ +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +################################################################################ +# +# THIS FILE SHOULD NOT BE MODIFIED +# +# This avoids having to re-apply the modifications when upgrading JMeter +# Instead only user.properties should be modified: +# 1/ copy the property you want to modify to user.properties from here +# 2/ Change its value there +# +################################################################################ +#--------------------------------------------------------------------------- +# Reporting configuration +#--------------------------------------------------------------------------- +# Sets the satisfaction threshold for the APDEX calculation (in milliseconds). +#jmeter.reportgenerator.apdex_satisfied_threshold=500 +# Sets the tolerance threshold for the APDEX calculation (in milliseconds). +#jmeter.reportgenerator.apdex_tolerated_threshold=1500 +# Sets satisfaction and tolerance threshold to specific samples. +# Use sample names or regular expression. +# Format is : sample_name:satisfaction|tolerance[;] +# Notice the colon between sample name and values, the pipe between thresholds and the +# semicolon at the end to separate different samples. Don't forget to escape after +# semicolon to span multiple lines. Ex : +#jmeter.reportgenerator.apdex_per_transaction=sample(\\d+):1000|2000,\ +# samples12:3000|4000;\ +# scenar01-12:5000|6000 +# This property is used by menu item "Export transactions for report" +# It is used to select which transactions by default will be exported +#jmeter.reportgenerator.exported_transactions_pattern=[a-zA-Z0-9_\\-{}\\$\\.]*[-_][0-9]* +# Regular Expression which Indicates which samples to keep for graphs and statistics generation. +# Empty value means no filtering +#jmeter.reportgenerator.sample_filter= +# Sets the temporary directory used by the generation process if it needs file I/O operations. +#jmeter.reportgenerator.temp_dir=temp +# Sets the size of the sliding window used by percentile evaluation. +# Caution : higher value provides a better accuracy but needs more memory. +#jmeter.reportgenerator.statistic_window = 20000 +# Configure this property to change the report title +#jmeter.reportgenerator.report_title=Apache JMeter Dashboard +# Default format +#jmeter.reportgenerator.date_format=yyyyMMddHHmmss +# Used to generate a report based on a date range +# If jmeter.save.saveservice.timestamp_format does not contain year +# then use 1970 as year +# Date range start date as per format declared in jmeter.reportgenerator.date_format +#jmeter.reportgenerator.start_date= +# Date range end date as per format declared in jmeter.reportgenerator.date_format +#jmeter.reportgenerator.end_date= +# Defines the overall granularity for over time graphs +# Granularity must be higher than 1000 (1second) otherwise Throughput graphs will be incorrect +# see Bug 60149 +jmeter.reportgenerator.overall_granularity=60000 +# Exclude transaction controller from analysis +# true by default +jmeter.reportgenerator.exclude_tc_from_top5_errors_by_sampler=true +# Response Time Percentiles graph definition +jmeter.reportgenerator.graph.responseTimePercentiles.classname=org.apache.jmeter.report.processor.graph.impl.ResponseTimePercentilesGraphConsumer +jmeter.reportgenerator.graph.responseTimePercentiles.title=Response Time Percentiles +# Response Time Distribution graph definition +jmeter.reportgenerator.graph.responseTimeDistribution.classname=org.apache.jmeter.report.processor.graph.impl.ResponseTimeDistributionGraphConsumer +jmeter.reportgenerator.graph.responseTimeDistribution.title=Response Time Distribution +jmeter.reportgenerator.graph.responseTimeDistribution.property.set_granularity=500 +# Active Threads Over Time graph definition +jmeter.reportgenerator.graph.activeThreadsOverTime.classname=org.apache.jmeter.report.processor.graph.impl.ActiveThreadsGraphConsumer +jmeter.reportgenerator.graph.activeThreadsOverTime.title=Active Threads Over Time +jmeter.reportgenerator.graph.activeThreadsOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Time VS Threads graph definition +#jmeter.reportgenerator.graph.timeVsThreads.classname=org.apache.jmeter.report.processor.graph.impl.TimeVSThreadGraphConsumer +#jmeter.reportgenerator.graph.timeVsThreads.title=Time VS Threads +# Bytes Throughput Over Time graph definition +#jmeter.reportgenerator.graph.bytesThroughputOverTime.classname=org.apache.jmeter.report.processor.graph.impl.BytesThroughputGraphConsumer +#jmeter.reportgenerator.graph.bytesThroughputOverTime.title=Bytes Throughput Over Time +#jmeter.reportgenerator.graph.bytesThroughputOverTime.exclude_controllers=true +#jmeter.reportgenerator.graph.bytesThroughputOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Response Time Over Time graph definition +jmeter.reportgenerator.graph.responseTimesOverTime.classname=org.apache.jmeter.report.processor.graph.impl.ResponseTimeOverTimeGraphConsumer +jmeter.reportgenerator.graph.responseTimesOverTime.title=Response Time Over Time +jmeter.reportgenerator.graph.responseTimesOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Percentiles Response Times over time +jmeter.reportgenerator.graph.responseTimePercentilesOverTime.classname=org.apache.jmeter.report.processor.graph.impl.ResponseTimePercentilesOverTimeGraphConsumer +jmeter.reportgenerator.graph.responseTimePercentilesOverTime.title Response Time Percentiles Over Time (successful requests only) +jmeter.reportgenerator.graph.responseTimePercentilesOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Synthetic Response Time Distribution +jmeter.reportgenerator.graph.syntheticResponseTimeDistribution.classname=org.apache.jmeter.report.processor.graph.impl.SyntheticResponseTimeDistributionGraphConsumer +jmeter.reportgenerator.graph.syntheticResponseTimeDistribution.title=Synthetic Response Times Distribution +jmeter.reportgenerator.graph.syntheticResponseTimeDistribution.exclude_controllers=true +jmeter.reportgenerator.graph.syntheticResponseTimeDistribution.property.set_satisfied_threshold=${jmeter.reportgenerator.apdex_satisfied_threshold} +jmeter.reportgenerator.graph.syntheticResponseTimeDistribution.property.set_tolerated_threshold=${jmeter.reportgenerator.apdex_tolerated_threshold} +# Latencies Over Time graph definition +jmeter.reportgenerator.graph.latenciesOverTime.classname=org.apache.jmeter.report.processor.graph.impl.LatencyOverTimeGraphConsumer +jmeter.reportgenerator.graph.latenciesOverTime.title=Latencies Over Time +jmeter.reportgenerator.graph.latenciesOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Connect Time Over Time graph definition +#jmeter.reportgenerator.graph.connectTimeOverTime.classname=org.apache.jmeter.report.processor.graph.impl.ConnectTimeOverTimeGraphConsumer +#jmeter.reportgenerator.graph.connectTimeOverTime.title=Connect Time Over Time +#jmeter.reportgenerator.graph.connectTimeOverTime.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Response Time Vs Request graph definition +jmeter.reportgenerator.graph.responseTimeVsRequest.classname=org.apache.jmeter.report.processor.graph.impl.ResponseTimeVSRequestGraphConsumer +jmeter.reportgenerator.graph.responseTimeVsRequest.title=Response Time Vs Request +jmeter.reportgenerator.graph.responseTimeVsRequest.exclude_controllers=true +jmeter.reportgenerator.graph.responseTimeVsRequest.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Latencies Vs Request graph definition +jmeter.reportgenerator.graph.latencyVsRequest.classname=org.apache.jmeter.report.processor.graph.impl.LatencyVSRequestGraphConsumer +jmeter.reportgenerator.graph.latencyVsRequest.title=Latencies Vs Request +jmeter.reportgenerator.graph.latencyVsRequest.exclude_controllers=true +jmeter.reportgenerator.graph.latencyVsRequest.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Hits Per Second graph definition +#jmeter.reportgenerator.graph.hitsPerSecond.classname=org.apache.jmeter.report.processor.graph.impl.HitsPerSecondGraphConsumer +#jmeter.reportgenerator.graph.hitsPerSecond.title=Hits Per Second +#jmeter.reportgenerator.graph.hitsPerSecond.exclude_controllers=true +#jmeter.reportgenerator.graph.hitsPerSecond.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Codes Per Second graph definition +#jmeter.reportgenerator.graph.codesPerSecond.classname=org.apache.jmeter.report.processor.graph.impl.CodesPerSecondGraphConsumer +#jmeter.reportgenerator.graph.codesPerSecond.title=Codes Per Second +#jmeter.reportgenerator.graph.codesPerSecond.exclude_controllers=true +#jmeter.reportgenerator.graph.codesPerSecond.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# Transactions Per Second graph definition +jmeter.reportgenerator.graph.transactionsPerSecond.classname=org.apache.jmeter.report.processor.graph.impl.TransactionsPerSecondGraphConsumer +jmeter.reportgenerator.graph.transactionsPerSecond.title=Transactions Per Second +jmeter.reportgenerator.graph.transactionsPerSecond.property.set_granularity=${jmeter.reportgenerator.overall_granularity} +# HTML Export +jmeter.reportgenerator.exporter.html.classname=org.apache.jmeter.report.dashboard.HtmlTemplateExporter +# Sets the source directory of templated files from which the html pages are generated. +#jmeter.reportgenerator.exporter.html.property.template_dir=report-template +# Sets the destination directory for generated html pages. +# This will be overridden by the command line option -o +#jmeter.reportgenerator.exporter.html.property.output_dir=report-output +# Regular Expression which Indicates which graph series are filtered in display +# Empty value means no filtering +#jmeter.reportgenerator.exporter.html.series_filter= +# Indicates whether series filter apply only on sample series or to all series +# setting this to false can lead to empty graphs if series_filter does not +# contain required series +#jmeter.reportgenerator.exporter.html.filters_only_sample_series=true +# Indicates whether only controller samples are displayed on graphs that support it. +#jmeter.reportgenerator.exporter.html.show_controllers_only=false