RPC muxing, multithreading, RPC driver, performance tests

This commit is contained in:
Andras Slemmer
2017-03-29 17:28:02 +01:00
parent 25dbac0f07
commit de88ad4f40
63 changed files with 3223 additions and 1417 deletions

View File

@ -0,0 +1,53 @@
package net.corda.testing
import kotlin.reflect.KCallable
import kotlin.reflect.jvm.reflect
/**
* These functions may be used to run measurements of a function where the parameters are chosen from corresponding
* [Iterable]s in a lexical manner. An example use case would be benchmarking the speed of a certain function call using
* different combinations of parameters.
*/
@Suppress("UNCHECKED_CAST")
fun <A, R> measure(a: Iterable<A>, f: (A) -> R) =
measure(listOf(a), f.reflect()!!) { (f as ((Any?)->R))(it[0]) }
@Suppress("UNCHECKED_CAST")
fun <A, B, R> measure(a: Iterable<A>, b: Iterable<B>, f: (A, B) -> R) =
measure(listOf(a, b), f.reflect()!!) { (f as ((Any?,Any?)->R))(it[0], it[1]) }
@Suppress("UNCHECKED_CAST")
fun <A, B, C, R> measure(a: Iterable<A>, b: Iterable<B>, c: Iterable<C>, f: (A, B, C) -> R) =
measure(listOf(a, b, c), f.reflect()!!) { (f as ((Any?,Any?,Any?)->R))(it[0], it[1], it[2]) }
@Suppress("UNCHECKED_CAST")
fun <A, B, C, D, R> measure(a: Iterable<A>, b: Iterable<B>, c: Iterable<C>, d: Iterable<D>, f: (A, B, C, D) -> R) =
measure(listOf(a, b, c, d), f.reflect()!!) { (f as ((Any?,Any?,Any?,Any?)->R))(it[0], it[1], it[2], it[3]) }
private fun <R> measure(paramIterables: List<Iterable<Any?>>, kCallable: KCallable<R>, call: (Array<Any?>) -> R): Iterable<MeasureResult<R>> {
val kParameters = kCallable.parameters
return iterateLexical(paramIterables).map { params ->
MeasureResult(
parameters = params.mapIndexed { index, param -> Pair(kParameters[index].name!!, param) },
result = call(params.toTypedArray())
)
}
}
data class MeasureResult<out R>(
val parameters: List<Pair<String, Any?>>,
val result: R
)
fun <A> iterateLexical(iterables: List<Iterable<A>>): Iterable<List<A>> {
val result = ArrayList<List<A>>()
fun iterateLexicalHelper(index: Int, list: List<A>) {
if (index < iterables.size) {
iterables[index].forEach {
iterateLexicalHelper(index + 1, list + it)
}
} else {
result.add(list)
}
}
iterateLexicalHelper(0, emptyList())
return result
}

View File

@ -0,0 +1,469 @@
package net.corda.testing
import com.google.common.net.HostAndPort
import com.google.common.util.concurrent.ListenableFuture
import net.corda.client.mock.Generator
import net.corda.client.mock.generateOrFail
import net.corda.client.mock.int
import net.corda.client.mock.string
import net.corda.client.rpc.internal.RPCClient
import net.corda.client.rpc.internal.RPCClientConfiguration
import net.corda.core.div
import net.corda.core.messaging.RPCOps
import net.corda.core.utilities.ProcessUtilities
import net.corda.node.driver.*
import net.corda.node.services.RPCUserService
import net.corda.node.services.messaging.ArtemisMessagingServer
import net.corda.node.services.messaging.RPCServer
import net.corda.node.services.messaging.RPCServerConfiguration
import net.corda.nodeapi.ArtemisTcpTransport
import net.corda.nodeapi.ConnectionDirection
import net.corda.nodeapi.RPCApi
import net.corda.nodeapi.User
import org.apache.activemq.artemis.api.core.SimpleString
import org.apache.activemq.artemis.api.core.TransportConfiguration
import org.apache.activemq.artemis.api.core.client.ActiveMQClient
import org.apache.activemq.artemis.api.core.client.ActiveMQClient.DEFAULT_ACK_BATCH_SIZE
import org.apache.activemq.artemis.api.core.client.ClientSession
import org.apache.activemq.artemis.api.core.management.ActiveMQServerControl
import org.apache.activemq.artemis.core.config.Configuration
import org.apache.activemq.artemis.core.config.CoreQueueConfiguration
import org.apache.activemq.artemis.core.config.impl.ConfigurationImpl
import org.apache.activemq.artemis.core.remoting.impl.invm.InVMAcceptorFactory
import org.apache.activemq.artemis.core.remoting.impl.invm.InVMConnectorFactory
import org.apache.activemq.artemis.core.remoting.impl.netty.NettyAcceptorFactory
import org.apache.activemq.artemis.core.security.CheckType
import org.apache.activemq.artemis.core.security.Role
import org.apache.activemq.artemis.core.server.embedded.EmbeddedActiveMQ
import org.apache.activemq.artemis.core.server.impl.ActiveMQServerImpl
import org.apache.activemq.artemis.core.settings.impl.AddressFullMessagePolicy
import org.apache.activemq.artemis.core.settings.impl.AddressSettings
import org.apache.activemq.artemis.spi.core.protocol.RemotingConnection
import org.apache.activemq.artemis.spi.core.security.ActiveMQSecurityManager3
import org.bouncycastle.asn1.x500.X500Name
import java.lang.reflect.Method
import java.nio.file.Path
import java.nio.file.Paths
import java.util.*
import javax.security.cert.X509Certificate
interface RPCDriverExposedDSLInterface : DriverDSLExposedInterface {
/**
* Starts an In-VM RPC server. Note that only a single one may be started.
*
* @param rpcUser The single user who can access the server through RPC, and their permissions.
* @param nodeLegalName The legal name of the node to check against to authenticate a super user.
* @param configuration The RPC server configuration.
* @param ops The server-side implementation of the RPC interface.
*/
fun <I : RPCOps> startInVmRpcServer(
rpcUser: User = rpcTestUser,
nodeLegalName: X500Name = fakeNodeLegalName,
maxFileSize: Int = ArtemisMessagingServer.MAX_FILE_SIZE,
maxBufferedBytesPerClient: Long = 10L * ArtemisMessagingServer.MAX_FILE_SIZE,
configuration: RPCServerConfiguration = RPCServerConfiguration.default,
ops : I
): ListenableFuture<Unit>
/**
* Starts an In-VM RPC client.
*
* @param rpcOpsClass The [Class] of the RPC interface.
* @param username The username to authenticate with.
* @param password The password to authenticate with.
* @param configuration The RPC client configuration.
*/
fun <I : RPCOps> startInVmRpcClient(
rpcOpsClass: Class<I>,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password,
configuration: RPCClientConfiguration = RPCClientConfiguration.default
): ListenableFuture<I>
/**
* Starts an In-VM Artemis session connecting to the RPC server.
*
* @param username The username to authenticate with.
* @param password The password to authenticate with.
*/
fun startInVmArtemisSession(
username: String = rpcTestUser.username,
password: String = rpcTestUser.password
): ClientSession
/**
* Starts a Netty RPC server.
*
* @param serverName The name of the server, to be used for the folder created for Artemis files.
* @param rpcUser The single user who can access the server through RPC, and their permissions.
* @param nodeLegalName The legal name of the node to check against to authenticate a super user.
* @param configuration The RPC server configuration.
* @param ops The server-side implementation of the RPC interface.
*/
fun <I : RPCOps> startRpcServer(
serverName: String = "driver-rpc-server",
rpcUser: User = rpcTestUser,
nodeLegalName: X500Name = fakeNodeLegalName,
maxFileSize: Int = ArtemisMessagingServer.MAX_FILE_SIZE,
maxBufferedBytesPerClient: Long = 10L * ArtemisMessagingServer.MAX_FILE_SIZE,
configuration: RPCServerConfiguration = RPCServerConfiguration.default,
ops : I
) : ListenableFuture<RpcServerHandle>
/**
* Starts a Netty RPC client.
*
* @param rpcOpsClass The [Class] of the RPC interface.
* @param rpcAddress The address of the RPC server to connect to.
* @param username The username to authenticate with.
* @param password The password to authenticate with.
* @param configuration The RPC client configuration.
*/
fun <I : RPCOps> startRpcClient(
rpcOpsClass: Class<I>,
rpcAddress: HostAndPort,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password,
configuration: RPCClientConfiguration = RPCClientConfiguration.default
): ListenableFuture<I>
/**
* Starts a Netty RPC client in a new JVM process that calls random RPCs with random arguments.
*
* @param rpcOpsClass The [Class] of the RPC interface.
* @param rpcAddress The address of the RPC server to connect to.
* @param username The username to authenticate with.
* @param password The password to authenticate with.
*/
fun <I : RPCOps> startRandomRpcClient(
rpcOpsClass: Class<I>,
rpcAddress: HostAndPort,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password
): ListenableFuture<Process>
/**
* Starts a Netty Artemis session connecting to an RPC server.
*
* @param rpcAddress The address of the RPC server.
* @param username The username to authenticate with.
* @param password The password to authenticate with.
*/
fun startArtemisSession(
rpcAddress: HostAndPort,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password
): ClientSession
}
inline fun <reified I : RPCOps> RPCDriverExposedDSLInterface.startInVmRpcClient(
username: String = rpcTestUser.username,
password: String = rpcTestUser.password,
configuration: RPCClientConfiguration = RPCClientConfiguration.default
) = startInVmRpcClient(I::class.java, username, password, configuration)
inline fun <reified I : RPCOps> RPCDriverExposedDSLInterface.startRandomRpcClient(
hostAndPort: HostAndPort,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password
) = startRandomRpcClient(I::class.java, hostAndPort, username, password)
inline fun <reified I : RPCOps> RPCDriverExposedDSLInterface.startRpcClient(
rpcAddress: HostAndPort,
username: String = rpcTestUser.username,
password: String = rpcTestUser.password,
configuration: RPCClientConfiguration = RPCClientConfiguration.default
) = startRpcClient(I::class.java, rpcAddress, username, password, configuration)
interface RPCDriverInternalDSLInterface : DriverDSLInternalInterface, RPCDriverExposedDSLInterface
data class RpcServerHandle(
val hostAndPort: HostAndPort,
val serverControl: ActiveMQServerControl
)
val rpcTestUser = User("user1", "test", permissions = emptySet())
val fakeNodeLegalName = X500Name("not:a:valid:name")
fun <A> rpcDriver(
isDebug: Boolean = false,
driverDirectory: Path = Paths.get("build", getTimestampAsDirectoryName()),
portAllocation: PortAllocation = PortAllocation.Incremental(10000),
debugPortAllocation: PortAllocation = PortAllocation.Incremental(5005),
systemProperties: Map<String, String> = emptyMap(),
useTestClock: Boolean = false,
automaticallyStartNetworkMap: Boolean = false,
dsl: RPCDriverExposedDSLInterface.() -> A
) = genericDriver(
driverDsl = RPCDriverDSL(
DriverDSL(
portAllocation = portAllocation,
debugPortAllocation = debugPortAllocation,
systemProperties = systemProperties,
driverDirectory = driverDirectory.toAbsolutePath(),
useTestClock = useTestClock,
automaticallyStartNetworkMap = automaticallyStartNetworkMap,
isDebug = isDebug
)
),
coerce = { it },
dsl = dsl
)
private class SingleUserSecurityManager(val rpcUser: User) : ActiveMQSecurityManager3 {
override fun validateUser(user: String?, password: String?) = isValid(user, password)
override fun validateUserAndRole(user: String?, password: String?, roles: MutableSet<Role>?, checkType: CheckType?) = isValid(user, password)
override fun validateUser(user: String?, password: String?, certificates: Array<out X509Certificate>?): String? {
return validate(user, password)
}
override fun validateUserAndRole(user: String?, password: String?, roles: MutableSet<Role>?, checkType: CheckType?, address: String?, connection: RemotingConnection?): String? {
return validate(user, password)
}
private fun isValid(user: String?, password: String?): Boolean {
return rpcUser.username == user && rpcUser.password == password
}
private fun validate(user: String?, password: String?): String? {
return if (isValid(user, password)) user else null
}
}
data class RPCDriverDSL(
val driverDSL: DriverDSL
) : DriverDSLInternalInterface by driverDSL, RPCDriverInternalDSLInterface {
private companion object {
val notificationAddress = "notifications"
private fun ConfigurationImpl.configureCommonSettings(maxFileSize: Int, maxBufferedBytesPerClient: Long) {
managementNotificationAddress = SimpleString(notificationAddress)
isPopulateValidatedUser = true
journalBufferSize_NIO = maxFileSize
journalBufferSize_AIO = maxFileSize
journalFileSize = maxFileSize
queueConfigurations = listOf(
CoreQueueConfiguration().apply {
name = RPCApi.RPC_SERVER_QUEUE_NAME
address = RPCApi.RPC_SERVER_QUEUE_NAME
isDurable = false
},
CoreQueueConfiguration().apply {
name = RPCApi.RPC_CLIENT_BINDING_REMOVALS
address = notificationAddress
filterString = RPCApi.RPC_CLIENT_BINDING_REMOVAL_FILTER_EXPRESSION
isDurable = false
}
)
addressesSettings = mapOf(
"${RPCApi.RPC_CLIENT_QUEUE_NAME_PREFIX}.#" to AddressSettings().apply {
maxSizeBytes = maxBufferedBytesPerClient
addressFullMessagePolicy = AddressFullMessagePolicy.FAIL
}
)
}
fun createInVmRpcServerArtemisConfig(maxFileSize: Int, maxBufferedBytesPerClient: Long): Configuration {
return ConfigurationImpl().apply {
acceptorConfigurations = setOf(TransportConfiguration(InVMAcceptorFactory::class.java.name))
isPersistenceEnabled = false
configureCommonSettings(maxFileSize, maxBufferedBytesPerClient)
}
}
fun createRpcServerArtemisConfig(maxFileSize: Int, maxBufferedBytesPerClient: Long, baseDirectory: Path, hostAndPort: HostAndPort): Configuration {
val connectionDirection = ConnectionDirection.Inbound(acceptorFactoryClassName = NettyAcceptorFactory::class.java.name)
return ConfigurationImpl().apply {
val artemisDir = "$baseDirectory/artemis"
bindingsDirectory = "$artemisDir/bindings"
journalDirectory = "$artemisDir/journal"
largeMessagesDirectory = "$artemisDir/large-messages"
acceptorConfigurations = setOf(ArtemisTcpTransport.tcpTransport(connectionDirection, hostAndPort, null))
configureCommonSettings(maxFileSize, maxBufferedBytesPerClient)
}
}
val inVmClientTransportConfiguration = TransportConfiguration(InVMConnectorFactory::class.java.name)
fun createNettyClientTransportConfiguration(hostAndPort: HostAndPort): TransportConfiguration {
return ArtemisTcpTransport.tcpTransport(ConnectionDirection.Outbound(), hostAndPort, null)
}
}
override fun <I : RPCOps> startInVmRpcServer(
rpcUser: User,
nodeLegalName: X500Name,
maxFileSize: Int,
maxBufferedBytesPerClient: Long,
configuration: RPCServerConfiguration,
ops: I
): ListenableFuture<Unit> {
return driverDSL.executorService.submit<Unit> {
val artemisConfig = createInVmRpcServerArtemisConfig(maxFileSize, maxBufferedBytesPerClient)
val server = EmbeddedActiveMQ()
server.setConfiguration(artemisConfig)
server.setSecurityManager(SingleUserSecurityManager(rpcUser))
server.start()
driverDSL.shutdownManager.registerShutdown {
server.activeMQServer.stop()
server.stop()
}
startRpcServerWithBrokerRunning(
rpcUser, nodeLegalName, configuration, ops, inVmClientTransportConfiguration,
server.activeMQServer.activeMQServerControl
)
}
}
override fun <I : RPCOps> startInVmRpcClient(rpcOpsClass: Class<I>, username: String, password: String, configuration: RPCClientConfiguration): ListenableFuture<I> {
return driverDSL.executorService.submit<I> {
val client = RPCClient<I>(inVmClientTransportConfiguration, configuration)
val connection = client.start(rpcOpsClass, username, password)
driverDSL.shutdownManager.registerShutdown {
connection.close()
}
connection.proxy
}
}
override fun startInVmArtemisSession(username: String, password: String): ClientSession {
val locator = ActiveMQClient.createServerLocatorWithoutHA(inVmClientTransportConfiguration)
val sessionFactory = locator.createSessionFactory()
val session = sessionFactory.createSession(username, password, false, true, true, locator.isPreAcknowledge, DEFAULT_ACK_BATCH_SIZE)
driverDSL.shutdownManager.registerShutdown {
session.close()
sessionFactory.close()
locator.close()
}
return session
}
override fun <I : RPCOps> startRpcServer(
serverName: String,
rpcUser: User,
nodeLegalName: X500Name,
maxFileSize: Int,
maxBufferedBytesPerClient: Long,
configuration: RPCServerConfiguration,
ops: I
): ListenableFuture<RpcServerHandle> {
val hostAndPort = driverDSL.portAllocation.nextHostAndPort()
return driverDSL.executorService.submit<RpcServerHandle> {
val artemisConfig = createRpcServerArtemisConfig(maxFileSize, maxBufferedBytesPerClient, driverDSL.driverDirectory / serverName, hostAndPort)
val server = ActiveMQServerImpl(artemisConfig, SingleUserSecurityManager(rpcUser))
server.start()
driverDSL.shutdownManager.registerShutdown {
server.stop()
addressMustNotBeBound(driverDSL.executorService, hostAndPort).get()
}
val transportConfiguration = createNettyClientTransportConfiguration(hostAndPort)
startRpcServerWithBrokerRunning(
rpcUser, nodeLegalName, configuration, ops, transportConfiguration,
server.activeMQServerControl
)
RpcServerHandle(hostAndPort, server.activeMQServerControl)
}
}
override fun <I : RPCOps> startRpcClient(
rpcOpsClass: Class<I>,
rpcAddress: HostAndPort,
username: String,
password: String,
configuration: RPCClientConfiguration
): ListenableFuture<I> {
return driverDSL.executorService.submit<I> {
val client = RPCClient<I>(ArtemisTcpTransport.tcpTransport(ConnectionDirection.Outbound(), rpcAddress, null), configuration)
val connection = client.start(rpcOpsClass, username, password)
driverDSL.shutdownManager.registerShutdown {
connection.close()
}
connection.proxy
}
}
override fun <I : RPCOps> startRandomRpcClient(rpcOpsClass: Class<I>, rpcAddress: HostAndPort, username: String, password: String): ListenableFuture<Process> {
val processFuture = driverDSL.executorService.submit<Process> {
ProcessUtilities.startJavaProcess<RandomRpcUser>(listOf(rpcOpsClass.name, rpcAddress.toString(), username, password))
}
driverDSL.shutdownManager.registerProcessShutdown(processFuture)
return processFuture
}
override fun startArtemisSession(rpcAddress: HostAndPort, username: String, password: String): ClientSession {
val locator = ActiveMQClient.createServerLocatorWithoutHA(createNettyClientTransportConfiguration(rpcAddress))
val sessionFactory = locator.createSessionFactory()
val session = sessionFactory.createSession(username, password, false, true, true, false, DEFAULT_ACK_BATCH_SIZE)
driverDSL.shutdownManager.registerShutdown {
session.close()
sessionFactory.close()
locator.close()
}
return session
}
private fun <I : RPCOps> startRpcServerWithBrokerRunning(
rpcUser: User,
nodeLegalName: X500Name,
configuration: RPCServerConfiguration,
ops: I,
transportConfiguration: TransportConfiguration,
serverControl: ActiveMQServerControl
) {
val locator = ActiveMQClient.createServerLocatorWithoutHA(transportConfiguration).apply {
minLargeMessageSize = ArtemisMessagingServer.MAX_FILE_SIZE
}
val userService = object : RPCUserService {
override fun getUser(username: String): User? = if (username == rpcUser.username) rpcUser else null
override val users: List<User> get() = listOf(rpcUser)
}
val rpcServer = RPCServer(
ops,
rpcUser.username,
rpcUser.password,
locator,
userService,
nodeLegalName,
configuration
)
driverDSL.shutdownManager.registerShutdown {
rpcServer.close()
locator.close()
}
rpcServer.start(serverControl)
}
}
/**
* An out-of-process RPC user that connects to an RPC server and issues random RPCs with random arguments.
*/
class RandomRpcUser {
companion object {
private inline fun <reified T> HashMap<Class<*>, Generator<*>>.add(generator: Generator<T>) = this.putIfAbsent(T::class.java, generator)
val generatorStore = HashMap<Class<*>, Generator<*>>().apply {
add(Generator.string())
add(Generator.int())
}
data class Call(val method: Method, val call: () -> Any?)
@JvmStatic
fun main(args: Array<String>) {
require(args.size == 4)
@Suppress("UNCHECKED_CAST")
val rpcClass = Class.forName(args[0]) as Class<RPCOps>
val hostAndPort = HostAndPort.fromString(args[1])
val username = args[2]
val password = args[3]
val handle = RPCClient<RPCOps>(hostAndPort, null).start(rpcClass, username, password)
val callGenerators = rpcClass.declaredMethods.map { method ->
Generator.sequence(method.parameters.map {
generatorStore[it.type] ?: throw Exception("No generator for ${it.type}")
}).map { arguments ->
Call(method, { method.invoke(handle.proxy, *arguments.toTypedArray()) })
}
}
val callGenerator = Generator.choice(callGenerators)
val random = SplittableRandom()
while (true) {
val call = callGenerator.generateOrFail(random)
call.call()
Thread.sleep(100)
}
}
}
}

View File

@ -2,15 +2,13 @@ package net.corda.testing.node
import com.google.common.util.concurrent.Futures
import com.google.common.util.concurrent.ListenableFuture
import net.corda.core.createDirectories
import net.corda.core.*
import net.corda.core.crypto.X509Utilities
import net.corda.core.crypto.commonName
import net.corda.core.div
import net.corda.core.flatMap
import net.corda.core.map
import net.corda.core.node.services.ServiceInfo
import net.corda.core.node.services.ServiceType
import net.corda.core.utilities.DUMMY_MAP
import net.corda.node.driver.addressMustNotBeBound
import net.corda.node.internal.Node
import net.corda.node.services.config.ConfigHelper
import net.corda.node.services.config.FullNodeConfiguration
@ -26,6 +24,7 @@ import org.junit.After
import org.junit.Rule
import org.junit.rules.TemporaryFolder
import java.util.*
import java.util.concurrent.Executors
import kotlin.concurrent.thread
/**
@ -53,9 +52,18 @@ abstract class NodeBasedTest {
*/
@After
fun stopAllNodes() {
val shutdownExecutor = Executors.newScheduledThreadPool(1)
nodes.forEach(Node::stop)
// Wait until ports are released
val portNotBoundChecks = nodes.flatMap {
listOf(
it.configuration.p2pAddress.let { addressMustNotBeBound(shutdownExecutor, it) },
it.configuration.rpcAddress?.let { addressMustNotBeBound(shutdownExecutor, it) }
)
}.filterNotNull()
nodes.clear()
_networkMapNode = null
Futures.allAsList(portNotBoundChecks).getOrThrow()
}
/**

View File

@ -57,7 +57,7 @@ class SimpleNode(val config: NodeConfiguration, val address: HostAndPort = freeL
},
userService)
thread(name = config.myLegalName.commonName) {
net.run()
net.run(broker.serverControl)
}
}