Merge from master

This commit is contained in:
Jose Coll
2016-11-02 14:20:51 +00:00
49 changed files with 626 additions and 1128 deletions

View File

@ -122,7 +122,7 @@ dependencies {
testCompile 'com.pholser:junit-quickcheck-core:0.6'
// For H2 database support in persistence
compile "com.h2database:h2:1.3.176"
compile "com.h2database:h2:1.4.192"
// Exposed: Kotlin SQL library - under evaluation
compile "org.jetbrains.exposed:exposed:0.5.0"

View File

@ -24,6 +24,10 @@ class JDBCHashMapTestSuite {
lateinit var dataSource: Closeable
lateinit var transaction: Transaction
lateinit var database: Database
lateinit var loadOnInitFalseMap: JDBCHashMap<String, String>
lateinit var loadOnInitTrueMap: JDBCHashMap<String, String>
lateinit var loadOnInitFalseSet: JDBCHashSet<String>
lateinit var loadOnInitTrueSet: JDBCHashSet<String>
@JvmStatic
@BeforeClass
@ -32,6 +36,10 @@ class JDBCHashMapTestSuite {
dataSource = dataSourceAndDatabase.first
database = dataSourceAndDatabase.second
setUpDatabaseTx()
loadOnInitFalseMap = JDBCHashMap<String, String>("test_map_false", loadOnInit = false)
loadOnInitTrueMap = JDBCHashMap<String, String>("test_map_true", loadOnInit = true)
loadOnInitFalseSet = JDBCHashSet<String>("test_set_false", loadOnInit = false)
loadOnInitTrueSet = JDBCHashSet<String>("test_set_true", loadOnInit = true)
}
@JvmStatic
@ -112,7 +120,8 @@ class JDBCHashMapTestSuite {
*/
class JDBCHashMapTestGenerator(val loadOnInit: Boolean) : com.google.common.collect.testing.TestStringMapGenerator() {
override fun create(elements: Array<Map.Entry<String, String>>): Map<String, String> {
val map = JDBCHashMap<String, String>("test_map_${System.nanoTime()}", loadOnInit = loadOnInit)
val map = if (loadOnInit) loadOnInitTrueMap else loadOnInitFalseMap
map.clear()
map.putAll(elements.associate { Pair(it.key, it.value) })
return map
}
@ -143,7 +152,8 @@ class JDBCHashMapTestSuite {
*/
class JDBCHashSetTestGenerator(val loadOnInit: Boolean) : com.google.common.collect.testing.TestStringSetGenerator() {
override fun create(elements: Array<String>): Set<String> {
val set = JDBCHashSet<String>("test_set_${System.nanoTime()}", loadOnInit = loadOnInit)
val set = if (loadOnInit) loadOnInitTrueSet else loadOnInitFalseSet
set.clear()
set.addAll(elements)
return set
}

View File

@ -276,6 +276,7 @@ open class DriverDSL(
val conn = url.openConnection() as HttpURLConnection
conn.requestMethod = "GET"
if (conn.responseCode != 200) {
log.error("Received response code ${conn.responseCode} from $url during startup.")
return null
}
// For now the NodeInfo is tunneled in its Kryo format over the Node's Web interface.
@ -285,6 +286,7 @@ open class DriverDSL(
om.registerModule(module)
return om.readValue(conn.inputStream, NodeInfo::class.java)
} catch(e: Exception) {
log.error("Could not query node info at $url due to an exception.", e)
return null
}
}

View File

@ -307,7 +307,7 @@ abstract class AbstractNode(open val configuration: NodeConfiguration, val netwo
val pluginServices = pluginRegistries.flatMap { x -> x.servicePlugins }
val serviceList = mutableListOf<Any>()
for (serviceClass in pluginServices) {
val service = serviceClass.getConstructor(ServiceHubInternal::class.java).newInstance(services)
val service = serviceClass.getConstructor(PluginServiceHub::class.java).newInstance(services)
serviceList.add(service)
tokenizableServices.add(service)
if (service is AcceptsFileUpload) {

View File

@ -33,6 +33,7 @@ import org.glassfish.jersey.servlet.ServletContainer
import org.jetbrains.exposed.sql.Database
import java.io.RandomAccessFile
import java.lang.management.ManagementFactory
import java.lang.reflect.InvocationTargetException
import java.nio.channels.FileLock
import java.time.Clock
import java.util.*
@ -143,13 +144,18 @@ class Node(override val configuration: FullNodeConfiguration, networkMapAddress:
// Export JMX monitoring statistics and data over REST/JSON.
if (configuration.exportJMXto.split(',').contains("http")) {
handlerCollection.addHandler(WebAppContext().apply {
// Find the jolokia WAR file on the classpath.
contextPath = "/monitoring/json"
setInitParameter("mimeType", "application/json")
val classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator"))
war = classpath.first { it.contains("jolokia-agent-war-2") && it.endsWith(".war") }
})
val classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator"))
val warpath = classpath.firstOrNull() { it.contains("jolokia-agent-war-2") && it.endsWith(".war") }
if (warpath != null) {
handlerCollection.addHandler(WebAppContext().apply {
// Find the jolokia WAR file on the classpath.
contextPath = "/monitoring/json"
setInitParameter("mimeType", "application/json")
war = warpath
})
} else {
log.warn("Unable to locate Jolokia WAR on classpath")
}
}
// API, data upload and download to services (attachments, rates oracles etc)
@ -157,7 +163,7 @@ class Node(override val configuration: FullNodeConfiguration, networkMapAddress:
val server = Server()
if (configuration.useHTTPS) {
val connector = if (configuration.useHTTPS) {
val httpsConfiguration = HttpConfiguration()
httpsConfiguration.outputBufferSize = 32768
httpsConfiguration.addCustomizer(SecureRequestCustomizer())
@ -173,14 +179,16 @@ class Node(override val configuration: FullNodeConfiguration, networkMapAddress:
sslContextFactory.setIncludeCipherSuites(".*AES.*GCM.*")
val sslConnector = ServerConnector(server, SslConnectionFactory(sslContextFactory, "http/1.1"), HttpConnectionFactory(httpsConfiguration))
sslConnector.port = configuration.webAddress.port
server.connectors = arrayOf<Connector>(sslConnector)
sslConnector
} else {
val httpConfiguration = HttpConfiguration()
httpConfiguration.outputBufferSize = 32768
val httpConnector = ServerConnector(server, HttpConnectionFactory(httpConfiguration))
httpConnector.port = configuration.webAddress.port
server.connectors = arrayOf<Connector>(httpConnector)
httpConnector
}
server.connectors = arrayOf<Connector>(connector)
log.info("Starting web API server on port ${connector.port}")
server.handler = handlerCollection
runOnStop += Runnable { server.stop() }
@ -203,8 +211,19 @@ class Node(override val configuration: FullNodeConfiguration, networkMapAddress:
val webAPIsOnClasspath = pluginRegistries.flatMap { x -> x.webApis }
for (webapi in webAPIsOnClasspath) {
log.info("Add Plugin web API from attachment ${webapi.name}")
val customAPI = webapi.getConstructor(ServiceHub::class.java).newInstance(services)
log.info("Add plugin web API from attachment ${webapi.name}")
val constructor = try {
webapi.getConstructor(ServiceHub::class.java)
} catch (ex: NoSuchMethodException) {
log.error("Missing constructor ${webapi.name}(ServiceHub)")
continue
}
val customAPI = try {
constructor.newInstance(services)
} catch (ex: InvocationTargetException) {
log.error("Constructor ${webapi.name}(ServiceHub) threw an error: ", ex.targetException)
continue
}
resourceConfig.register(customAPI)
}
@ -270,7 +289,13 @@ class Node(override val configuration: FullNodeConfiguration, networkMapAddress:
super.start()
// Only start the service API requests once the network map registration is complete
networkMapRegistrationFuture.then {
webServer = initWebServer()
try {
webServer = initWebServer()
} catch(ex: Exception) {
// TODO: We need to decide if this is a fatal error, given the API is unavailable, or whether the API
// is not critical and we continue anyway.
log.error("Web server startup failed", ex)
}
// Begin exporting our own metrics via JMX.
JmxReporter.
forRegistry(services.monitoringService.metrics).

View File

@ -92,7 +92,8 @@ class ServerRPCOps(
val builder: TransactionBuilder = TransactionType.General.Builder(null)
// TODO: Have some way of restricting this to states the caller controls
try {
val (spendTX, keysForSigning) = services.vaultService.generateSpend(builder, req.amount.withoutIssuer(), req.recipient.owningKey)
val (spendTX, keysForSigning) = services.vaultService.generateSpend(builder,
req.amount.withoutIssuer(), req.recipient.owningKey, setOf(req.amount.token.issuer.party))
keysForSigning.forEach {
val key = services.keyManagementService.keys[it] ?: throw IllegalStateException("Could not find signing key for ${it.toStringShort()}")

View File

@ -1,6 +1,7 @@
package com.r3corda.node.services
import com.r3corda.core.node.CordaPluginRegistry
import com.r3corda.core.node.PluginServiceHub
import com.r3corda.core.serialization.SingletonSerializeAsToken
import com.r3corda.node.services.api.ServiceHubInternal
import com.r3corda.protocols.NotaryChangeProtocol
@ -14,7 +15,7 @@ object NotaryChange {
* A service that monitors the network for requests for changing the notary of a state,
* and immediately runs the [NotaryChangeProtocol] if the auto-accept criteria are met.
*/
class Service(services: ServiceHubInternal) : SingletonSerializeAsToken() {
class Service(services: PluginServiceHub) : SingletonSerializeAsToken() {
init {
services.registerProtocolInitiator(NotaryChangeProtocol.Instigator::class) { NotaryChangeProtocol.Acceptor(it) }
}

View File

@ -3,6 +3,7 @@ package com.r3corda.node.services.api
import com.google.common.util.concurrent.ListenableFuture
import com.r3corda.core.crypto.Party
import com.r3corda.core.messaging.MessagingService
import com.r3corda.core.node.PluginServiceHub
import com.r3corda.core.node.ServiceHub
import com.r3corda.core.node.services.TxWritableStorageService
import com.r3corda.core.protocols.ProtocolLogic
@ -37,7 +38,7 @@ interface MessagingServiceBuilder<out T : MessagingServiceInternal> {
private val log = LoggerFactory.getLogger(ServiceHubInternal::class.java)
abstract class ServiceHubInternal : ServiceHub {
abstract class ServiceHubInternal : PluginServiceHub {
abstract val monitoringService: MonitoringService
abstract val protocolLogicRefFactory: ProtocolLogicRefFactory
abstract val schemaService: SchemaService
@ -71,24 +72,6 @@ abstract class ServiceHubInternal : ServiceHub {
*/
abstract fun <T> startProtocol(logic: ProtocolLogic<T>): ListenableFuture<T>
/**
* Register the protocol factory we wish to use when a initiating party attempts to communicate with us. The
* registration is done against a marker [KClass] which is sent in the session handshake by the other party. If this
* marker class has been registered then the corresponding factory will be used to create the protocol which will
* communicate with the other side. If there is no mapping then the session attempt is rejected.
* @param markerClass The marker [KClass] present in a session initiation attempt, which is a 1:1 mapping to a [Class]
* using the <pre>::class</pre> construct. Conventionally this is a [ProtocolLogic] subclass, however any class can
* be used, with the default being the class of the initiating protocol. This enables the registration to be of the
* form: registerProtocolInitiator(InitiatorProtocol::class, ::InitiatedProtocol)
* @param protocolFactory The protocol factory generating the initiated protocol.
*/
abstract fun registerProtocolInitiator(markerClass: KClass<*>, protocolFactory: (Party) -> ProtocolLogic<*>)
/**
* Return the protocol factory that has been registered with [markerClass], or null if no factory is found.
*/
abstract fun getProtocolFactory(markerClass: Class<*>): ((Party) -> ProtocolLogic<*>)?
override fun <T : Any> invokeProtocolAsync(logicType: Class<out ProtocolLogic<T>>, vararg args: Any?): ListenableFuture<T> {
val logicRef = protocolLogicRefFactory.create(logicType, *args)
@Suppress("UNCHECKED_CAST")

View File

@ -6,6 +6,7 @@ import com.google.common.util.concurrent.SettableFuture
import com.r3corda.core.bufferUntilSubscribed
import com.r3corda.core.contracts.Contract
import com.r3corda.core.crypto.Party
import com.r3corda.core.crypto.toStringShort
import com.r3corda.core.map
import com.r3corda.core.messaging.MessagingService
import com.r3corda.core.messaging.SingleMessageRecipient
@ -72,9 +73,16 @@ open class InMemoryNetworkMapCache : SingletonSerializeAsToken(), NetworkMapCach
override fun get(serviceType: ServiceType) = registeredNodes.filterValues { it.advertisedServices.any { it.info.type.isSubTypeOf(serviceType) } }.map { it.value }
override fun getRecommended(type: ServiceType, contract: Contract, vararg party: Party): NodeInfo? = get(type).firstOrNull()
override fun getNodeByLegalName(name: String) = get().singleOrNull { it.legalIdentity.name == name }
override fun getNodeByPublicKey(publicKey: PublicKey) = get().singleOrNull {
(it.legalIdentity.owningKey == publicKey)
|| it.advertisedServices.any { it.identity.owningKey == publicKey }
override fun getNodeByPublicKey(publicKey: PublicKey): NodeInfo? {
// Although we should never have more than one match, it is theoretically possible. Report an error if it happens.
val candidates = get().filter {
(it.legalIdentity.owningKey == publicKey)
|| it.advertisedServices.any { it.identity.owningKey == publicKey }
}
if (candidates.size > 1) {
throw IllegalStateException("Found more than one match for key ${publicKey.toStringShort()}")
}
return candidates.singleOrNull()
}
override fun addMapService(net: MessagingService, networkMapAddress: SingleMessageRecipient, subscribe: Boolean,

View File

@ -3,6 +3,7 @@ package com.r3corda.node.services.persistence
import co.paralleluniverse.fibers.Suspendable
import com.r3corda.core.crypto.Party
import com.r3corda.core.node.CordaPluginRegistry
import com.r3corda.core.node.PluginServiceHub
import com.r3corda.core.node.recordTransactions
import com.r3corda.core.protocols.ProtocolLogic
import com.r3corda.core.serialization.SingletonSerializeAsToken
@ -31,10 +32,7 @@ object DataVending {
* Additionally, because nodes do not store invalid transactions, requesting such a transaction will always yield null.
*/
@ThreadSafe
// TODO: I don't like that this needs ServiceHubInternal, but passing in a state machine breaks MockServices because
// the state machine isn't set when this is constructed. [NodeSchedulerService] has the same problem, and both
// should be fixed at the same time.
class Service(services: ServiceHubInternal) : SingletonSerializeAsToken() {
class Service(services: PluginServiceHub) : SingletonSerializeAsToken() {
companion object {
val logger = loggerFor<DataVending.Service>()

View File

@ -183,7 +183,7 @@ class NodeVaultService(private val services: ServiceHub) : SingletonSerializeAsT
var acceptableCoins = run {
val ofCurrency = assetsStates.filter { it.state.data.amount.token.product == currency }
if (onlyFromParties != null)
ofCurrency.filter { it.state.data.deposit.party in onlyFromParties }
ofCurrency.filter { it.state.data.amount.token.issuer.party in onlyFromParties }
else
ofCurrency
}
@ -196,27 +196,32 @@ class NodeVaultService(private val services: ServiceHub) : SingletonSerializeAsT
val (gathered, gatheredAmount) = gatherCoins(acceptableCoins, amount)
val takeChangeFrom = gathered.firstOrNull()
val change = if (takeChangeFrom != null && gatheredAmount > amount) {
Amount(gatheredAmount.quantity - amount.quantity, takeChangeFrom.state.data.issuanceDef)
Amount(gatheredAmount.quantity - amount.quantity, takeChangeFrom.state.data.amount.token)
} else {
null
}
val keysUsed = gathered.map { it.state.data.owner }.toSet()
val states = gathered.groupBy { it.state.data.deposit }.map {
val states = gathered.groupBy { it.state.data.amount.token.issuer }.map {
val coins = it.value
val totalAmount = coins.map { it.state.data.amount }.sumOrThrow()
deriveState(coins.first().state, totalAmount, to)
}
}.sortedBy { it.data.amount.quantity }
val outputs = if (change != null) {
// Just copy a key across as the change key. In real life of course, this works but leaks private data.
// In bitcoinj we derive a fresh key here and then shuffle the outputs to ensure it's hard to follow
// value flows through the transaction graph.
val changeKey = gathered.first().state.data.owner
val existingOwner = gathered.first().state.data.owner
// Add a change output and adjust the last output downwards.
states.subList(0, states.lastIndex) +
states.last().let { deriveState(it, it.data.amount - change, it.data.owner) } +
deriveState(gathered.last().state, change, changeKey)
states.last().let {
val spent = it.data.amount.withoutIssuer() - change.withoutIssuer()
deriveState(it, Amount(spent.quantity, it.data.amount.token), it.data.owner)
} +
states.last().let {
deriveState(it, Amount(change.quantity, it.data.amount.token), existingOwner)
}
} else states
for (state in gathered) tx.addInputState(state)

View File

@ -6,7 +6,7 @@ keyStorePassword = "cordacadevpass"
trustStorePassword = "trustpass"
dataSourceProperties = {
dataSourceClassName = org.h2.jdbcx.JdbcDataSource
"dataSource.url" = "jdbc:h2:file:"${basedir}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;MVCC=true;MV_STORE=true;WRITE_DELAY=0;AUTO_SERVER_PORT="${h2port}
"dataSource.url" = "jdbc:h2:file:"${basedir}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=0;AUTO_SERVER_PORT="${h2port}
"dataSource.user" = sa
"dataSource.password" = ""
}

View File

@ -1,8 +1,14 @@
package com.r3corda.node.services
import com.r3corda.core.crypto.generateKeyPair
import com.r3corda.core.node.services.ServiceInfo
import com.r3corda.node.services.network.NetworkMapService
import com.r3corda.node.services.transactions.SimpleNotaryService
import com.r3corda.testing.expect
import com.r3corda.testing.node.MockNetwork
import org.junit.Before
import org.junit.Test
import kotlin.test.assertEquals
class InMemoryNetworkMapCacheTest {
lateinit var network: MockNetwork
@ -20,4 +26,20 @@ class InMemoryNetworkMapCacheTest {
network.runNetwork()
future.get()
}
@Test
fun `key collision`() {
val keyPair = generateKeyPair()
val nodeA = network.createNode(null, -1, MockNetwork.DefaultFactory, true, "Node A", keyPair, ServiceInfo(NetworkMapService.type))
val nodeB = network.createNode(null, -1, MockNetwork.DefaultFactory, true, "Node B", keyPair, ServiceInfo(NetworkMapService.type))
// Node A currently knows only about itself, so this returns node A
assertEquals(nodeA.netMapCache.getNodeByPublicKey(keyPair.public), nodeA.info)
nodeA.netMapCache.addNode(nodeB.info)
// Now both nodes match, so it throws an error
expect<IllegalStateException> {
nodeA.netMapCache.getNodeByPublicKey(keyPair.public)
}
}
}

View File

@ -1,35 +1,32 @@
package com.r3corda.node.services
import com.google.common.jimfs.Configuration
import com.google.common.jimfs.Jimfs
import com.r3corda.core.contracts.*
import com.r3corda.core.days
import com.r3corda.core.node.ServiceHub
import com.r3corda.core.node.recordTransactions
import com.r3corda.core.node.services.VaultService
import com.r3corda.core.protocols.ProtocolLogic
import com.r3corda.core.protocols.ProtocolLogicRef
import com.r3corda.core.protocols.ProtocolLogicRefFactory
import com.r3corda.core.serialization.SingletonSerializeAsToken
import com.r3corda.core.transactions.SignedTransaction
import com.r3corda.core.utilities.DUMMY_NOTARY
import com.r3corda.node.services.events.NodeSchedulerService
import com.r3corda.node.services.persistence.DBCheckpointStorage
import com.r3corda.node.services.statemachine.StateMachineManager
import com.r3corda.node.services.vault.NodeVaultService
import com.r3corda.node.utilities.AddOrRemove
import com.r3corda.node.utilities.AffinityExecutor
import com.r3corda.node.utilities.configureDatabase
import com.r3corda.node.utilities.databaseTransaction
import com.r3corda.testing.ALICE_KEY
import com.r3corda.testing.node.*
import com.r3corda.testing.node.InMemoryMessagingNetwork
import com.r3corda.testing.node.MockKeyManagementService
import com.r3corda.testing.node.TestClock
import com.r3corda.testing.node.makeTestDataSourceProperties
import org.assertj.core.api.Assertions.assertThat
import org.jetbrains.exposed.sql.Database
import org.junit.After
import org.junit.Before
import org.junit.Test
import java.io.Closeable
import java.nio.file.FileSystem
import java.security.PublicKey
import java.time.Clock
import java.time.Instant
@ -39,8 +36,6 @@ import java.util.concurrent.TimeUnit
import kotlin.test.assertTrue
class NodeSchedulerServiceTest : SingletonSerializeAsToken() {
// Use an in memory file system for testing attachment storage.
val fs: FileSystem = Jimfs.newFileSystem(Configuration.unix())
val realClock: Clock = Clock.systemUTC()
val stoppedClock = Clock.fixed(realClock.instant(), realClock.zone)
@ -82,19 +77,7 @@ class NodeSchedulerServiceTest : SingletonSerializeAsToken() {
dataSource = dataSourceAndDatabase.first
database = dataSourceAndDatabase.second
// Switched from InMemoryVault usage to NodeVault
databaseTransaction(database) {
val services1 = object : MockServices() {
override val vaultService: VaultService = NodeVaultService(this)
override fun recordTransactions(txs: Iterable<SignedTransaction>) {
for (stx in txs) {
storageService.validatedTransactions.addTransaction(stx)
vaultService.notify(stx.tx)
}
}
}
val kms = MockKeyManagementService(ALICE_KEY)
val mockMessagingService = InMemoryMessagingNetwork(false).InMemoryMessaging(false, InMemoryMessagingNetwork.Handle(0, "None"), AffinityExecutor.ServiceAffinityExecutor("test", 1), database)
services = object : MockServiceHubInternal(overrideClock = testClock, keyManagement = kms, net = mockMessagingService), TestReference {

View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="utf-8"?>
<restrict>
<http>
<method>post</method>
<method>get</method>
</http>
<commands>
<command>read</command>
<command>list</command>
</commands>
<!-- allow anyone to force a garbage collection -->
<allow>
<mbean>
<name>java.lang:type=Memory</name>
<operation>gc</operation>
</mbean>
</allow>
<!-- in case we ever end up using c3pio connection pooling, this example from the docs prevents the password being exported -->
<deny>
<mbean>
<name>com.mchange.v2.c3p0:type=PooledDataSource,*</name>
<attribute>properties</attribute>
</mbean>
</deny>
</restrict>