Merge remote-tracking branch 'open/master' into bogdan-merge-20-03-18

# Conflicts:
#	docs/source/changelog.rst

client/rpc/build.gradle

@@ -91,6 +91,7 @@ dependencies {
 
     testCompile project(':node-driver')
     testCompile project(':client:mock')
+    integrationTestCompile project(path: ':node-api', configuration: 'testArtifacts')
 
     // Smoke tests do NOT have any Node code on the classpath!
     smokeTestCompile project(':smoke-test-utils')

client/rpc/src/integration-test/kotlin/net/corda/client/rpc/RPCStabilityTests.kt

@@ -22,6 +22,7 @@ import net.corda.core.serialization.serialize
 import net.corda.core.utilities.*
 import net.corda.node.services.messaging.RPCServerConfiguration
 import net.corda.nodeapi.RPCApi
+import net.corda.nodeapi.eventually
 import net.corda.testing.core.SerializationEnvironmentRule
 import net.corda.testing.internal.testThreadFactory
 import net.corda.testing.node.internal.*
@@ -257,9 +258,8 @@ class RPCStabilityTests {
             assertEquals("pong", client.ping())
             serverFollower.shutdown()
             startRpcServer<ReconnectOps>(ops = ops, customPort = serverPort).getOrThrow()
-            Thread.sleep(1000) //wait for the server to come back up
+            val response = eventually<RPCException, String>(10.seconds) { client.ping() }
-            val pingFuture = pool.fork(client::ping)
+            assertEquals("pong", response)
-            assertEquals("pong", pingFuture.getOrThrow(10.seconds))
             clientFollower.shutdown() // Driver would do this after the new server, causing hang.
         }
     }

client/rpc/src/main/kotlin/net/corda/client/rpc/internal/RPCClientProxyHandler.kt

@@ -51,10 +51,9 @@ import java.lang.reflect.Method
 import java.time.Instant
 import java.util.*
 import java.util.concurrent.*
+import java.util.concurrent.atomic.AtomicBoolean
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.atomic.AtomicLong
-import java.util.concurrent.locks.ReentrantReadWriteLock
-import kotlin.concurrent.withLock
 import kotlin.reflect.jvm.javaMethod
 
 /**
@@ -183,9 +182,7 @@ class RPCClientProxyHandler(
     private val deduplicationChecker = DeduplicationChecker(rpcConfiguration.deduplicationCacheExpiry)
     private val deduplicationSequenceNumber = AtomicLong(0)
 
-    private val lock = ReentrantReadWriteLock()
+    private val sendingEnabled = AtomicBoolean(true)
-    @Volatile
-    private var sendingEnabled = true
 
     /**
      * Start the client. This creates the per-client queue, starts the consumer session and the reaper.
@@ -229,10 +226,8 @@ class RPCClientProxyHandler(
             throw RPCException("RPC Proxy is closed")
         }
 
-        lock.readLock().withLock {
+        if (!sendingEnabled.get())
-            if (!sendingEnabled)
             throw RPCException("RPC server is not available.")
-        }
 
         val replyId = InvocationId.newInstance()
         callSiteMap?.set(replyId, Throwable("<Call site of root RPC '${method.name}'>"))
@@ -421,11 +416,8 @@ class RPCClientProxyHandler(
     private fun failoverHandler(event: FailoverEventType) {
         when (event) {
             FailoverEventType.FAILURE_DETECTED -> {
-                lock.writeLock().withLock {
+                sendingEnabled.set(false)
-                    sendingEnabled = false
-                }
 
-                log.warn("RPC server unavailable. RPC calls are being buffered.")
                 log.warn("Terminating observables.")
                 val m = observableContext.observableMap.asMap()
                 m.keys.forEach { k ->
@@ -444,9 +436,7 @@ class RPCClientProxyHandler(
             }
 
             FailoverEventType.FAILOVER_COMPLETED -> {
-                lock.writeLock().withLock {
+                sendingEnabled.set(true)
-                    sendingEnabled = true
-                }
                 log.info("RPC server available.")
             }
 

docs/source/changelog.rst

@@ -1,12 +1,12 @@
 Changelog
 =========
 
-Here are brief summaries of what's changed between each release. This includes some guidance on how to upgrade code
+Unreleased
-from previous releases. Please refer to :doc:`upgrade-notes` for detailed instructions.
-
-UNRELEASED
 ----------
 
+Here are brief summaries of what's changed between each snapshot release. This includes guidance on how to upgrade code
+from the previous milestone release.
+
 * Node can be shut down abruptly by ``shutdown`` function in `CordaRPCOps` or gracefully (draining flows first) through ``gracefulShutdown`` command from shell.
 
 * Parsing of ``NodeConfiguration`` will now fail if unknown configuration keys are found.
@@ -110,6 +110,10 @@ R3 Corda 3.0 Developer Preview
    * Single node notaries no longer have a second separate notary identity. Their main identity *is* their notary identity.
      Use ``NetworkMapCache.notaryIdentities`` to get the list of available notaries.
 
+  * Added ``NetworkMapCache.getNodesByLegalName`` for querying nodes belonging to a distributed service such as a notary cluster
+    where they all share a common identity. ``NetworkMapCache.getNodeByLegalName`` has been tightened to throw if more than
+    one node with the legal name is found.
+
    * The common name in the node's X.500 legal name is no longer reserved and can be used as part of the node's name.
 
    * Moved ``NodeInfoSchema`` to internal package as the node info's database schema is not part of the public API. This

docs/source/generating-a-node.rst

@@ -152,6 +152,24 @@ You can extend ``deployNodes`` to generate additional nodes.
 
 .. warning:: When adding nodes, make sure that there are no port clashes!
 
+To extend node configuration beyond the properties defined in the ``deployNodes`` task use the ``configFile`` property with the path (relative or absolute) set to an additional configuration file.
+This file should follow the standard :doc:`corda-configuration-file` format, as per node.conf. The properties from this file will be appended to the generated node configuration. Note, if you add a property already created by the 'deployNodes' task, both properties will be present in the file.
+The path to the file can also be added while running the Gradle task via the ``-PconfigFile`` command line option. However, the same file will be applied to all nodes.
+Following the previous example ``PartyB`` node will have additional configuration options added from a file ``none-b.conf``:
+
+.. sourcecode:: groovy
+
+    task deployNodes(type: net.corda.plugins.Cordform, dependsOn: ['jar']) {
+        [...]
+        node {
+            name "O=PartyB,L=New York,C=US"
+            [...]
+            // Grants user1 the ability to start the MyFlow flow.
+            rpcUsers = [[ user: "user1", "password": "test", "permissions": ["StartFlow.net.corda.flows.MyFlow"]]]
+            configFile = "samples/trader-demo/src/main/resources/none-b.conf"
+        }
+    }
+
 Specifying a custom webserver
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 By default, any node listing a webport will use the default development webserver, which is not production-ready. You

docs/source/setting-up-a-corda-network.rst

@@ -61,9 +61,11 @@ be done with the network bootstrapper. This is a tool that scans all the node co
 generate the network parameters file which is copied to the nodes' directories. It also copies each node's node-info file
 to every other node so that they can all transact with each other.
 
-The bootstrapper tool can be downloaded from http://downloads.corda.net/network-bootstrapper-corda-X.Y.jar, where ``X`` is the major Corda version and ``Y`` is the minor Corda version.
+The bootstrapper tool can be downloaded from http://downloads.corda.net/network-bootstrapper-corda-X.Y.jar, where ``X``
+is the major Corda version and ``Y`` is the minor Corda version.
 
-To use it, create a directory containing a ``node.conf`` file for each node you want to create. Then run the following command:
+To use it, create a directory containing a node config file, ending in "_node.conf", for each node you want to create.
+Then run the following command:
 
 ``java -jar network-bootstrapper.jar <nodes-root-dir>``
 
@@ -72,9 +74,9 @@ For example running the command on a directory containing these files :
 .. sourcecode:: none
 
     .
-    ├── notary.conf             // The notary's node.conf file
+    ├── notary_node.conf             // The notary's node.conf file
-    ├── partya.conf             // Party A's node.conf file
+    ├── partya_node.conf             // Party A's node.conf file
-    └── partyb.conf             // Party B's node.conf file
+    └── partyb_node.conf             // Party B's node.conf file
 
 Would generate directories containing three nodes: notary, partya and partyb.
 

node-api/build.gradle

@@ -72,6 +72,19 @@ dependencies {
     testCompile project(':node-driver')
 }
 
+configurations {
+        testArtifacts.extendsFrom testRuntime
+}
+
+task testJar(type: Jar) {
+    classifier "tests"
+    from sourceSets.test.output
+}
+
+artifacts {
+    testArtifacts testJar
+}
+
 jar {
     baseName 'corda-node-api'
 }

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/AMQPSerializationScheme.kt

@@ -81,6 +81,7 @@ abstract class AbstractAMQPSerializationScheme(val cordappLoader: List<Cordapp>)
             register(net.corda.nodeapi.internal.serialization.amqp.custom.PeriodSerializer(this))
             register(net.corda.nodeapi.internal.serialization.amqp.custom.ClassSerializer(this))
             register(net.corda.nodeapi.internal.serialization.amqp.custom.X509CertificateSerializer)
+            register(net.corda.nodeapi.internal.serialization.amqp.custom.X509CRLSerializer)
             register(net.corda.nodeapi.internal.serialization.amqp.custom.CertPathSerializer(this))
             register(net.corda.nodeapi.internal.serialization.amqp.custom.StringBufferSerializer)
             register(net.corda.nodeapi.internal.serialization.amqp.custom.SimpleStringSerializer)

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/amqp/custom/X509CRLSerializer.kt

@@ -0,0 +1,27 @@
+package net.corda.nodeapi.internal.serialization.amqp.custom
+
+import net.corda.nodeapi.internal.crypto.X509CertificateFactory
+import net.corda.nodeapi.internal.serialization.amqp.*
+import org.apache.qpid.proton.codec.Data
+import java.lang.reflect.Type
+import java.security.cert.X509CRL
+
+object X509CRLSerializer : CustomSerializer.Implements<X509CRL>(X509CRL::class.java) {
+    override val schemaForDocumentation = Schema(listOf(RestrictedType(
+            type.toString(),
+            "",
+            listOf(type.toString()),
+            SerializerFactory.primitiveTypeName(ByteArray::class.java)!!,
+            descriptor,
+            emptyList()
+    )))
+
+    override fun writeDescribedObject(obj: X509CRL, data: Data, type: Type, output: SerializationOutput) {
+        output.writeObject(obj.encoded, data, clazz)
+    }
+
+    override fun readObject(obj: Any, schemas: SerializationSchemas, input: DeserializationInput): X509CRL {
+        val bytes = input.readObject(obj, schemas, ByteArray::class.java) as ByteArray
+        return X509CertificateFactory().delegate.generateCRL(bytes.inputStream()) as X509CRL
+    }
+}

node-api/src/test/kotlin/net/corda/nodeapi/internal/serialization/amqp/SerializationOutputTests.kt

@@ -18,15 +18,19 @@ import net.corda.client.rpc.RPCException
 import net.corda.core.CordaException
 import net.corda.core.CordaRuntimeException
 import net.corda.core.contracts.*
+import net.corda.core.crypto.Crypto
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.secureRandomBytes
 import net.corda.core.flows.FlowException
 import net.corda.core.identity.AbstractParty
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.internal.AbstractAttachment
+import net.corda.core.internal.x500Name
 import net.corda.core.serialization.*
 import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.utilities.OpaqueBytes
+import net.corda.nodeapi.internal.DEV_INTERMEDIATE_CA
+import net.corda.nodeapi.internal.crypto.ContentSignerBuilder
 import net.corda.nodeapi.internal.serialization.*
 import net.corda.nodeapi.internal.serialization.amqp.SerializerFactory.Companion.isPrimitive
 import net.corda.testing.contracts.DummyContract
@@ -39,6 +43,9 @@ import org.apache.qpid.proton.amqp.*
 import org.apache.qpid.proton.codec.DecoderImpl
 import org.apache.qpid.proton.codec.EncoderImpl
 import org.assertj.core.api.Assertions.*
+import org.bouncycastle.cert.X509v2CRLBuilder
+import org.bouncycastle.cert.jcajce.JcaX509CRLConverter
+import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.junit.Assert.*
 import org.junit.Ignore
 import org.junit.Rule
@@ -51,6 +58,7 @@ import java.io.IOException
 import java.io.NotSerializableException
 import java.math.BigDecimal
 import java.math.BigInteger
+import java.security.cert.X509CRL
 import java.time.*
 import java.time.temporal.ChronoUnit
 import java.util.*
@@ -61,6 +69,7 @@ import kotlin.test.assertTrue
 
 object AckWrapper {
     object Ack
+
     fun serialize() {
         val factory = testDefaultFactoryNoEvolution()
         SerializationOutput(factory).serialize(Ack)
@@ -69,6 +78,7 @@ object AckWrapper {
 
 object PrivateAckWrapper {
     private object Ack
+
     fun serialize() {
         val factory = testDefaultFactoryNoEvolution()
         SerializationOutput(factory).serialize(Ack)
@@ -640,6 +650,7 @@ class SerializationOutputTests(private val compression: CordaSerializationEncodi
     }
 
     private val FOO_PROGRAM_ID = "net.corda.nodeapi.internal.serialization.amqp.SerializationOutputTests.FooContract"
+
     class FooState : ContractState {
         override val participants: List<AbstractParty> = emptyList()
     }
@@ -1002,6 +1013,25 @@ class SerializationOutputTests(private val compression: CordaSerializationEncodi
         assertEquals(objCopy.a, objCopy.b)
     }
 
+    private fun emptyCrl(): X509CRL {
+        val builder = X509v2CRLBuilder(CordaX500Name.build(DEV_INTERMEDIATE_CA.certificate.issuerX500Principal).x500Name, Date())
+        val provider = BouncyCastleProvider()
+        val crlHolder = builder.build(ContentSignerBuilder.build(Crypto.RSA_SHA256, Crypto.generateKeyPair(Crypto.RSA_SHA256).private, provider))
+        return JcaX509CRLConverter().setProvider(provider).getCRL(crlHolder)
+    }
+
+    @Test
+    fun `test X509CRL custom serializer`() {
+        val factory = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())
+        factory.register(net.corda.nodeapi.internal.serialization.amqp.custom.X509CRLSerializer)
+
+        val factory2 = SerializerFactory(AllWhitelist, ClassLoader.getSystemClassLoader())
+        factory2.register(net.corda.nodeapi.internal.serialization.amqp.custom.X509CRLSerializer)
+
+        val obj = emptyCrl()
+        serdes(obj, factory, factory2)
+    }
+
     data class ByteArrays(val a: ByteArray, val b: ByteArray)
 
     @Test
@@ -1207,7 +1237,8 @@ class SerializationOutputTests(private val compression: CordaSerializationEncodi
     @Test
     fun throwable() {
         class TestException(message: String?, cause: Throwable?) : CordaException(message, cause)
-        val testExcp = TestException("hello", Throwable().apply { stackTrace = Thread.currentThread().stackTrace } )
+
+        val testExcp = TestException("hello", Throwable().apply { stackTrace = Thread.currentThread().stackTrace })
         val factory = testDefaultFactoryNoEvolution()
         SerializationOutput(factory).serialize(testExcp)