Merge remote-tracking branch 'public/master' into chrisr3-merge-os

O/S merge up to ef703c50

buildSrc/build.gradle

@@ -6,11 +6,16 @@ buildscript {
 }
 
 apply plugin: 'maven'
+apply plugin: 'java'
 
 repositories {
+    mavenLocal()
     mavenCentral()
 }
 
 dependencies {
-    compile "com.google.guava:guava:$guava_version"
+    // Add the top-level projects ONLY to the host project.
+    runtime project.childProjects.values().collect {
+        project(it.path)
+    }
 }

buildSrc/canonicalizer/build.gradle

@@ -0,0 +1,22 @@
+plugins {
+    id 'groovy'
+    id 'java-gradle-plugin'
+}
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+gradlePlugin {
+    plugins {
+        canonicalizerPlugin {
+            id = 'net.corda.plugins.canonicalizer'
+            implementationClass = 'CanonicalizerPlugin'
+        }
+    }
+}
+
+dependencies {
+    compile "com.google.guava:guava:$guava_version"
+}

buildSrc/settings.gradle

@@ -0,0 +1,2 @@
+rootProject.name = 'buildSrc'
+include 'canonicalizer'

core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt

@@ -5,7 +5,6 @@ package net.corda.core.internal
 import net.corda.core.cordapp.Cordapp
 import net.corda.core.cordapp.CordappConfig
 import net.corda.core.cordapp.CordappContext
-import net.corda.core.cordapp.CordappProvider
 import net.corda.core.crypto.*
 import net.corda.core.flows.NotarisationRequest
 import net.corda.core.flows.NotarisationRequestSignature
@@ -38,6 +37,7 @@ import java.nio.charset.Charset
 import java.nio.charset.StandardCharsets.UTF_8
 import java.nio.file.*
 import java.nio.file.attribute.FileAttribute
+import java.nio.file.attribute.FileTime
 import java.security.KeyPair
 import java.security.PrivateKey
 import java.security.cert.X509Certificate
@@ -130,6 +130,7 @@ fun Path.moveTo(target: Path, vararg options: CopyOption): Path = Files.move(thi
 fun Path.isRegularFile(vararg options: LinkOption): Boolean = Files.isRegularFile(this, *options)
 fun Path.isDirectory(vararg options: LinkOption): Boolean = Files.isDirectory(this, *options)
 inline val Path.size: Long get() = Files.size(this)
+fun Path.lastModifiedTime(vararg options: LinkOption): FileTime = Files.getLastModifiedTime(this, *options)
 inline fun <R> Path.list(block: (Stream<Path>) -> R): R = Files.list(this).use(block)
 fun Path.deleteIfExists(): Boolean = Files.deleteIfExists(this)
 fun Path.reader(charset: Charset = UTF_8): BufferedReader = Files.newBufferedReader(this, charset)
@@ -257,6 +258,13 @@ fun IntProgression.stream(parallel: Boolean = false): IntStream = StreamSupport.
 // When toArray has filled in the array, the component type is no longer T? but T (that may itself be nullable):
 inline fun <reified T> Stream<out T>.toTypedArray(): Array<T> = uncheckedCast(toArray { size -> arrayOfNulls<T>(size) })
 
+inline fun <T, R : Any> Stream<T>.mapNotNull(crossinline transform: (T) -> R?): Stream<R> {
+    return flatMap {
+        val value = transform(it)
+        if (value != null) Stream.of(value) else Stream.empty()
+    }
+}
+
 fun <T> Class<T>.castIfPossible(obj: Any): T? = if (isInstance(obj)) cast(obj) else null
 
 /** Returns a [DeclaredField] wrapper around the declared (possibly non-public) static field of the receiver [Class]. */

docs/source/event-scheduling.rst

@@ -70,7 +70,7 @@ Let's take an example of the interest rate swap fixings for our scheduled events
 
 .. container:: codeset
 
-    .. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/contract/IRS.kt
+    .. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/contract/IRS.kt
         :language: kotlin
         :start-after: DOCSTART 1
         :end-before: DOCEND 1

docs/source/oracles.rst

@@ -166,7 +166,7 @@ parameter and ``CommandData`` classes.
 
 Let's see how the ``sign`` method for ``NodeInterestRates.Oracle`` is written:
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
    :language: kotlin
    :start-after: DOCSTART 1
    :end-before: DOCEND 1
@@ -192,7 +192,7 @@ Binding to the network
 The first step is to create the oracle as a service by annotating its class with ``@CordaService``.  Let's see how that's
 done:
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
    :language: kotlin
    :start-after: DOCSTART 3
    :end-before: DOCEND 3
@@ -201,7 +201,7 @@ done:
 The Corda node scans for any class with this annotation and initialises them. The only requirement is that the class provide
 a constructor with a single parameter of type ``ServiceHub``.
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
    :language: kotlin
    :start-after: DOCSTART 2
    :end-before: DOCEND 2
@@ -219,7 +219,7 @@ We mentioned the client sub-flow briefly above.  They are the mechanism that cli
 use to interact with your oracle.  Typically there will be one for querying and one for signing.  Let's take a look at
 those for ``NodeInterestRates.Oracle``.
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/flows/RatesFixFlow.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/RatesFixFlow.kt
    :language: kotlin
    :start-after: DOCSTART 1
    :end-before: DOCEND 1
@@ -238,7 +238,7 @@ The oracle is invoked through sub-flows to query for values, add them to the tra
 the transaction signed by the oracle.  Following on from the above examples, this is all encapsulated in a sub-flow
 called ``RatesFixFlow``.  Here's the ``call`` method of that flow.
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/flows/RatesFixFlow.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/RatesFixFlow.kt
    :language: kotlin
    :start-after: DOCSTART 2
    :end-before: DOCEND 2
@@ -255,7 +255,7 @@ As you can see, this:
 
 Here's an example of it in action from ``FixingFlow.Fixer``.
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/flows/FixingFlow.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/flows/FixingFlow.kt
    :language: kotlin
    :start-after: DOCSTART 1
    :end-before: DOCEND 1

docs/source/tutorial-cordapp.rst

@@ -222,28 +222,27 @@ For each node, the ``runnodes`` script creates a node tab/window:
 
 .. sourcecode:: none
 
-       ______               __
+      ______               __
-      / ____/     _________/ /___ _
+     / ____/     _________/ /___ _
-     / /     __  / ___/ __  / __ `/         It's kind of like a block chain but
+    / /     __  / ___/ __  / __ `/         Top tip: never say "oops", instead
-    / /___  /_/ / /  / /_/ / /_/ /          cords sounded healthier than chains.
+   / /___  /_/ / /  / /_/ / /_/ /          always say "Ah, Interesting!"
-    \____/     /_/   \__,_/\__,_/
+   \____/     /_/   \__,_/\__,_/
 
-    --- Corda Open Source 0.12.1 (da47f1c) -----------------------------------------------
+   --- Corda Open Source corda-3.0 (4157c25) -----------------------------------------------
-
-    📚  New! Training now available worldwide, see https://corda.net/corda-training/
-
-    Logs can be found in                    : /Users/username/Desktop/cordapp-example/kotlin-source/build/nodes/PartyA/logs
-    Database connection url is              : jdbc:h2:tcp://10.163.199.132:54763/node
-    Listening on address                    : 127.0.0.1:10005
-    RPC service listening on address        : localhost:10006
-    Loaded plugins                          : com.example.plugin.ExamplePlugin
-    Node for "PartyA" started up and registered in 35.0 sec
 
 
-    Welcome to the Corda interactive shell.
+   Logs can be found in                    : /Users/joeldudley/Desktop/cordapp-example/kotlin-source/build/nodes/PartyA/logs
-    Useful commands include 'help' to see what is available, and 'bye' to shut down the node.
+   Database connection url is              : jdbc:h2:tcp://localhost:59472/node
+   Incoming connection address             : localhost:10005
+   Listening on port                       : 10005
+   Loaded CorDapps                         : corda-finance-corda-3.0, cordapp-example-0.1, corda-core-corda-3.0
+   Node for "PartyA" started up and registered in 38.59 sec
 
-    Fri Jul 07 10:33:47 BST 2017>>>
+
+   Welcome to the Corda interactive shell.
+   Useful commands include 'help' to see what is available, and 'bye' to shut down the node.
+
+   Fri Mar 02 17:34:02 GMT 2018>>> 
 
 For every node except the notary, the script also creates a webserver terminal tab/window:
 

docs/source/tutorial-tear-offs.rst

@@ -43,7 +43,7 @@ transaction components is exactly the same. Note that unlike ``WireTransaction``
 
 The following code snippet is taken from ``NodeInterestRates.kt`` and implements a signing part of an Oracle.
 
-.. literalinclude:: ../../samples/irs-demo/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
+.. literalinclude:: ../../samples/irs-demo/cordapp/src/main/kotlin/net/corda/irs/api/NodeInterestRates.kt
     :language: kotlin
     :start-after: DOCSTART 1
     :end-before: DOCEND 1

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/ServerContexts.kt

@@ -35,14 +35,16 @@ val KRYO_STORAGE_CONTEXT = SerializationContextImpl(kryoMagic,
         emptyMap(),
         true,
         SerializationContext.UseCase.Storage,
-        null)
+        null,
+        AlwaysAcceptEncodingWhitelist)
 val AMQP_STORAGE_CONTEXT = SerializationContextImpl(amqpMagic,
         SerializationDefaults.javaClass.classLoader,
         AllButBlacklisted,
         emptyMap(),
         true,
         SerializationContext.UseCase.Storage,
-        null)
+        null,
+        AlwaysAcceptEncodingWhitelist)
 val AMQP_RPC_SERVER_CONTEXT = SerializationContextImpl(amqpMagic,
         SerializationDefaults.javaClass.classLoader,
         GlobalTransientClassWhiteList(BuiltInExceptionsWhitelist()),

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/SharedContexts.kt

@@ -2,8 +2,10 @@
 
 package net.corda.nodeapi.internal.serialization
 
+import net.corda.core.serialization.EncodingWhitelist
 import net.corda.core.serialization.SerializationContext
 import net.corda.core.serialization.SerializationDefaults
+import net.corda.core.serialization.SerializationEncoding
 import net.corda.nodeapi.internal.serialization.amqp.amqpMagic
 import net.corda.nodeapi.internal.serialization.kryo.kryoMagic
 
@@ -28,7 +30,8 @@ val KRYO_CHECKPOINT_CONTEXT = SerializationContextImpl(kryoMagic,
         emptyMap(),
         true,
         SerializationContext.UseCase.Checkpoint,
-        null)
+        null,
+        AlwaysAcceptEncodingWhitelist)
 val AMQP_P2P_CONTEXT = SerializationContextImpl(amqpMagic,
         SerializationDefaults.javaClass.classLoader,
         GlobalTransientClassWhiteList(BuiltInExceptionsWhitelist()),
@@ -36,3 +39,7 @@ val AMQP_P2P_CONTEXT = SerializationContextImpl(amqpMagic,
         true,
         SerializationContext.UseCase.P2P,
         null)
+
+internal object AlwaysAcceptEncodingWhitelist : EncodingWhitelist {
+    override fun acceptEncoding(encoding: SerializationEncoding) = true
+}

node/src/main/kotlin/net/corda/node/services/network/NodeInfoWatcher.kt

@@ -8,6 +8,7 @@ import net.corda.core.serialization.internal.SerializationEnvironmentImpl
 import net.corda.core.serialization.internal._contextSerializationEnv
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.contextLogger
+import net.corda.core.utilities.debug
 import net.corda.core.utilities.seconds
 import net.corda.nodeapi.internal.NodeInfoAndSigned
 import net.corda.nodeapi.internal.SignedNodeInfo
@@ -17,13 +18,12 @@ import net.corda.nodeapi.internal.serialization.SerializationFactoryImpl
 import net.corda.nodeapi.internal.serialization.amqp.AMQPServerSerializationScheme
 import rx.Observable
 import rx.Scheduler
-import java.io.IOException
 import java.nio.file.Path
 import java.nio.file.Paths
 import java.nio.file.StandardCopyOption.REPLACE_EXISTING
+import java.nio.file.attribute.FileTime
 import java.time.Duration
 import java.util.concurrent.TimeUnit
-import java.util.stream.Stream
 import kotlin.streams.toList
 
 /**
@@ -58,20 +58,14 @@ class NodeInfoWatcher(private val nodePath: Path,
         }
     }
 
-    private val nodeInfoDirectory = nodePath / CordformNode.NODE_INFO_DIRECTORY
+    private val nodeInfosDir = nodePath / CordformNode.NODE_INFO_DIRECTORY
-
+    private val nodeInfoFiles = HashMap<Path, FileTime>()
     private val _processedNodeInfoHashes = HashSet<SecureHash>()
     val processedNodeInfoHashes: Set<SecureHash> get() = _processedNodeInfoHashes
 
     init {
         require(pollInterval >= 5.seconds) { "Poll interval must be 5 seconds or longer." }
-        if (!nodeInfoDirectory.isDirectory()) {
+        nodeInfosDir.createDirectories()
-            try {
-                nodeInfoDirectory.createDirectories()
-            } catch (e: IOException) {
-                logger.info("Failed to create $nodeInfoDirectory", e)
-            }
-        }
     }
 
     /**
@@ -93,42 +87,32 @@ class NodeInfoWatcher(private val nodePath: Path,
         return Companion.saveToFile(nodePath, nodeInfoAndSigned)
     }
 
-    /**
-     * Loads all the files contained in a given path and returns the deserialized [NodeInfo]s.
-     * Signatures are checked before returning a value.
-     *
-     * @return a list of [NodeInfo]s
-     */
     private fun loadFromDirectory(): List<NodeInfo> {
-        if (!nodeInfoDirectory.isDirectory()) {
+        val result = nodeInfosDir.list { paths ->
-            return emptyList()
-        }
-        val result = nodeInfoDirectory.list { paths ->
             paths
                     .filter { it.isRegularFile() }
-                    .flatMap { path ->
+                    .filter { file ->
-                        val nodeInfo = processFile(path)?.let {
+                        val lastModifiedTime = file.lastModifiedTime()
-                            if (_processedNodeInfoHashes.add(it.signed.raw.hash)) it.nodeInfo else null
+                        val previousLastModifiedTime = nodeInfoFiles[file]
+                        val newOrChangedFile = previousLastModifiedTime == null || lastModifiedTime > previousLastModifiedTime
+                        nodeInfoFiles[file] = lastModifiedTime
+                        newOrChangedFile
+                    }
+                    .mapNotNull { file ->
+                        logger.debug { "Reading SignedNodeInfo from $file" }
+                        try {
+                            NodeInfoAndSigned(file.readObject())
+                        } catch (e: Exception) {
+                            logger.warn("Unable to read SignedNodeInfo from $file", e)
+                            null
                         }
-                        if (nodeInfo != null) Stream.of(nodeInfo) else Stream.empty()
                     }
                     .toList()
         }
-        if (result.isNotEmpty()) {
-            logger.info("Successfully read ${result.size} NodeInfo files from disk.")
-        }
-        return result
-    }
 
-    private fun processFile(file: Path): NodeInfoAndSigned? {
+        logger.debug { "Read ${result.size} NodeInfo files from $nodeInfosDir" }
-        return try {
+        _processedNodeInfoHashes += result.map { it.signed.raw.hash }
-            logger.info("Reading NodeInfo from file: $file")
+        return result.map { it.nodeInfo }
-            val signedNodeInfo = file.readObject<SignedNodeInfo>()
-            NodeInfoAndSigned(signedNodeInfo)
-        } catch (e: Exception) {
-            logger.warn("Exception parsing NodeInfo from file. $file", e)
-            null
-        }
     }
 }