Merge remote-tracking branch 'open/master' into kat-merge-20180626

build.gradle

@@ -100,7 +100,6 @@ buildscript {
     ext.jcabi_manifests_version = '1.1'
     ext.picocli_version = '3.0.0'
 
-    ext.deterministic_rt_version = '1.0-SNAPSHOT'
     // Name of the IntelliJ SDK created for the deterministic Java rt.jar.
     // ext.deterministic_idea_sdk = '1.8 (Deterministic)'
 

core-deterministic/testing/common/src/main/kotlin/net/corda/deterministic/common/TransactionVerificationRequest.kt

@@ -19,7 +19,7 @@ class TransactionVerificationRequest(val wtxToVerify: SerializedBytes<WireTransa
         val deps = dependencies.map { it.deserialize() }.associateBy(WireTransaction::id)
         val attachments = attachments.map { it.deserialize<Attachment>() }
         val attachmentMap = attachments.mapNotNull { it as? MockContractAttachment }
-                .associateBy(Attachment::id, { ContractAttachment(it, it.contract, uploader=TEST_UPLOADER) })
+                .associateBy(Attachment::id) { ContractAttachment(it, it.contract, uploader=TEST_UPLOADER) }
         val contractAttachmentMap = emptyMap<ContractClassName, ContractAttachment>()
         @Suppress("DEPRECATION")
         return wtxToVerify.deserialize().toLedgerTransaction(

create-jdk8u/build.gradle

@@ -57,6 +57,7 @@ task makeJdk(type: Exec) {
 
 task runtimeJar(type: Jar, dependsOn: makeJdk) {
     baseName 'deterministic-rt'
+    inputs.dir "libs"
 
     from(zipTree("libs/rt.jar"))
     from(zipTree("libs/jce.jar"))

docs/source/deterministic-modules.rst

@@ -104,54 +104,60 @@ This is possible, but slightly tricky to configure because IntelliJ will not rec
 Gradle be configured to use the Project's SDK.
 
 Creating the Deterministic SDK
-    #. Create a JDK Home directory with the following contents:
+    Gradle creates a suitable JDK image in the project's ``jdk8u-deterministic/jdk`` directory, and you can
+    configure IntelliJ to use this location for this SDK. However, you should also be aware that IntelliJ SDKs
+    are available for *all* projects to use.
 
-        ``jre/lib/rt.jar``
+    To create this JDK image, execute the following:
 
-       where ``rt.jar`` here is this renamed artifact:
+    .. code-block:: bash
 
-       .. code-block:: xml
+        $ gradlew jdk8u-deterministic:copyJdk
 
-           <dependency>
-               <groupId>net.corda</groupId>
-               <artifactId>deterministic-rt</artifactId>
-               <classifier>api</classifier>
-           </dependency>
+    ..
 
-       ..
+    Now select ``File/Project Structure/Platform Settings/SDKs`` and add a new JDK SDK with the
+    ``jdk8u-deterministic/jdk`` directory as its home. Rename this SDK to something like "1.8 (Deterministic)".
 
-       .. note:: Gradle already creates this JDK in the project's ``jdk8u-deterministic/jdk`` directory, and you could
-                 configure IntelliJ to use this location as well. However, you should also be aware that IntelliJ SDKs
-                 are available for *all* projects to use.
+    This *should* be sufficient for IntelliJ. However, if IntelliJ realises that this SDK does not contain a
+    full JDK then you will need to configure the new SDK by hand:
 
-                 To create this deterministic JDK image, execute the following:
+        #. Create a JDK Home directory with the following contents:
 
-                 .. code-block:: bash
+            ``jre/lib/rt.jar``
 
-                     $ gradlew jdk8u-deterministic:copyJdk
+           where ``rt.jar`` here is this renamed artifact:
 
-                 ..
+           .. code-block:: xml
 
-    #. While IntelliJ is *not* running, locate the ``config/options/jdk.table.xml`` file in IntelliJ's configuration
-       directory. Add an empty ``<jdk>`` section to this file:
+               <dependency>
+                   <groupId>net.corda</groupId>
+                   <artifactId>deterministic-rt</artifactId>
+                   <classifier>api</classifier>
+               </dependency>
 
-       .. code-block:: xml
+           ..
 
-           <jdk version="2">
-               <name value="1.8 (Deterministic)"/>
-               <type value="JavaSDK"/>
-               <version value="java version &quot;1.8.0&quot;"/>
-               <homePath value=".. path to the deterministic JDK directory .."/>
-               <roots>
-               </roots>
-           </jdk>
+        #. While IntelliJ is *not* running, locate the ``config/options/jdk.table.xml`` file in IntelliJ's configuration
+           directory. Add an empty ``<jdk>`` section to this file:
 
-       ..
+           .. code-block:: xml
 
-    #. Open IntelliJ and select ``File/Project Structure/Platform Settings/SDKs``. The "1.8 (Deterministic)" SDK should
-       now be present. Select it and then click on the ``Classpath`` tab. Press the "Add" / "Plus" button to add
-       ``rt.jar`` to the SDK's classpath. Then select the ``Annotations`` tab and include the same JAR(s) as the other
-       SDKs.
+               <jdk version="2">
+                   <name value="1.8 (Deterministic)"/>
+                   <type value="JavaSDK"/>
+                   <version value="java version &quot;1.8.0&quot;"/>
+                   <homePath value=".. path to the deterministic JDK directory .."/>
+                   <roots>
+                   </roots>
+               </jdk>
+
+           ..
+
+        #. Open IntelliJ and select ``File/Project Structure/Platform Settings/SDKs``. The "1.8 (Deterministic)" SDK
+           should now be present. Select it and then click on the ``Classpath`` tab. Press the "Add" / "Plus" button to
+           add ``rt.jar`` to the SDK's classpath. Then select the ``Annotations`` tab and include the same JAR(s) as
+           the other SDKs.
 
 Configuring the Corda Project
     #. Open the root ``build.gradle`` file and define this property:

jdk8u-deterministic/build.gradle

@@ -8,23 +8,39 @@ repositories {
 }
 
 ext {
-    jdk_home = "$projectDir/jdk"
+    jdk_home = "$projectDir/jdk".toString()
     rt_jar = "$jdk_home/jre/lib/rt.jar".toString()
 }
 
 configurations {
-    jdk
+    jdk.resolutionStrategy {
+        cacheChangingModulesFor 0, 'seconds'
+    }
 }
 
 dependencies {
-    jdk "net.corda:deterministic-rt:$deterministic_rt_version:api"
+    // Ensure everyone uses the latest SNAPSHOT.
+    jdk "net.corda:deterministic-rt:latest.integration:api"
 }
 
 task copyJdk(type: Copy) {
+    outputs.dir jdk_home
+
     from(configurations.jdk.asPath) {
         rename 'deterministic-rt-(.*).jar', 'rt.jar'
     }
     into "$jdk_home/jre/lib"
+
+    doLast {
+        def eol = System.getProperty('line.separator')
+        file("$jdk_home/release").write "JAVA_VERSION=\"1.8.0_172\"$eol"
+        mkdir "$jdk_home/bin"
+        file("$jdk_home/bin/javac").with {
+            write "#!/bin/sh\necho \"javac 1.8.0_172\"\n"
+            setExecutable true, false
+            return
+        }
+    }
 }
 
 assemble.dependsOn copyJdk

node/src/main/kotlin/net/corda/node/internal/NodeStartup.kt

@@ -26,6 +26,7 @@ import net.corda.node.NodeRegistrationOption
 import net.corda.node.SerialFilter
 import net.corda.node.VersionInfo
 import net.corda.node.defaultSerialFilter
+import net.corda.node.internal.cordapp.MultipleCordappsForFlowException
 import net.corda.node.services.config.NodeConfiguration
 import net.corda.node.services.config.NodeConfigurationImpl
 import net.corda.node.services.config.shouldStartLocalShell
@@ -148,9 +149,13 @@ open class NodeStartup(val args: Array<String>) {
         try {
             cmdlineOptions.baseDirectory.createDirectories()
             startNode(conf, versionInfo, startTime, cmdlineOptions)
+
         } catch (e: DatabaseMigrationException) {
             logger.error(e.message)
             return false
+        } catch (e: MultipleCordappsForFlowException) {
+            logger.error(e.message)
+            return false
         } catch (e: CouldNotCreateDataSourceException) {
             logger.error(e.message, e.cause)
             return false

node/src/main/kotlin/net/corda/node/internal/cordapp/CordappLoader.kt

@@ -80,7 +80,7 @@ class CordappLoader private constructor(private val cordappJarPaths: List<Restri
         cordapps.flatMap { corDapp -> corDapp.allFlows.map { flow -> flow to corDapp } }
                 .groupBy { it.first }
                 .mapValues {
-                    require(it.value.size == 1) { "There are multiple CorDapp jars on the classpath for flow ${it.value.first().first.name}: ${it.value.map { it.second.name }.joinToString()}." }
+                    if(it.value.size > 1) { throw MultipleCordappsForFlowException("There are multiple CorDapp JARs on the classpath for flow ${it.value.first().first.name}: [ ${it.value.joinToString { it.second.name }} ].")  }
                     it.value.single().second
                 }
     }
@@ -401,3 +401,8 @@ class CordappLoader private constructor(private val cordappJarPaths: List<Restri
         }
     }
 }
+
+/**
+ * Thrown when scanning CorDapps.
+ */
+class MultipleCordappsForFlowException(message: String) : Exception(message)

node/src/main/kotlin/net/corda/node/internal/cordapp/CordappProviderImpl.kt

@@ -51,6 +51,9 @@ open class CordappProviderImpl(private val cordappLoader: CordappLoader,
 
     private fun verifyInstalledCordapps(attachmentStorage: AttachmentStorage) {
 
+        // This will invoke the lazy flowCordappMap property, thus triggering the MultipleCordappsForFlow check.
+        cordappLoader.flowCordappMap
+
         if (whitelistedContractImplementations.isEmpty()) {
             log.warn("The network parameters don't specify any whitelisted contract implementations. Please contact your zone operator. See https://docs.corda.net/network-map.html")
             return

serialization/src/main/kotlin/net/corda/serialization/internal/amqp/EvolutionSerializer.kt

@@ -24,6 +24,7 @@ import java.lang.reflect.Type
 import kotlin.reflect.KFunction
 import kotlin.reflect.full.findAnnotation
 import kotlin.reflect.jvm.javaType
+import kotlin.reflect.jvm.jvmErasure
 
 
 /**
@@ -127,12 +128,29 @@ abstract class EvolutionSerializer(
                 readersAsSerialized: Map<String, OldParam>): AMQPSerializer<Any> {
             val constructorArgs = arrayOfNulls<Any?>(constructor.parameters.size)
 
+            // Java doesn't care about nullability unless it's a primitive in which
+            // case it can't be referenced. Unfortunately whilst Kotlin does apply
+            // Nullability annotations we cannot use them here as they aren't
+            // retained at runtime so we cannot rely on the absence of
+            // any particular NonNullable annotation type to indicate cross
+            // compiler nullability
+            val isKotlin = (new.type.javaClass.declaredAnnotations.any {
+                        it.annotationClass.qualifiedName == "kotlin.Metadata"
+            })
+
             constructor.parameters.withIndex().forEach {
-                readersAsSerialized[it.value.name!!]?.apply {
-                    this.resultsIndex = it.index
-                } ?: if (!it.value.type.isMarkedNullable) {
-                    throw NotSerializableException(
-                            "New parameter ${it.value.name} is mandatory, should be nullable for evolution to work")
+                if ((readersAsSerialized[it.value.name!!] ?.apply { this.resultsIndex = it.index }) == null) {
+                    // If there is no value in the byte stream to map to the parameter of the constructor
+                    // this is ok IFF it's a Kotlin class and the parameter is non nullable OR
+                    // its a Java class and the parameter is anything but an unboxed primitive.
+                    // Otherwise we throw the error and leave
+                    if ((isKotlin && !it.value.type.isMarkedNullable)
+                            || (!isKotlin && isJavaPrimitive(it.value.type.jvmErasure.java))
+                    ) {
+                        throw NotSerializableException(
+                                "New parameter \"${it.value.name}\" is mandatory, should be nullable for evolution " +
+                                        "to work, isKotlinClass=$isKotlin type=${it.value.type}")
+                    }
                 }
             }
             return EvolutionSerializerViaConstructor(new.type, factory, readersAsSerialized, constructor, constructorArgs)
@@ -161,8 +179,10 @@ abstract class EvolutionSerializer(
          * @param factory the [SerializerFactory] associated with the serialization
          * context this serializer is being built for
          */
-        fun make(old: CompositeType, new: ObjectSerializer,
-                 factory: SerializerFactory): AMQPSerializer<Any> {
+        fun make(old: CompositeType,
+                 new: ObjectSerializer,
+                 factory: SerializerFactory
+        ): AMQPSerializer<Any> {
             // The order in which the properties were serialised is important and must be preserved
             val readersAsSerialized = LinkedHashMap<String, OldParam>()
             old.fields.forEach {

serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializationHelper.kt

@@ -550,3 +550,18 @@ fun hasCordaSerializable(type: Class<*>): Boolean {
             || type.interfaces.any(::hasCordaSerializable)
             || (type.superclass != null && hasCordaSerializable(type.superclass))
 }
+
+fun isJavaPrimitive(type: Class<*>) = type in JavaPrimitiveTypes.primativeTypes
+
+private object JavaPrimitiveTypes {
+    val primativeTypes = hashSetOf<Class<*>>(
+    Boolean::class.java,
+    Char::class.java,
+    Byte::class.java,
+    Short::class.java,
+    Int::class.java,
+    Long::class.java,
+    Float::class.java,
+    Double::class.java,
+    Void::class.java)
+}

serialization/src/main/kotlin/net/corda/serialization/internal/amqp/SerializerFactory.kt

@@ -88,7 +88,7 @@ open class SerializerFactory(
                 lenientCarpenter: Boolean = false,
                 evolutionSerializerGetter: EvolutionSerializerGetterBase = EvolutionSerializerGetter(),
                 fingerPrinter: FingerPrinter = SerializerFingerPrinter()
-    ) : this(whitelist, ClassCarpenterImpl(classLoader, whitelist, lenientCarpenter), evolutionSerializerGetter, fingerPrinter)
+    ) : this(whitelist, ClassCarpenterImpl(whitelist, classLoader, lenientCarpenter), evolutionSerializerGetter, fingerPrinter)
 
     init {
         fingerPrinter.setOwner(this)

serialization/src/main/kotlin/net/corda/serialization/internal/carpenter/ClassCarpenter.kt

@@ -114,10 +114,10 @@ interface ClassCarpenter {
  * Equals/hashCode methods are not yet supported.
  */
 @DeleteForDJVM
-class ClassCarpenterImpl(cl: ClassLoader = Thread.currentThread().contextClassLoader,
-                         override val whitelist: ClassWhitelist,
-                         private val lenient: Boolean = false) : ClassCarpenter {
-
+class ClassCarpenterImpl @JvmOverloads constructor (override val whitelist: ClassWhitelist,
+                         cl: ClassLoader = Thread.currentThread().contextClassLoader,
+                         private val lenient: Boolean = false
+) : ClassCarpenter {
     // TODO: Generics.
     // TODO: Sandbox the generated code when a security manager is in use.
     // TODO: Generate equals/hashCode.

serialization/src/test/java/net/corda/serialization/internal/amqp/JavaEvolutionTests.java

@@ -0,0 +1,100 @@
+package net.corda.serialization.internal.amqp;
+
+import kotlin.Suppress;
+import net.corda.core.serialization.SerializedBytes;
+import net.corda.serialization.internal.amqp.testutils.AMQPTestUtilsKt;
+import net.corda.serialization.internal.amqp.testutils.TestSerializationContext;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.io.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class JavaEvolutionTests {
+    @Rule
+    public final ExpectedException exception = ExpectedException.none();
+
+    // Class as it was when it was serialized and written to disk. Uncomment
+    // if the test referencing the object needs regenerating.
+    /*
+    static class N1 {
+        private String word;
+        public N1(String word) { this.word = word; }
+        public String getWord() { return word; }
+    }
+    */
+    // Class as it exists now with the newly added element
+    static class N1 {
+        private String word;
+        private Integer wibble;
+
+        public N1(String word, Integer wibble) {
+            this.word = word;
+            this.wibble = wibble;
+        }
+        public String getWord() { return word; }
+        public Integer getWibble() { return wibble; }
+    }
+
+    // Class as it was when it was serialized and written to disk. Uncomment
+    // if the test referencing the object needs regenerating.
+    /*
+    static class N2 {
+        private String word;
+        public N2(String word) { this.word = word; }
+        public String getWord() { return word; }
+    }
+    */
+
+    // Class as it exists now with the newly added element
+    @SuppressWarnings("unused")
+    static class N2 {
+        private String word;
+        private float wibble;
+
+        public N2(String word, float wibble) {
+            this.word = word;
+            this.wibble = wibble;
+        }
+        public String getWord() { return word; }
+        public float getWibble() { return wibble; }
+    }
+
+    SerializerFactory factory = AMQPTestUtilsKt.testDefaultFactory();
+
+    @Test
+    public void testN1AddsNullableInt() throws IOException {
+        // Uncomment to regenerate the base state of the test
+        /*
+        N1 n = new N1("potato");
+        AMQPTestUtilsKt.writeTestResource(this, new SerializationOutput(factory).serialize(
+                n, TestSerializationContext.testSerializationContext));
+        */
+
+        N1 n2 = new DeserializationInput(factory).deserialize(
+                new SerializedBytes<>(AMQPTestUtilsKt.readTestResource(this)),
+                N1.class,
+                TestSerializationContext.testSerializationContext);
+        assertEquals(n2.getWord(), "potato");
+        assertNull(n2.getWibble());
+    }
+
+    @Test
+    public void testN2AddsPrimitive() throws IOException {
+        // Uncomment to regenerate the base state of the test
+        /*
+        N2 n = new N2("This is only a test");
+
+        AMQPTestUtilsKt.writeTestResource(this, new SerializationOutput(factory).serialize(
+                n, TestSerializationContext.testSerializationContext));
+        */
+
+        exception.expect(NotSerializableException.class);
+        new DeserializationInput(factory).deserialize(
+                new SerializedBytes<>(AMQPTestUtilsKt.readTestResource(this)),
+                N2.class,
+                TestSerializationContext.testSerializationContext);
+    }
+}

serialization/src/test/kotlin/net/corda/serialization/internal/amqp/StaticInitialisationOfSerializedObjectTest.kt

@@ -108,7 +108,7 @@ class StaticInitialisationOfSerializedObjectTest {
     // Version of a serializer factory that will allow the class carpenter living on the
     // factory to have a different whitelist applied to it than the factory
     class TestSerializerFactory(wl1: ClassWhitelist, wl2: ClassWhitelist) :
-            SerializerFactory(wl1, ClassCarpenterImpl(ClassLoader.getSystemClassLoader(), wl2))
+            SerializerFactory(wl1, ClassCarpenterImpl(wl2, ClassLoader.getSystemClassLoader()))
 
     // This time have the serialization factory and the carpenter use different whitelists
     @Test

serialization/src/test/kotlin/net/corda/serialization/internal/carpenter/CarpenterExceptionTests.kt

@@ -89,7 +89,7 @@ class CarpenterExceptionTests {
         // carpent that class up. However, when looking at the fields specified as properties of that class
         // we set the class loader of the ClassCarpenter to reject one of them, resulting in a CarpentryError
         // which we then  want the code to wrap in a NotSerializeableException
-        val cc = ClassCarpenterImpl(TestClassLoader(listOf(C2::class.jvmName)), AllWhitelist)
+        val cc = ClassCarpenterImpl(AllWhitelist, TestClassLoader(listOf(C2::class.jvmName)))
         val factory = TestFactory(cc)
 
         Assertions.assertThatThrownBy {

tools/explorer/capsule/build.gradle

@@ -12,9 +12,15 @@
  * This build.gradle exists to package Node Explorer as an executable fat jar.
  */
 apply plugin: 'us.kirchmeier.capsule'
+apply plugin: 'net.corda.plugins.publish-utils'
+apply plugin: 'com.jfrog.artifactory'
 
 description 'Node Explorer'
 
+configurations {
+    runtimeArtifacts.extendsFrom runtime
+}
+
 repositories {
     mavenLocal()
     mavenCentral()
@@ -57,3 +63,19 @@ task buildExplorerJAR(type: FatCapsule, dependsOn: project(':tools:explorer').co
 }
 
 build.dependsOn buildExplorerJAR
+
+artifacts {
+    runtimeArtifacts buildExplorerJAR
+    publish buildExplorerJAR {
+        classifier ""
+    }
+}
+
+jar {
+    classifier "ignore"
+}
+
+publish {
+    disableDefaultJar = true
+    name 'corda-tools-explorer'
+}