Merge branch 'master' into release/4

.ci/api-current.txt

@@ -88,7 +88,6 @@ public final class net.corda.core.concurrent.ConcurrencyUtils extends java.lang.
   @NotNull
   public static final String shortCircuitedTaskFailedMessage = "Short-circuited task failed:"
 ##
-@CordaSerializable
 public interface net.corda.core.concurrent.CordaFuture extends java.util.concurrent.Future
   public abstract void then(kotlin.jvm.functions.Function1<? super net.corda.core.concurrent.CordaFuture<V>, ? extends W>)
   @NotNull
@@ -571,7 +570,6 @@ public interface net.corda.core.contracts.Contract
   public abstract void verify(net.corda.core.transactions.LedgerTransaction)
 ##
 @DoNotImplement
-@CordaSerializable
 public final class net.corda.core.contracts.ContractAttachment extends java.lang.Object implements net.corda.core.contracts.Attachment
   public <init>(net.corda.core.contracts.Attachment, String)
   public <init>(net.corda.core.contracts.Attachment, String, java.util.Set<String>)
@@ -1013,7 +1011,6 @@ public final class net.corda.core.contracts.TransactionState extends java.lang.O
 ##
 public final class net.corda.core.contracts.TransactionStateKt extends java.lang.Object
 ##
-@CordaSerializable
 public abstract class net.corda.core.contracts.TransactionVerificationException extends net.corda.core.flows.FlowException
   public <init>(net.corda.core.crypto.SecureHash, String, Throwable)
   @NotNull
@@ -1039,13 +1036,13 @@ public static final class net.corda.core.contracts.TransactionVerificationExcept
 ##
 @CordaSerializable
 public static final class net.corda.core.contracts.TransactionVerificationException$ContractCreationError extends net.corda.core.contracts.TransactionVerificationException
-  public <init>(net.corda.core.crypto.SecureHash, String, Throwable)
+  public <init>(net.corda.core.crypto.SecureHash, String, Throwable, String)
   @NotNull
   public final String getContractClass()
 ##
 @CordaSerializable
 public static final class net.corda.core.contracts.TransactionVerificationException$ContractRejection extends net.corda.core.contracts.TransactionVerificationException
-  public <init>(net.corda.core.crypto.SecureHash, String, Throwable)
+  public <init>(net.corda.core.crypto.SecureHash, String, Throwable, String)
   public <init>(net.corda.core.crypto.SecureHash, net.corda.core.contracts.Contract, Throwable)
   @NotNull
   public final String getContractClass()
@@ -1390,7 +1387,6 @@ public class net.corda.core.crypto.Base58 extends java.lang.Object
   public static java.math.BigInteger decodeToBigInteger(String)
   public static String encode(byte[])
 ##
-@CordaSerializable
 public final class net.corda.core.crypto.CompositeKey extends java.lang.Object implements java.security.PublicKey
   public <init>(int, java.util.List, kotlin.jvm.internal.DefaultConstructorMarker)
   public final void checkValidity()
@@ -1746,7 +1742,6 @@ public final class net.corda.core.crypto.NullKeys extends java.lang.Object
   public final net.corda.core.crypto.TransactionSignature getNULL_SIGNATURE()
   public static final net.corda.core.crypto.NullKeys INSTANCE
 ##
-@CordaSerializable
 public static final class net.corda.core.crypto.NullKeys$NullPublicKey extends java.lang.Object implements java.security.PublicKey, java.lang.Comparable
   public int compareTo(java.security.PublicKey)
   @NotNull
@@ -6225,7 +6220,6 @@ public final class net.corda.core.transactions.SignedTransaction extends java.la
   public final net.corda.core.transactions.SignedTransaction withAdditionalSignatures(Iterable<net.corda.core.crypto.TransactionSignature>)
   public static final net.corda.core.transactions.SignedTransaction$Companion Companion
 ##
-@CordaSerializable
 public static final class net.corda.core.transactions.SignedTransaction$SignaturesMissingException extends java.security.SignatureException implements net.corda.core.CordaThrowable, net.corda.core.contracts.NamedByHash
   public <init>(java.util.Set<? extends java.security.PublicKey>, java.util.List<String>, net.corda.core.crypto.SecureHash)
   public void addSuppressed(Throwable[])
@@ -6412,7 +6406,6 @@ public final class net.corda.core.utilities.ByteArrays extends java.lang.Object
   @NotNull
   public static final String toHexString(byte[])
 ##
-@CordaSerializable
 public abstract class net.corda.core.utilities.ByteSequence extends java.lang.Object implements java.lang.Comparable
   public <init>(byte[], int, int, kotlin.jvm.internal.DefaultConstructorMarker)
   public int compareTo(net.corda.core.utilities.ByteSequence)
@@ -6628,7 +6621,6 @@ public static final class net.corda.core.utilities.OpaqueBytes$Companion extends
   @NotNull
   public final net.corda.core.utilities.OpaqueBytes of(byte...)
 ##
-@CordaSerializable
 public final class net.corda.core.utilities.OpaqueBytesSubSequence extends net.corda.core.utilities.ByteSequence
   public <init>(byte[], int, int)
   @NotNull

.idea/codeStyles/codeStyleConfig.xml

@@ -1,5 +1,6 @@
 <component name="ProjectCodeStyleConfiguration">
   <state>
     <option name="USE_PER_PROJECT_SETTINGS" value="true" />
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
   </state>
-</component>
+</component>

build.gradle

@@ -4,7 +4,7 @@ buildscript {
     file("$projectDir/constants.properties").withInputStream { constants.load(it) }
 
     // Our version: bump this on release.
-    ext.corda_release_version = "4.0"
+    ext.corda_release_version = constants.getProperty("cordaVersion")
     ext.corda_platform_version = constants.getProperty("platformVersion")
     ext.gradle_plugins_version = constants.getProperty("gradlePluginsVersion")
 
@@ -14,7 +14,7 @@ buildscript {
     ext.kotlin_version = constants.getProperty("kotlinVersion")
 
     ext.quasar_group = 'co.paralleluniverse'
-    ext.quasar_version = '0.7.10'
+    ext.quasar_version = constants.getProperty("quasarVersion")
 
     // gradle-capsule-plugin:1.0.2 contains capsule:1.0.1 by default.
     // We must configure it manually to use the latest capsule version.
@@ -57,6 +57,7 @@ buildscript {
     ext.jsr305_version = constants.getProperty("jsr305Version")
     ext.shiro_version = '1.4.0'
     ext.artifactory_plugin_version = constants.getProperty('artifactoryPluginVersion')
+    ext.hikari_version = '2.5.1'
     ext.liquibase_version = '3.5.5'
     ext.artifactory_contextUrl = 'https://ci-artifactory.corda.r3cev.com/artifactory'
     ext.snake_yaml_version = constants.getProperty('snakeYamlVersion')
@@ -80,7 +81,7 @@ buildscript {
     // Update 121 is required for ObjectInputFilter.
     // Updates [131, 161] also have zip compression bugs on MacOS (High Sierra).
     // when the java version in NodeStartup.hasMinimumJavaVersion() changes, so must this check
-    ext.java8_minUpdateVersion = '171'
+    ext.java8_minUpdateVersion = constants.getProperty('java8MinUpdateVersion')
 
     repositories {
         mavenLocal()
@@ -161,9 +162,9 @@ allprojects {
         suppressionFile = '.ci/dependency-checker/suppressedLibraries.xml'
         cveValidForHours = 1
         format = 'ALL'
-        failOnError = project.getProperty('owasp.failOnError')
+        failOnError = project.property('owasp.failOnError')
         // by default CVSS is '11' which passes everything. Set between 0-10 to catch vulnerable deps
-        failBuildOnCVSS = project.getProperty('owasp.failBuildOnCVSS').toFloat()
+        failBuildOnCVSS = project.property('owasp.failBuildOnCVSS').toFloat()
 
         analyzers {
             assemblyEnabled = false
@@ -179,7 +180,7 @@ allprojects {
         options.encoding = 'UTF-8'
     }
 
-    tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all {
+    tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) {
         kotlinOptions {
             languageVersion = "1.2"
             apiVersion = "1.2"
@@ -207,8 +208,12 @@ allprojects {
         // Prevent the project from creating temporary files outside of the build directory.
         systemProperty 'java.io.tmpdir', buildDir.absolutePath
 
+        if (project.hasProperty('test.parallel') && project.property('test.parallel').toBoolean()) {
+            maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) as int ?: 1
+        }
+
         if (System.getProperty("test.maxParallelForks") != null) {
-            maxParallelForks = Integer.valueOf(System.getProperty("test.maxParallelForks"))
+            maxParallelForks = Integer.getInteger('test.maxParallelForks')
             logger.debug("System property test.maxParallelForks found - setting max parallel forks to $maxParallelForks for $project")
         }
 
@@ -232,6 +237,7 @@ allprojects {
         jcenter()
         maven { url "$artifactory_contextUrl/corda-dependencies" }
         maven { url 'https://jitpack.io' }
+        maven { url 'https://repo.gradle.org/gradle/libs-releases' }
     }
 
     configurations {
@@ -271,14 +277,6 @@ allprojects {
     }
 }
 
-subprojects {
-    tasks.withType(Test) {
-        if (project.getProperty('test.parallel').toBoolean()) {
-            maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
-        }
-    }
-}
-
 // Check that we are running on a Java 8 JDK. The source/targetCompatibility values above aren't sufficient to
 // guarantee this because those are properties checked by the Java plugin, but we're using Kotlin.
 //
@@ -361,7 +359,6 @@ bintrayConfig {
             'corda-core',
             'corda-core-deterministic',
             'corda-deterministic-verifier',
-            'corda-djvm',
             'corda',
             'corda-finance-workflows',
             'corda-finance-contracts',

client/jackson/src/main/kotlin/net/corda/client/jackson/internal/CordaModule.kt

@@ -15,16 +15,20 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize
 import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier
 import com.fasterxml.jackson.databind.deser.ContextualDeserializer
 import com.fasterxml.jackson.databind.deser.std.DelegatingDeserializer
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer
 import com.fasterxml.jackson.databind.module.SimpleModule
 import com.fasterxml.jackson.databind.node.IntNode
 import com.fasterxml.jackson.databind.node.ObjectNode
 import com.fasterxml.jackson.databind.ser.BeanPropertyWriter
 import com.fasterxml.jackson.databind.ser.BeanSerializerModifier
+import com.fasterxml.jackson.databind.ser.std.StdScalarSerializer
+import com.fasterxml.jackson.databind.ser.std.UUIDSerializer
 import com.google.common.primitives.Booleans
 import net.corda.client.jackson.JacksonSupport
 import net.corda.core.contracts.*
 import net.corda.core.crypto.*
 import net.corda.core.crypto.PartialMerkleTree.PartialTree
+import net.corda.core.flows.StateMachineRunId
 import net.corda.core.identity.*
 import net.corda.core.internal.DigitalSignatureWithCert
 import net.corda.core.internal.createComponentGroups
@@ -40,7 +44,6 @@ import net.corda.core.utilities.parseAsHex
 import net.corda.core.utilities.toHexString
 import net.corda.serialization.internal.AllWhitelist
 import net.corda.serialization.internal.amqp.*
-import net.corda.serialization.internal.model.LocalTypeInformation
 import java.math.BigDecimal
 import java.security.PublicKey
 import java.security.cert.CertPath
@@ -79,6 +82,7 @@ class CordaModule : SimpleModule("corda-core") {
         context.setMixInAnnotations(SignatureMetadata::class.java, SignatureMetadataMixin::class.java)
         context.setMixInAnnotations(PartialTree::class.java, PartialTreeMixin::class.java)
         context.setMixInAnnotations(NodeInfo::class.java, NodeInfoMixin::class.java)
+        context.setMixInAnnotations(StateMachineRunId::class.java, StateMachineRunIdMixin::class.java)
     }
 }
 
@@ -418,6 +422,28 @@ private interface SecureHashSHA256Mixin
 @JsonDeserialize(using = JacksonSupport.PublicKeyDeserializer::class)
 private interface PublicKeyMixin
 
+@JsonSerialize(using = StateMachineRunIdSerializer::class)
+@JsonDeserialize(using = StateMachineRunIdDeserializer::class)
+private interface StateMachineRunIdMixin
+
+private class StateMachineRunIdSerializer : StdScalarSerializer<StateMachineRunId>(StateMachineRunId::class.java) {
+    private val uuidSerializer = UUIDSerializer()
+
+    override fun isEmpty(provider: SerializerProvider?, value: StateMachineRunId): Boolean {
+        return uuidSerializer.isEmpty(provider, value.uuid)
+    }
+
+    override fun serialize(value: StateMachineRunId, gen: JsonGenerator?, provider: SerializerProvider?) {
+        uuidSerializer.serialize(value.uuid, gen, provider)
+    }
+}
+
+private class StateMachineRunIdDeserializer : FromStringDeserializer<StateMachineRunId>(StateMachineRunId::class.java) {
+    override fun _deserialize(value: String, ctxt: DeserializationContext?): StateMachineRunId {
+        return StateMachineRunId(UUID.fromString(value))
+    }
+}
+
 @Suppress("unused_parameter")
 @ToStringSerialize
 private abstract class AmountMixin @JsonCreator(mode = DISABLED) constructor(

client/jackson/src/test/kotlin/net/corda/client/jackson/StateMachineRunIdTest.kt

@@ -0,0 +1,29 @@
+package net.corda.client.jackson
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import net.corda.client.jackson.internal.CordaModule
+import net.corda.core.flows.StateMachineRunId
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import java.util.*
+
+class StateMachineRunIdTest {
+    private companion object {
+        private const val ID = "a9da3d32-a08d-4add-a633-66bc6bf6183d"
+        private val jsonMapper: ObjectMapper = ObjectMapper().registerModule(CordaModule())
+    }
+
+    @Test
+    fun `state machine run ID deserialise`() {
+        val str = """"$ID""""
+        val runID = jsonMapper.readValue(str, StateMachineRunId::class.java)
+        assertEquals(StateMachineRunId(UUID.fromString(ID)), runID)
+    }
+
+    @Test
+    fun `state machine run ID serialise`() {
+        val runId = StateMachineRunId(UUID.fromString(ID))
+        val str = jsonMapper.writeValueAsString(runId)
+        assertEquals(""""$ID"""", str)
+    }
+}

client/jfx/src/main/kotlin/net/corda/client/jfx/model/NodeMonitorModel.kt

@@ -119,19 +119,19 @@ class NodeMonitorModel : AutoCloseable {
                 }.toSet()
                 val consumedStates = statesSnapshot.states.toSet() - unconsumedStates
                 val initialVaultUpdate = Vault.Update(consumedStates, unconsumedStates, references = emptySet())
-                vaultUpdates.startWith(initialVaultUpdate).subscribe({ vaultUpdatesSubject.onNext(it) }, {})
+                vaultUpdates.startWith(initialVaultUpdate).subscribe(vaultUpdatesSubject::onNext, {})
 
                 // Transactions
                 val (transactions, newTransactions) = proxy.internalVerifiedTransactionsFeed()
-                newTransactions.startWith(transactions).subscribe({ transactionsSubject.onNext(it) }, {})
+                newTransactions.startWith(transactions).subscribe(transactionsSubject::onNext, {})
 
                 // SM -> TX mapping
                 val (smTxMappings, futureSmTxMappings) = proxy.stateMachineRecordedTransactionMappingFeed()
-                futureSmTxMappings.startWith(smTxMappings).subscribe({ stateMachineTransactionMappingSubject.onNext(it) }, {})
+                futureSmTxMappings.startWith(smTxMappings).subscribe(stateMachineTransactionMappingSubject::onNext, {})
 
                 // Parties on network
                 val (parties, futurePartyUpdate) = proxy.networkMapFeed()
-                futurePartyUpdate.startWith(parties.map { MapChange.Added(it) }).subscribe({ networkMapSubject.onNext(it) }, {})
+                futurePartyUpdate.startWith(parties.map(MapChange::Added)).subscribe(networkMapSubject::onNext, {})
             }
         }
 

client/jfx/src/main/kotlin/net/corda/client/jfx/model/TransactionDataModel.kt

@@ -5,11 +5,24 @@ import net.corda.client.jfx.utils.distinctBy
 import net.corda.client.jfx.utils.lift
 import net.corda.client.jfx.utils.map
 import net.corda.client.jfx.utils.recordInSequence
-import net.corda.core.contracts.ContractState
+import net.corda.core.contracts.*
-import net.corda.core.contracts.StateAndRef
+import net.corda.core.crypto.entropyToKeyPair
-import net.corda.core.contracts.StateRef
+import net.corda.core.identity.AbstractParty
+import net.corda.core.identity.CordaX500Name
+import net.corda.core.identity.Party
+import net.corda.core.internal.eagerDeserialise
+import net.corda.core.transactions.LedgerTransaction
 import net.corda.core.transactions.SignedTransaction
 import net.corda.core.transactions.WireTransaction
+import java.math.BigInteger.ZERO
+
+private class Unknown : Contract {
+    override fun verify(tx: LedgerTransaction) = throw UnsupportedOperationException()
+
+    object State : ContractState {
+        override val participants: List<AbstractParty> = emptyList()
+    }
+}
 
 /**
  * [PartiallyResolvedTransaction] holds a [SignedTransaction] that has zero or more inputs resolved. The intent is
@@ -41,10 +54,24 @@ data class PartiallyResolvedTransaction(
     }
 
     companion object {
+        private val DUMMY_NOTARY = Party(CordaX500Name("Dummy Notary", "Nowhere", "ZZ"), entropyToKeyPair(ZERO).public)
+
         fun fromSignedTransaction(
                 transaction: SignedTransaction,
                 inputTransactions: Map<StateRef, SignedTransaction?>
         ): PartiallyResolvedTransaction {
+            /**
+             * Forcibly deserialize our transaction outputs up-front.
+             * Replace any [TransactionState] objects that fail to
+             * deserialize with a dummy transaction state that uses
+             * the transaction's notary.
+             */
+            val unknownTransactionState = TransactionState(
+                data = Unknown.State,
+                contract = Unknown::class.java.name,
+                notary = transaction.notary ?: DUMMY_NOTARY
+            )
+            transaction.coreTransaction.outputs.eagerDeserialise { _, _ -> unknownTransactionState }
             return PartiallyResolvedTransaction(
                     transaction = transaction,
                     inputs = transaction.inputs.map { stateRef ->

client/rpc/src/integration-test/kotlin/net/corda/client/rpc/RPCStabilityTests.kt

@@ -11,7 +11,8 @@ import net.corda.core.serialization.serialize
 import net.corda.core.utilities.*
 import net.corda.node.services.rpc.RPCServerConfiguration
 import net.corda.nodeapi.RPCApi
-import net.corda.nodeapi.eventually
+import net.corda.testing.common.internal.eventually
+import net.corda.testing.common.internal.succeeds
 import net.corda.testing.core.SerializationEnvironmentRule
 import net.corda.testing.driver.internal.incrementalPortAllocation
 import net.corda.testing.internal.testThreadFactory
@@ -249,7 +250,9 @@ class RPCStabilityTests {
             assertEquals("pong", client.ping())
             serverFollower.shutdown()
             startRpcServer<ReconnectOps>(ops = ops, customPort = serverPort).getOrThrow()
-            val response = eventually<RPCException, String>(10.seconds) { client.ping() }
+            val response = eventually {
+                succeeds { client.ping() }
+            }
             assertEquals("pong", response)
             clientFollower.shutdown() // Driver would do this after the new server, causing hang.
         }
@@ -316,13 +319,13 @@ class RPCStabilityTests {
                      })
 
             serverFollower.shutdown()
-            Thread.sleep(100)
-
-            assertTrue(terminateHandlerCalled)
-            assertTrue(errorHandlerCalled)
-            assertEquals("Connection failure detected.", exceptionMessage)
-            assertTrue(subscription.isUnsubscribed)
 
+            eventually {
+                assertTrue(terminateHandlerCalled)
+                assertTrue(errorHandlerCalled)
+                assertEquals("Connection failure detected.", exceptionMessage)
+                assertTrue(subscription.isUnsubscribed)
+            }
             clientFollower.shutdown() // Driver would do this after the new server, causing hang.
         }
     }

client/rpc/src/smoke-test/java/net/corda/java/rpc/StandaloneCordaRPCJavaClientTest.java

@@ -25,6 +25,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Stream;
 
+import static java.util.Collections.singletonList;
 import static kotlin.test.AssertionsKt.assertEquals;
 import static kotlin.test.AssertionsKt.fail;
 import static net.corda.finance.workflows.GetBalances.getCashBalance;
@@ -51,13 +52,12 @@ public class StandaloneCordaRPCJavaClientTest {
         }
     }
 
-    private List<String> perms = Collections.singletonList("ALL");
+    private List<String> perms = singletonList("ALL");
     private Set<String> permSet = new HashSet<>(perms);
-    private User rpcUser = new User("user1", "test", permSet);
+    private User superUser = new User("superUser", "test", permSet);
 
     private AtomicInteger port = new AtomicInteger(15000);
 
-    private NodeProcess.Factory factory;
     private NodeProcess notary;
     private CordaRPCOps rpcProxy;
     private CordaRPCConnection connection;
@@ -69,16 +69,16 @@ public class StandaloneCordaRPCJavaClientTest {
             port.getAndIncrement(),
             port.getAndIncrement(),
             true,
-            Collections.singletonList(rpcUser),
+            singletonList(superUser),
             true
     );
 
     @Before
     public void setUp() {
-        factory = new NodeProcess.Factory();
+        NodeProcess.Factory factory = new NodeProcess.Factory();
         copyCordapps(factory, notaryConfig);
         notary = factory.create(notaryConfig);
-        connection = notary.connect();
+        connection = notary.connect(superUser);
         rpcProxy = connection.getProxy();
         notaryNodeIdentity = rpcProxy.nodeInfo().getLegalIdentities().get(0);
     }
@@ -95,7 +95,7 @@ public class StandaloneCordaRPCJavaClientTest {
     }
 
     @Test
-    public void testCashBalances() throws NoSuchFieldException, ExecutionException, InterruptedException {
+    public void testCashBalances() throws ExecutionException, InterruptedException {
         Amount<Currency> dollars123 = new Amount<>(123, Currency.getInstance("USD"));
 
         FlowHandle<AbstractCashFlow.Result> flowHandle = rpcProxy.startFlowDynamic(CashIssueFlow.class,
@@ -105,7 +105,7 @@ public class StandaloneCordaRPCJavaClientTest {
         flowHandle.getReturnValue().get();
 
         Amount<Currency> balance = getCashBalance(rpcProxy, Currency.getInstance("USD"));
-        System.out.print("Balance: " + balance + "\n");
+        System.out.println("Balance: " + balance);
 
         assertEquals(dollars123, balance, "matching");
     }

client/rpc/src/smoke-test/kotlin/net/corda/kotlin/rpc/StandaloneCordaRPClientTest.kt

@@ -3,6 +3,7 @@ package net.corda.kotlin.rpc
 import com.google.common.hash.Hashing
 import com.google.common.hash.HashingInputStream
 import net.corda.client.rpc.CordaRPCConnection
+import net.corda.client.rpc.PermissionException
 import net.corda.core.crypto.SecureHash
 import net.corda.core.identity.CordaX500Name
 import net.corda.core.identity.Party
@@ -29,10 +30,8 @@ import net.corda.nodeapi.internal.config.User
 import net.corda.smoketesting.NodeConfig
 import net.corda.smoketesting.NodeProcess
 import org.apache.commons.io.output.NullOutputStream
-import org.junit.After
+import org.junit.*
-import org.junit.Before
+import org.junit.rules.ExpectedException
-import org.junit.Ignore
-import org.junit.Test
 import java.io.FilterInputStream
 import java.io.InputStream
 import java.util.*
@@ -46,7 +45,10 @@ import kotlin.test.assertTrue
 class StandaloneCordaRPClientTest {
     private companion object {
         private val log = contextLogger()
-        val user = User("user1", "test", permissions = setOf("ALL"))
+        val superUser = User("superUser", "test", permissions = setOf("ALL"))
+        val nonUser = User("nonUser", "test", permissions = emptySet())
+        val rpcUser = User("rpcUser", "test", permissions = setOf("InvokeRpc.startFlow", "InvokeRpc.killFlow"))
+        val flowUser = User("flowUser", "test", permissions = setOf("StartFlow.net.corda.finance.flows.CashIssueFlow"))
         val port = AtomicInteger(15200)
         const val attachmentSize = 2116
         val timeout = 60.seconds
@@ -65,15 +67,18 @@ class StandaloneCordaRPClientTest {
             rpcPort = port.andIncrement,
             rpcAdminPort = port.andIncrement,
             isNotary = true,
-            users = listOf(user)
+            users = listOf(superUser, nonUser, rpcUser, flowUser)
     )
 
+    @get:Rule
+    val exception: ExpectedException = ExpectedException.none()
+
     @Before
     fun setUp() {
         factory = NodeProcess.Factory()
         StandaloneCordaRPCJavaClientTest.copyCordapps(factory, notaryConfig)
         notary = factory.create(notaryConfig)
-        connection = notary.connect()
+        connection = notary.connect(superUser)
         rpcProxy = connection.proxy
         notaryNode = fetchNotaryIdentity()
         notaryNodeIdentity = rpcProxy.nodeInfo().legalIdentitiesAndCerts.first().party
@@ -81,9 +86,7 @@ class StandaloneCordaRPClientTest {
 
     @After
     fun done() {
-        try {
+        connection.use {
-            connection.close()
-        } finally {
             notary.close()
         }
     }
@@ -232,6 +235,27 @@ class StandaloneCordaRPClientTest {
         assertEquals(629.DOLLARS, balance)
     }
 
+    @Test
+    fun `test kill flow without killFlow permission`() {
+        exception.expect(PermissionException::class.java)
+        exception.expectMessage("User not authorized to perform RPC call killFlow")
+
+        val flowHandle = rpcProxy.startFlow(::CashIssueFlow, 10.DOLLARS, OpaqueBytes.of(0), notaryNodeIdentity)
+        notary.connect(nonUser).use { connection ->
+            val rpcProxy = connection.proxy
+            rpcProxy.killFlow(flowHandle.id)
+        }
+    }
+
+    @Test
+    fun `test kill flow with killFlow permission`() {
+        val flowHandle = rpcProxy.startFlow(::CashIssueFlow, 83.DOLLARS, OpaqueBytes.of(0), notaryNodeIdentity)
+        notary.connect(rpcUser).use { connection ->
+            val rpcProxy = connection.proxy
+            assertTrue(rpcProxy.killFlow(flowHandle.id))
+        }
+    }
+
     private fun fetchNotaryIdentity(): NodeInfo {
         val nodeInfo = rpcProxy.networkMapSnapshot()
         assertEquals(1, nodeInfo.size)

config/dev/log4j2.xml

@@ -4,6 +4,7 @@
     <Properties>
         <Property name="log-path">${sys:log-path:-logs}</Property>
         <Property name="log-name">node-${hostName}</Property>
+        <Property name="diagnostic-log-name">diagnostic-${hostName}</Property>
         <Property name="archive">${log-path}/archive</Property>
         <Property name="defaultLogLevel">${sys:defaultLogLevel:-info}</Property>
         <Property name="consoleLogLevel">${sys:consoleLogLevel:-error}</Property>
@@ -105,6 +106,46 @@
 
         </RollingRandomAccessFile>
 
+        <!-- Will generate up to 100 log files for a given day. During every rollover it will delete
+             those that are older than 60 days, but keep the most recent 10 GB -->
+        <RollingRandomAccessFile name="Diagnostic-RollingFile-Appender"
+                     fileName="${log-path}/${diagnostic-log-name}.log"
+                     filePattern="${archive}/${diagnostic-log-name}.%date{yyyy-MM-dd}-%i.log.gz">
+
+            <PatternLayout>
+                <ScriptPatternSelector defaultPattern="[%-5level] %date{ISO8601}{UTC}Z [%t] %c{2}.%method - %msg%n">
+                    <Script name="MDCSelector" language="javascript"><![CDATA[
+                    result = null;
+                    if (!logEvent.getContextData().size() == 0) {
+                        result = "WithMDC";
+                    } else {
+                        result = null;
+                    }
+                    result;
+               ]]>
+                    </Script>
+                    <PatternMatch key="WithMDC" pattern="[%-5level] %date{ISO8601}{UTC}Z [%t] %c{2}.%method - %msg %X%n"/>
+                </ScriptPatternSelector>
+            </PatternLayout>
+
+            <Policies>
+                <TimeBasedTriggeringPolicy/>
+                <SizeBasedTriggeringPolicy size="100MB"/>
+            </Policies>
+
+            <DefaultRolloverStrategy min="1" max="100">
+                <Delete basePath="${archive}" maxDepth="1">
+                    <IfFileName glob="${log-name}*.log.gz"/>
+                    <IfLastModified age="60d">
+                        <IfAny>
+                            <IfAccumulatedFileSize exceeds="10 GB"/>
+                        </IfAny>
+                    </IfLastModified>
+                </Delete>
+            </DefaultRolloverStrategy>
+
+        </RollingRandomAccessFile>
+
         <Rewrite name="Console-ErrorCode-Selector">
             <AppenderRef ref="Console-Selector"/>
             <ErrorCodeRewritePolicy/>
@@ -119,6 +160,10 @@
             <AppenderRef ref="RollingFile-Appender"/>
             <ErrorCodeRewritePolicy/>
         </Rewrite>
+        <Rewrite name="Diagnostic-RollingFile-ErrorCode-Appender">
+            <AppenderRef ref="Diagnostic-RollingFile-Appender"/>
+            <ErrorCodeRewritePolicy/>
+        </Rewrite>
     </Appenders>
 
     <Loggers>
@@ -130,6 +175,9 @@
             <AppenderRef ref="Console-ErrorCode-Appender-Println"/>
             <AppenderRef ref="RollingFile-ErrorCode-Appender"/>
         </Logger>
+        <Logger name="org.hibernate" level="warn" additivity="false">
+            <AppenderRef ref="Diagnostic-RollingFile-ErrorCode-Appender"/>
+        </Logger>
         <Logger name="org.hibernate.SQL" level="info" additivity="false">
             <AppenderRef ref="Console-ErrorCode-Selector"/>
             <AppenderRef ref="RollingFile-ErrorCode-Appender"/>

constants.properties

@@ -1,11 +1,18 @@
-gradlePluginsVersion=4.0.39
+# This file is parsed from Python in the docs/source/conf.py file
+# because some versions here need to be matched by app authors in
+# their own projects. So don't get fancy with syntax!
+
+cordaVersion=5.0-SNAPSHOT
+gradlePluginsVersion=4.0.42
 kotlinVersion=1.2.71
+java8MinUpdateVersion=171
 # ***************************************************************#
 # When incrementing platformVersion make sure to update          #
 # net.corda.core.internal.CordaUtilsKt.PLATFORM_VERSION as well. #
 # ***************************************************************#
 platformVersion=4
 guavaVersion=25.1-jre
+quasarVersion=0.7.10
 proguardVersion=6.0.3
 bouncycastleVersion=1.60
 disruptorVersion=3.4.2

core-deterministic/build.gradle

@@ -22,7 +22,7 @@ dependencies {
     // and without any obviously non-deterministic ones such as Hibernate.
     deterministicLibraries "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
     deterministicLibraries "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
-    deterministicLibraries "org.hibernate.javax.persistence:hibernate-jpa-2.1-api:1.0.0.Final"
+    deterministicLibraries "javax.persistence:javax.persistence-api:2.2"
     deterministicLibraries "org.bouncycastle:bcprov-jdk15on:$bouncycastle_version"
     deterministicLibraries "org.bouncycastle:bcpkix-jdk15on:$bouncycastle_version"
     deterministicLibraries "com.google.code.findbugs:jsr305:$jsr305_version"

core/build.gradle

@@ -106,8 +106,8 @@ dependencies {
     compile "org.bouncycastle:bcprov-jdk15on:${bouncycastle_version}"
     compile "org.bouncycastle:bcpkix-jdk15on:${bouncycastle_version}"
 
-    // JPA 2.1 annotations.
+    // JPA 2.2 annotations.
-    compile "org.hibernate.javax.persistence:hibernate-jpa-2.1-api:1.0.0.Final"
+    compile "javax.persistence:javax.persistence-api:2.2"
 
     // required to use @Type annotation
     compile "org.hibernate:hibernate-core:$hibernate_version"

core/src/main/kotlin/net/corda/core/concurrent/CordaFuture.kt

@@ -1,6 +1,5 @@
 package net.corda.core.concurrent
 
-import net.corda.core.serialization.CordaSerializable
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.Future
 
@@ -8,7 +7,6 @@ import java.util.concurrent.Future
  * Same as [Future] with additional methods to provide some of the features of [java.util.concurrent.CompletableFuture] while minimising the API surface area.
  * In Kotlin, to avoid compile errors, whenever CordaFuture is used in a parameter or extension method receiver type, its type parameter should be specified with out variance.
  */
-@CordaSerializable
 interface CordaFuture<V> : Future<V> {
     /**
      * Run the given callback when this future is done, on the completion thread.

core/src/main/kotlin/net/corda/core/contracts/ContractAttachment.kt

@@ -14,7 +14,6 @@ import java.security.PublicKey
  * @property additionalContracts Additional contract names contained within the JAR.
  */
 @KeepForDJVM
-@CordaSerializable
 class ContractAttachment private constructor(
         val attachment: Attachment,
         val contract: ContractClassName,

core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt

@@ -46,7 +46,6 @@ class AttachmentResolutionException(val hash: SecureHash) : FlowException("Attac
  * @property txId the Merkle root hash (identifier) of the transaction that failed verification.
  */
 @Suppress("MemberVisibilityCanBePrivate")
-@CordaSerializable
 abstract class TransactionVerificationException(val txId: SecureHash, message: String, cause: Throwable?)
     : FlowException("$message, transaction: $txId", cause) {
 
@@ -57,8 +56,8 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S
      * @property contractClass The fully qualified class name of the failing contract.
      */
     @KeepForDJVM
-    class ContractRejection(txId: SecureHash, val contractClass: String, cause: Throwable) : TransactionVerificationException(txId, "Contract verification failed: ${cause.message}, contract: $contractClass", cause) {
+    class ContractRejection internal constructor(txId: SecureHash, val contractClass: String, cause: Throwable?, message: String) : TransactionVerificationException(txId, "Contract verification failed: $message, contract: $contractClass", cause) {
-        constructor(txId: SecureHash, contract: Contract, cause: Throwable) : this(txId, contract.javaClass.name, cause)
+        internal constructor(txId: SecureHash, contract: Contract, cause: Throwable) : this(txId, contract.javaClass.name, cause, cause.message ?: "")
     }
 
     /**
@@ -121,8 +120,10 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S
      * @property contractClass The fully qualified class name of the failing contract.
      */
     @KeepForDJVM
-    class ContractCreationError(txId: SecureHash, val contractClass: String, cause: Throwable)
+    class ContractCreationError internal constructor(txId: SecureHash, val contractClass: String, cause: Throwable?, message: String)
-        : TransactionVerificationException(txId, "Contract verification failed: ${cause.message}, could not create contract class: $contractClass", cause)
+        : TransactionVerificationException(txId, "Contract verification failed: $message, could not create contract class: $contractClass", cause) {
+        internal constructor(txId: SecureHash, contractClass: String, cause: Throwable) : this(txId, contractClass, cause, cause.message ?: "")
+    }
 
     /**
      * An output state has a notary that doesn't match the transaction's notary field. It must!
@@ -267,7 +268,7 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S
      */
     @CordaSerializable
     @KeepForDJVM
-    class OverlappingAttachmentsException(txId: SecureHash, path: String) : TransactionVerificationException(txId, "Multiple attachments define a file at $path.", null)
+    class OverlappingAttachmentsException(txId: SecureHash, val path: String) : TransactionVerificationException(txId, "Multiple attachments define a file at $path.", null)
 
     /**
      * Thrown to indicate that a contract attachment is not signed by the network-wide package owner. Please note that
@@ -275,22 +276,29 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S
      * and because attachment classloaders are reused this is independent of any particular transaction.
      */
     @CordaSerializable
-    class PackageOwnershipException(txId: SecureHash, val attachmentHash: AttachmentId, val invalidClassName: String, val packageName: String) : TransactionVerificationException(txId,
+    class PackageOwnershipException(txId: SecureHash, @Suppress("unused") val attachmentHash: AttachmentId, @Suppress("unused") val invalidClassName: String, val packageName: String) : TransactionVerificationException(txId,
             """The attachment JAR: $attachmentHash containing the class: $invalidClassName is not signed by the owner of package $packageName specified in the network parameters.
            Please check the source of this attachment and if it is malicious contact your zone operator to report this incident.
            For details see: https://docs.corda.net/network-map.html#network-parameters""".trimIndent(), null)
 
     @CordaSerializable
-    class InvalidAttachmentException(txId: SecureHash, attachmentHash: AttachmentId) : TransactionVerificationException(txId,
+    class InvalidAttachmentException(txId: SecureHash, @Suppress("unused") val attachmentHash: AttachmentId) : TransactionVerificationException(txId,
             "The attachment $attachmentHash is not a valid ZIP or JAR file.".trimIndent(), null)
 
     // TODO: Make this descend from TransactionVerificationException so that untrusted attachments cause flows to be hospitalized.
     /** Thrown during classloading upon encountering an untrusted attachment (eg. not in the [TRUSTED_UPLOADERS] list) */
     @KeepForDJVM
     @CordaSerializable
-    class UntrustedAttachmentsException(txId: SecureHash, val ids: List<SecureHash>) :
+    class UntrustedAttachmentsException(val txId: SecureHash, val ids: List<SecureHash>) :
             CordaException("Attempting to load untrusted transaction attachments: $ids. " +
                     "At this time these are not loadable because the DJVM sandbox has not yet been integrated. " +
                     "You will need to install that app version yourself, to whitelist it for use. " +
                     "Please follow the operational steps outlined in https://docs.corda.net/cordapp-build-systems.html#cordapp-contract-attachments to learn more and continue.")
+
+    /*
+    If you add a new class extending [TransactionVerificationException], please add a test in `TransactionVerificationExceptionSerializationTests`
+    proving that it can actually be serialised. As a rule, exceptions intended to be serialised _must_ have a corresponding readable property
+    for every named constructor parameter - so make your constructor parameters `val`s even if nothing other than the serializer is ever
+    going to read them.
+    */
 }

core/src/main/kotlin/net/corda/core/crypto/CompositeKey.kt

@@ -29,7 +29,6 @@ import java.util.*
  * signatures required) to satisfy the sub-tree rooted at this node.
  */
 @KeepForDJVM
-@CordaSerializable
 class CompositeKey private constructor(val threshold: Int, children: List<NodeAndWeight>) : PublicKey {
     companion object {
         const val KEY_ALGORITHM = "COMPOSITE"

core/src/main/kotlin/net/corda/core/crypto/Crypto.kt

@@ -436,7 +436,7 @@ object Crypto {
             "Unsupported key/algorithm for schemeCodeName: ${signatureScheme.schemeCodeName}"
         }
         require(clearData.isNotEmpty()) { "Signing of an empty array is not permitted!" }
-        val signature = Signature.getInstance(signatureScheme.signatureName, providerMap[signatureScheme.providerName])
+        val signature = Instances.getSignatureInstance(signatureScheme.signatureName, providerMap[signatureScheme.providerName])
         // Note that deterministic signature schemes, such as EdDSA, original SPHINCS-256 and RSA PKCS#1, do not require
         // extra randomness, but we have to ensure that non-deterministic algorithms (i.e., ECDSA) use non-blocking
         // SecureRandom implementation. Also, SPHINCS-256 implementation in BouncyCastle 1.60 fails with
@@ -640,7 +640,7 @@ object Crypto {
         require(isSupportedSignatureScheme(signatureScheme)) {
             "Unsupported key/algorithm for schemeCodeName: ${signatureScheme.schemeCodeName}"
         }
-        val signature = Signature.getInstance(signatureScheme.signatureName, providerMap[signatureScheme.providerName])
+        val signature = Instances.getSignatureInstance(signatureScheme.signatureName, providerMap[signatureScheme.providerName])
         signature.initVerify(publicKey)
         signature.update(clearData)
         return signature.verify(signatureData)

core/src/main/kotlin/net/corda/core/crypto/NullKeys.kt

@@ -7,7 +7,6 @@ import java.security.PublicKey
 
 @KeepForDJVM
 object NullKeys {
-    @CordaSerializable
     object NullPublicKey : PublicKey, Comparable<PublicKey> {
         override fun getAlgorithm() = "NULL"
         override fun getEncoded() = byteArrayOf(0)

core/src/main/kotlin/net/corda/core/crypto/internal/Instances.kt

@@ -0,0 +1,12 @@
+package net.corda.core.crypto.internal
+
+import java.security.Provider
+import java.security.Signature
+
+/**
+ * This is a collection of crypto related getInstance methods that tend to be quite inefficient and we want to be able to
+ * optimise them en masse.
+ */
+object Instances {
+    fun getSignatureInstance(algorithm: String, provider: Provider?) = Signature.getInstance(algorithm, provider)
+}

core/src/main/kotlin/net/corda/core/flows/FinalityFlow.kt

@@ -114,10 +114,6 @@ class FinalityFlow private constructor(val transaction: SignedTransaction,
     @Throws(NotaryException::class)
     override fun call(): SignedTransaction {
         if (!newApi) {
-            require(CordappResolver.currentTargetVersion < 4) {
-                "A flow session for each external participant to the transaction must be provided. If you wish to continue " +
-                        "using this insecure API then specify a target platform version of less than 4 for your CorDapp."
-            }
             logger.warnOnce("The current usage of FinalityFlow is unsafe. Please consider upgrading your CorDapp to use " +
                     "FinalityFlow with FlowSessions. (${CordappResolver.currentCordapp?.info})")
         } else {

core/src/main/kotlin/net/corda/core/flows/FlowLogic.kt

@@ -468,7 +468,7 @@ abstract class FlowLogic<out T> {
         val theirs = subLogic.progressTracker
         if (ours != null && theirs != null && ours != theirs) {
             if (ours.currentStep == ProgressTracker.UNSTARTED) {
-                logger.warn("ProgressTracker has not been started")
+                logger.debug { "Initializing the progress tracker for flow: ${this::class.java.name}." }
                 ours.nextStep()
             }
             ours.setChildProgressTracker(ours.currentStep, theirs)

core/src/main/kotlin/net/corda/core/flows/ReceiveTransactionFlow.kt

@@ -18,7 +18,8 @@ import java.security.SignatureException
  * [SignedTransaction] and perform the resolution back-and-forth required to check the dependencies and download any missing
  * attachments. The flow will return the [SignedTransaction] after it is resolved and then verified using [SignedTransaction.verify].
  *
- * Please note that it will *not* store the transaction to the vault unless that is explicitly requested.
+ * Please note that it will *not* store the transaction to the vault unless that is explicitly requested and checkSufficientSignatures is true.
+ * Setting statesToRecord to anything else when checkSufficientSignatures is false will *not* update the vault.
  *
  * @property otherSideSession session to the other side which is calling [SendTransactionFlow].
  * @property checkSufficientSignatures if true checks all required signatures are present. See [SignedTransaction.verify].

core/src/main/kotlin/net/corda/core/internal/ConstraintsUtils.kt

@@ -1,7 +1,6 @@
 package net.corda.core.internal
 
 import net.corda.core.contracts.*
-import net.corda.core.crypto.isFulfilledBy
 import net.corda.core.crypto.keys
 import net.corda.core.internal.cordapp.CordappImpl
 import net.corda.core.utilities.loggerFor

core/src/main/kotlin/net/corda/core/internal/CordaUtils.kt

@@ -1,6 +1,7 @@
 package net.corda.core.internal
 
 import net.corda.core.DeleteForDJVM
+import net.corda.core.contracts.Attachment
 import net.corda.core.contracts.ContractAttachment
 import net.corda.core.contracts.ContractClassName
 import net.corda.core.flows.DataVendingFlow
@@ -41,6 +42,9 @@ fun checkMinimumPlatformVersion(minimumPlatformVersion: Int, requiredMinPlatform
     }
 }
 
+@Throws(NumberFormatException::class)
+fun getJavaUpdateVersion(javaVersion: String): Long = javaVersion.substringAfter("_").substringBefore("-").toLong()
+
 /** Provide access to internal method for AttachmentClassLoaderTests. */
 @DeleteForDJVM
 fun TransactionBuilder.toWireTransaction(services: ServicesForResolution, serializationContext: SerializationContext): WireTransaction {
@@ -106,15 +110,15 @@ fun noPackageOverlap(packages: Collection<String>): Boolean {
 }
 
 /**
- * Scans trusted (installed locally) contract attachments to find all that contain the [className].
+ * Scans trusted (installed locally) attachments to find all that contain the [className].
  * This is required as a workaround until explicit cordapp dependencies are implemented.
  * DO NOT USE IN CLIENT code.
  *
- * @return the contract attachments with the highest version.
+ * @return the attachments with the highest version.
  *
  * TODO: Should throw when the class is found in multiple contract attachments (not different versions).
  */
-fun AttachmentStorage.internalFindTrustedAttachmentForClass(className: String): ContractAttachment?{
+fun AttachmentStorage.internalFindTrustedAttachmentForClass(className: String): Attachment? {
     val allTrusted = queryAttachments(
             AttachmentQueryCriteria.AttachmentsQueryCriteria().withUploader(Builder.`in`(TRUSTED_UPLOADERS)),
             AttachmentSort(listOf(AttachmentSort.AttachmentSortColumn(AttachmentSort.AttachmentSortAttribute.VERSION, Sort.Direction.DESC))))
@@ -122,7 +126,7 @@ fun AttachmentStorage.internalFindTrustedAttachmentForClass(className: String):
     // TODO - add caching if performance is affected.
     for (attId in allTrusted) {
         val attch = openAttachment(attId)!!
-        if (attch is ContractAttachment && attch.openAsJAR().use { hasFile(it, "$className.class") }) return attch
+        if (attch.openAsJAR().use { hasFile(it, "$className.class") }) return attch
     }
     return null
 }

core/src/main/kotlin/net/corda/core/internal/InternalUtils.kt

@@ -11,6 +11,7 @@ import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.UntrustworthyData
+import net.corda.core.utilities.seconds
 import org.slf4j.Logger
 import rx.Observable
 import rx.Observer
@@ -387,7 +388,17 @@ val Class<*>.location: URL get() = protectionDomain.codeSource.location
 
 /** Convenience method to get the package name of a class literal. */
 val KClass<*>.packageName: String get() = java.packageName
-val Class<*>.packageName: String get() = requireNotNull(`package`?.name) { "$this not defined inside a package" }
+val Class<*>.packageName: String get() = requireNotNull(this.packageNameOrNull) { "$this not defined inside a package" }
+val Class<*>.packageNameOrNull: String? // This intentionally does not go via `package` as that code path is slow and contended and just ends up doing this.
+    get() {
+        val name = this.getName()
+        val i = name.lastIndexOf('.')
+        if (i != -1) {
+            return name.substring(0, i)
+        } else {
+            return null
+        }
+    }
 
 inline val Class<*>.isAbstractClass: Boolean get() = Modifier.isAbstract(modifiers)
 
@@ -403,8 +414,15 @@ inline val Member.isFinal: Boolean get() = Modifier.isFinal(modifiers)
 
 @DeleteForDJVM fun URL.toPath(): Path = toURI().toPath()
 
+val DEFAULT_HTTP_CONNECT_TIMEOUT = 30.seconds.toMillis()
+val DEFAULT_HTTP_READ_TIMEOUT = 30.seconds.toMillis()
+
 @DeleteForDJVM
-fun URL.openHttpConnection(): HttpURLConnection = openConnection() as HttpURLConnection
+fun URL.openHttpConnection(): HttpURLConnection = openConnection().also {
+    // The default values are 0 which means infinite timeout.
+    it.connectTimeout = DEFAULT_HTTP_CONNECT_TIMEOUT.toInt()
+    it.readTimeout = DEFAULT_HTTP_READ_TIMEOUT.toInt()
+} as HttpURLConnection
 
 @DeleteForDJVM
 fun URL.post(serializedData: OpaqueBytes, vararg properties: Pair<String, String>): ByteArray {
@@ -524,6 +542,7 @@ fun <E> MutableSet<E>.toSynchronised(): MutableSet<E> = Collections.synchronized
 /**
  * List implementation that applies the expensive [transform] function only when the element is accessed and caches calculated values.
  * Size is very cheap as it doesn't call [transform].
+ * Used internally by [net.corda.core.transactions.TraversableTransaction].
  */
 class LazyMappedList<T, U>(val originalList: List<T>, val transform: (T, Int) -> U) : AbstractList<U>() {
     private val partialResolvedList = MutableList<U?>(originalList.size) { null }
@@ -532,6 +551,15 @@ class LazyMappedList<T, U>(val originalList: List<T>, val transform: (T, Int) ->
         return partialResolvedList[index]
                 ?: transform(originalList[index], index).also { computed -> partialResolvedList[index] = computed }
     }
+    internal fun eager(onError: (TransactionDeserialisationException, Int) -> U?) {
+        for (i in 0 until size) {
+            try {
+                get(i)
+            } catch (ex: TransactionDeserialisationException) {
+                partialResolvedList[i] = onError(ex, i)
+            }
+        }
+    }
 }
 
 /**
@@ -540,6 +568,17 @@ class LazyMappedList<T, U>(val originalList: List<T>, val transform: (T, Int) ->
  */
 fun <T, U> List<T>.lazyMapped(transform: (T, Int) -> U): List<U> = LazyMappedList(this, transform)
 
+/**
+ * Iterate over a [LazyMappedList], forcing it to transform all of its elements immediately.
+ * This transformation is assumed to be "deserialisation". Does nothing for any other kind of [List].
+ * WARNING: Any changes made to the [LazyMappedList] contents are PERMANENT!
+ */
+fun <T> List<T>.eagerDeserialise(onError: (TransactionDeserialisationException, Int) -> T? = { ex, _ -> throw ex }) {
+    if (this is LazyMappedList<*, T>) {
+        eager(onError)
+    }
+}
+
 private const val MAX_SIZE = 100
 private val warnings = Collections.newSetFromMap(createSimpleCache<String, Boolean>(MAX_SIZE)).toSynchronised()
 

core/src/main/kotlin/net/corda/core/internal/JarSignatureCollector.kt

@@ -19,6 +19,11 @@ object JarSignatureCollector {
      */
     private val unsignableEntryName = "META-INF/(?:(?:.*[.](?:SF|DSA|RSA|EC)|SIG-.*)|INDEX\\.LIST)".toRegex()
 
+    /**
+     * @return if the [entry] [JarEntry] can be signed.
+     */
+    fun isNotSignable(entry: JarEntry): Boolean = entry.isDirectory || unsignableEntryName.matches(entry.name)
+
     /**
      * Returns an ordered list of every [PublicKey] which has signed every signable item in the given [JarInputStream].
      *
@@ -57,8 +62,7 @@ object JarSignatureCollector {
     private val JarInputStream.fileSignerSets: List<Pair<String, Set<CodeSigner>>> get() =
             entries.thatAreSignable.shreddedFrom(this).toFileSignerSet().toList()
 
-    private val Sequence<JarEntry>.thatAreSignable: Sequence<JarEntry> get() =
+    private val Sequence<JarEntry>.thatAreSignable: Sequence<JarEntry> get() = filterNot { isNotSignable(it) }
-            filterNot { entry -> entry.isDirectory || unsignableEntryName.matches(entry.name) }
 
     private fun Sequence<JarEntry>.shreddedFrom(jar: JarInputStream): Sequence<JarEntry> = map { entry ->
         val shredder = ByteArray(1024) // can't share or re-use this, as it's used to compute CRCs during shredding

core/src/main/kotlin/net/corda/core/internal/LazyPool.kt

@@ -62,7 +62,8 @@ class LazyPool<A>(
      */
     fun close(): Iterable<A> {
         lifeCycle.justTransition(State.FINISHED)
-        val elements = poolQueue.toList()
+        // Does not use kotlin toList() as it currently is not safe to use on concurrent data structures.
+        val elements = ArrayList(poolQueue)
         poolQueue.clear()
         return elements
     }

core/src/main/kotlin/net/corda/core/internal/LazyStickyPool.kt

@@ -25,7 +25,12 @@ class LazyStickyPool<A : Any>(
     private val boxes = Array(size) { InstanceBox<A>() }
 
     private fun toIndex(stickTo: Any): Int {
-        return Math.abs(stickTo.hashCode()) % boxes.size
+        return stickTo.hashCode().let { hashCode ->
+            when (hashCode) {
+                Int.MIN_VALUE -> 0
+                else -> Math.abs(hashCode) % boxes.size
+            }
+        }
     }
 
     fun borrow(stickTo: Any): A {

core/src/main/kotlin/net/corda/core/internal/ResolveTransactionsFlow.kt

@@ -64,8 +64,7 @@ class ResolveTransactionsFlow(txHashesArg: Set<SecureHash>,
             return sort.complete()
         }
     }
-
+    
-    @CordaSerializable
     class ExcessivelyLargeTransactionGraph : FlowException()
 
     // TODO: Figure out a more appropriate DOS limit here, 5000 is simply a very bad guess.

core/src/main/kotlin/net/corda/core/internal/StatePointerSearch.kt

@@ -43,7 +43,7 @@ class StatePointerSearch(val state: ContractState) {
         val fieldsWithObjects = fields.mapNotNull { field ->
             // Ignore classes which have not been loaded.
             // Assumption: all required state classes are already loaded.
-            val packageName = field.type.`package`?.name
+            val packageName = field.type.packageNameOrNull
             if (packageName == null) {
                 null
             } else {
@@ -72,7 +72,7 @@ class StatePointerSearch(val state: ContractState) {
             is StatePointer<*> -> statePointers.add(obj)
             is Iterable<*> -> handleIterable(obj)
             else -> {
-                val packageName = obj.javaClass.`package`.name
+                val packageName = obj.javaClass.packageNameOrNull ?: ""
                 val isBlackListed = blackListedPackages.any { packageName.startsWith(it) }
                 if (isBlackListed.not()) fieldQueue.addAllFields(obj)
             }

core/src/main/kotlin/net/corda/core/node/NetworkParameters.kt

@@ -98,15 +98,20 @@ data class NetworkParameters(
         require(noPackageOverlap(packageOwnership.keys)) { "Multiple packages added to the packageOwnership overlap." }
     }
 
-    fun copy(minimumPlatformVersion: Int,
+    /**
-             notaries: List<NotaryInfo>,
+     * This is to address backwards compatibility of the API, invariant to package ownership
-             maxMessageSize: Int,
+     * addresses bug CORDA-2769
-             maxTransactionSize: Int,
+     */
-             modifiedTime: Instant,
+    fun copy(minimumPlatformVersion: Int = this.minimumPlatformVersion,
-             epoch: Int,
+             notaries: List<NotaryInfo> = this.notaries,
-             whitelistedContractImplementations: Map<String, List<AttachmentId>>
+             maxMessageSize: Int = this.maxMessageSize,
+             maxTransactionSize: Int = this.maxTransactionSize,
+             modifiedTime: Instant = this.modifiedTime,
+             epoch: Int = this.epoch,
+             whitelistedContractImplementations: Map<String, List<AttachmentId>> = this.whitelistedContractImplementations,
+             eventHorizon: Duration = this.eventHorizon
     ): NetworkParameters {
-        return copy(
+        return NetworkParameters(
                 minimumPlatformVersion = minimumPlatformVersion,
                 notaries = notaries,
                 maxMessageSize = maxMessageSize,
@@ -114,20 +119,24 @@ data class NetworkParameters(
                 modifiedTime = modifiedTime,
                 epoch = epoch,
                 whitelistedContractImplementations = whitelistedContractImplementations,
-                eventHorizon = eventHorizon
+                eventHorizon = eventHorizon,
+                packageOwnership = packageOwnership
         )
     }
 
-    fun copy(minimumPlatformVersion: Int,
+    /**
-             notaries: List<NotaryInfo>,
+     * This is to address backwards compatibility of the API, invariant to package ownership
-             maxMessageSize: Int,
+     * addresses bug CORDA-2769
-             maxTransactionSize: Int,
+     */
-             modifiedTime: Instant,
+    fun copy(minimumPlatformVersion: Int = this.minimumPlatformVersion,
-             epoch: Int,
+             notaries: List<NotaryInfo> = this.notaries,
-             whitelistedContractImplementations: Map<String, List<AttachmentId>>,
+             maxMessageSize: Int = this.maxMessageSize,
-             eventHorizon: Duration
+             maxTransactionSize: Int = this.maxTransactionSize,
+             modifiedTime: Instant = this.modifiedTime,
+             epoch: Int = this.epoch,
+             whitelistedContractImplementations: Map<String, List<AttachmentId>> = this.whitelistedContractImplementations
     ): NetworkParameters {
-        return copy(
+        return NetworkParameters(
                 minimumPlatformVersion = minimumPlatformVersion,
                 notaries = notaries,
                 maxMessageSize = maxMessageSize,
@@ -135,7 +144,8 @@ data class NetworkParameters(
                 modifiedTime = modifiedTime,
                 epoch = epoch,
                 whitelistedContractImplementations = whitelistedContractImplementations,
-                eventHorizon = eventHorizon
+                eventHorizon = eventHorizon,
+                packageOwnership = packageOwnership
         )
     }
 

core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteria.kt

@@ -11,6 +11,7 @@ import net.corda.core.identity.AbstractParty
 import net.corda.core.node.services.Vault
 import net.corda.core.schemas.StatePersistable
 import net.corda.core.serialization.CordaSerializable
+import net.corda.core.serialization.DeprecatedConstructorForDeserialization
 import net.corda.core.utilities.OpaqueBytes
 import java.security.PublicKey
 import java.time.Instant
@@ -102,6 +103,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
     ) : CommonQueryCriteria() {
         // V3 c'tors
         // These have to be manually specified as @JvmOverloads for some reason causes declaration clashes
+        @DeprecatedConstructorForDeserialization(version = 6)
         constructor(
                 status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED,
                 contractStateTypes: Set<Class<out ContractState>>? = null,
@@ -110,14 +112,19 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
                 softLockingCondition: SoftLockingCondition? = null,
                 timeCondition: TimeCondition? = null
         ) : this(status, contractStateTypes, stateRefs, notary, softLockingCondition, timeCondition, participants = null)
+        @DeprecatedConstructorForDeserialization(version = 1)
         constructor(status: Vault.StateStatus) : this(status, participants = null)
+        @DeprecatedConstructorForDeserialization(version = 2)
         constructor(status: Vault.StateStatus, contractStateTypes: Set<Class<out ContractState>>?) : this(status, contractStateTypes, participants = null)
+        @DeprecatedConstructorForDeserialization(version = 3)
         constructor(status: Vault.StateStatus, contractStateTypes: Set<Class<out ContractState>>?, stateRefs: List<StateRef>?) : this(
                 status, contractStateTypes, stateRefs, participants = null
         )
+        @DeprecatedConstructorForDeserialization(version = 4)
         constructor(status: Vault.StateStatus, contractStateTypes: Set<Class<out ContractState>>?, stateRefs: List<StateRef>?, notary: List<AbstractParty>?) : this(
                 status, contractStateTypes, stateRefs, notary, participants = null
         )
+        @DeprecatedConstructorForDeserialization(version = 5)
         constructor(status: Vault.StateStatus, contractStateTypes: Set<Class<out ContractState>>?, stateRefs: List<StateRef>?, notary: List<AbstractParty>?, softLockingCondition: SoftLockingCondition?) : this(
                 status, contractStateTypes, stateRefs, notary, softLockingCondition, participants = null
         )
@@ -174,6 +181,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
     ) : CommonQueryCriteria() {
         // V3 c'tor
         @JvmOverloads
+        @DeprecatedConstructorForDeserialization(version = 2)
         constructor(
                 participants: List<AbstractParty>? = null,
                 uuid: List<UUID>? = null,
@@ -182,6 +190,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
                 contractStateTypes: Set<Class<out ContractState>>? = null
         ) : this(participants, uuid, externalId, status, contractStateTypes, Vault.RelevancyStatus.ALL)
 
+        @DeprecatedConstructorForDeserialization(version = 3)
         constructor(
                 participants: List<AbstractParty>? = null,
                 linearId: List<UniqueIdentifier>? = null,
@@ -191,6 +200,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
         ) : this(participants, linearId?.map { it.id }, linearId?.mapNotNull { it.externalId }, status, contractStateTypes, relevancyStatus)
 
         // V3 c'tor
+        @DeprecatedConstructorForDeserialization(version = 1)
         constructor(
                 participants: List<AbstractParty>? = null,
                 linearId: List<UniqueIdentifier>? = null,
@@ -264,6 +274,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
             override val relevancyStatus: Vault.RelevancyStatus
     ) : CommonQueryCriteria() {
         @JvmOverloads
+        @DeprecatedConstructorForDeserialization(version = 1)
         constructor(
                 participants: List<AbstractParty>? = null,
                 owner: List<AbstractParty>? = null,
@@ -325,6 +336,7 @@ sealed class QueryCriteria : GenericQueryCriteria<QueryCriteria, IQueryCriteriaP
             override val relevancyStatus: Vault.RelevancyStatus = Vault.RelevancyStatus.ALL
     ) : CommonQueryCriteria() {
         @JvmOverloads
+        @DeprecatedConstructorForDeserialization(version = 1)
         constructor(
                 expression: CriteriaExpression<L, Boolean>,
                 status: Vault.StateStatus = Vault.StateStatus.UNCONSUMED,
@@ -381,10 +393,13 @@ sealed class AttachmentQueryCriteria : GenericQueryCriteria<AttachmentQueryCrite
                                         val isSignedCondition: ColumnPredicate<Boolean>? = null,
                                         val versionCondition: ColumnPredicate<Int>? = null) : AttachmentQueryCriteria() {
         // V3 c'tors
+        @DeprecatedConstructorForDeserialization(version = 3)
         constructor(uploaderCondition: ColumnPredicate<String>? = null,
                     filenameCondition: ColumnPredicate<String>? = null,
                     uploadDateCondition: ColumnPredicate<Instant>? = null) : this(uploaderCondition, filenameCondition, uploadDateCondition, null)
+        @DeprecatedConstructorForDeserialization(version = 1)
         constructor(uploaderCondition: ColumnPredicate<String>?) : this(uploaderCondition, null)
+        @DeprecatedConstructorForDeserialization(version = 2)
         constructor(uploaderCondition: ColumnPredicate<String>?, filenameCondition: ColumnPredicate<String>?) : this(uploaderCondition, filenameCondition, null)
 
         override fun visit(parser: AttachmentsQueryCriteriaParser): Collection<Predicate> {

core/src/main/kotlin/net/corda/core/serialization/SerializationAPI.kt

@@ -1,8 +1,6 @@
 @file:KeepForDJVM
 package net.corda.core.serialization
 
-import co.paralleluniverse.io.serialization.Serialization
-import net.corda.core.CordaInternal
 import net.corda.core.DeleteForDJVM
 import net.corda.core.DoNotImplement
 import net.corda.core.KeepForDJVM
@@ -12,6 +10,7 @@ import net.corda.core.serialization.internal.effectiveSerializationEnv
 import net.corda.core.utilities.ByteSequence
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.sequence
+import java.io.NotSerializableException
 import java.sql.Blob
 
 data class ObjectWithCompatibleContext<out T : Any>(val obj: T, val context: SerializationContext)
@@ -152,7 +151,15 @@ interface SerializationContext {
      */
     val lenientCarpenterEnabled: Boolean
     /**
-     * If true the serialization evolver will fail if the binary to be deserialized contains more fields then the current object from the classpath.
+     * If true, deserialization calls using this context will not fallback to using the Class Carpenter to attempt
+     * to construct classes present in the schema but not on the current classpath.
+     *
+     * The default is false.
+     */
+    val carpenterDisabled: Boolean
+    /**
+     * If true the serialization evolver will fail if the binary to be deserialized contains more fields then the current object from
+     * the classpath.
      *
      * The default is false.
      */
@@ -182,6 +189,12 @@ interface SerializationContext {
      */
     fun withLenientCarpenter(): SerializationContext
 
+    /**
+     * Returns a copy of the current context with carpentry of unknown classes disabled. On encountering
+     * such a class during deserialization the Serialization framework will throw a [NotSerializableException].
+     */
+    fun withoutCarpenter() : SerializationContext
+
     /**
      * Return a new context based on this one but with a strict evolution.
      * @see preventDataLoss
@@ -317,6 +330,7 @@ fun <T : Any> T.serialize(serializationFactory: SerializationFactory = Serializa
  */
 @Suppress("unused")
 @KeepForDJVM
+@CordaSerializable
 class SerializedBytes<T : Any>(bytes: ByteArray) : OpaqueBytes(bytes) {
     companion object {
         /**

core/src/main/kotlin/net/corda/core/serialization/internal/AttachmentsClassLoader.kt

@@ -327,6 +327,7 @@ object AttachmentsClassLoaderBuilder {
                     .withClassLoader(transactionClassLoader)
                     .withWhitelist(whitelistedClasses)
                     .withCustomSerializers(serializers)
+                    .withoutCarpenter()
         }
 
         // Deserialize all relevant classes in the transaction classloader.

core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt

@@ -28,12 +28,22 @@ import java.util.function.Predicate
  * - Deserialising the output states.
  *
  * All the above refer to inputs using a (txhash, output index) pair.
+ *
+ * Usage notes:
+ *
+ * [LedgerTransaction] is an abstraction that is meant to be used during the transaction verification stage.
+ * It needs full access to input states that might be in transactions that are encrypted and unavailable for code running outside the secure enclave.
+ * Also, it might need to deserialize states with code that might not be available on the classpath.
+ *
+ * Because of this, trying to create or use a [LedgerTransaction] for any other purpose then transaction verification can result in unexpected exceptions,
+ * which need de be handled.
+ *
+ * [LedgerTransaction]s should never be instantiated directly from client code, but rather via WireTransaction.toLedgerTransaction
  */
 @KeepForDJVM
 @CordaSerializable
 class LedgerTransaction
 @ConstructorForDeserialization
-// LedgerTransaction is not meant to be created directly from client code, but rather via WireTransaction.toLedgerTransaction
 private constructor(
         // DOCSTART 1
         /** The resolved input states which will be consumed/invalidated by the execution of this transaction. */
@@ -67,7 +77,6 @@ private constructor(
     private var serializedReferences: List<SerializedStateAndRef>? = null
 
     init {
-        checkBaseInvariants()
         if (timeWindow != null) check(notary != null) { "Transactions with time-windows must be notarised" }
         checkNotaryWhitelisted()
     }
@@ -133,6 +142,9 @@ private constructor(
         // Switch thread local deserialization context to using a cached attachments classloader. This classloader enforces various rules
         // like no-overlap, package namespace ownership and (in future) deterministic Java.
         return AttachmentsClassLoaderBuilder.withAttachmentsClassloaderContext(this.attachments + extraAttachments, getParamsWithGoo(), id) { transactionClassLoader ->
+            // Create a copy of the outer LedgerTransaction which deserializes all fields inside the [transactionClassLoader].
+            // Only the copy will be used for verification, and the outer shell will be discarded.
+            // This artifice is required to preserve backwards compatibility.
             Verifier(createLtxForVerification(), transactionClassLoader)
         }
     }
@@ -163,12 +175,17 @@ private constructor(
         return FlowLogic.currentTopLevel?.serviceHub?.networkParameters
     }
 
+    /**
+     * Create the [LedgerTransaction] instance that will be used by contract verification.
+     *
+     * This method needs to run in the special transaction attachments classloader context.
+     */
     private fun createLtxForVerification(): LedgerTransaction {
         val serializedInputs = this.serializedInputs
         val serializedReferences = this.serializedReferences
         val componentGroups = this.componentGroups
 
-        return if (serializedInputs != null && serializedReferences != null && componentGroups != null) {
+        val transaction= if (serializedInputs != null && serializedReferences != null && componentGroups != null) {
             // Deserialize all relevant classes in the transaction classloader.
             val deserializedInputs = serializedInputs.map { it.toStateAndRef() }
             val deserializedReferences = serializedReferences.map { it.toStateAndRef() }
@@ -198,6 +215,12 @@ private constructor(
                     "The result of the verify method might not be accurate.")
             this
         }
+
+        // This check accesses input states and must be run in this context.
+        // It must run on the instance that is verified, not on the outer LedgerTransaction shell.
+        transaction.checkBaseInvariants()
+
+        return transaction
     }
 
     /**

core/src/main/kotlin/net/corda/core/transactions/SignedTransaction.kt

@@ -7,6 +7,7 @@ import net.corda.core.KeepForDJVM
 import net.corda.core.contracts.*
 import net.corda.core.crypto.*
 import net.corda.core.identity.Party
+import net.corda.core.internal.TransactionDeserialisationException
 import net.corda.core.internal.TransactionVerifierServiceInternal
 import net.corda.core.internal.VisibleForTesting
 import net.corda.core.internal.internalFindTrustedAttachmentForClass
@@ -18,6 +19,7 @@ import net.corda.core.serialization.deserialize
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.contextLogger
 import net.corda.core.utilities.getOrThrow
+import java.io.NotSerializableException
 import java.security.KeyPair
 import java.security.PublicKey
 import java.security.SignatureException
@@ -226,27 +228,53 @@ data class SignedTransaction(val txBits: SerializedBytes<CoreTransaction>,
             // TODO: allow non-blocking verification.
             services.transactionVerifierService.verify(ltx).getOrThrow()
         } catch (e: NoClassDefFoundError) {
-            // Transactions created before Corda 4 can be missing dependencies on other cordapps.
+            if (e.message != null) {
-            // This code attempts to find the missing dependency in the attachment storage among the trusted contract attachments.
+                verifyWithExtraDependency(e.message!!, ltx, services, e)
-            // When it finds one, it instructs the verifier to use it to create the transaction classloader.
+            } else {
-            // TODO - add check that transaction was created before Corda 4.
+                throw e
-
+            }
-            // TODO - should this be a [TransactionVerificationException]?
+        } catch (e: NotSerializableException) {
-            val missingClass = requireNotNull(e.message) { "Transaction $ltx is incorrectly formed." }
+            if (e.cause is ClassNotFoundException && e.cause!!.message != null) {
-
+                verifyWithExtraDependency(e.cause!!.message!!.replace(".", "/"), ltx, services, e)
-            val attachment = requireNotNull(services.attachments.internalFindTrustedAttachmentForClass(missingClass)) {
+            } else {
-                "Transaction $ltx is incorrectly formed. Could not find local dependency for class: $missingClass."
+                throw e
+            }
+        } catch (e: TransactionDeserialisationException) {
+            if (e.cause is NotSerializableException && e.cause.cause is ClassNotFoundException && e.cause.cause!!.message != null) {
+                verifyWithExtraDependency(e.cause.cause!!.message!!.replace(".", "/"), ltx, services, e)
+            } else {
+                throw e
             }
-
-            log.warn("""Detected that transaction ${this.id} does not contain all cordapp dependencies.
-                |This may be the result of a bug in a previous version of Corda.
-                |Attempting to verify using the additional dependency: $attachment.
-                |Please check with the originator that this is a valid transaction.""".trimMargin())
-
-            (services.transactionVerifierService as TransactionVerifierServiceInternal).verify(ltx, listOf(attachment)).getOrThrow()
         }
     }
 
+    // Transactions created before Corda 4 can be missing dependencies on other CorDapps.
+    // This code attempts to find the missing dependency in the attachment storage among the trusted attachments.
+    // When it finds one, it instructs the verifier to use it to create the transaction classloader.
+    private fun verifyWithExtraDependency(missingClass: String, ltx: LedgerTransaction, services: ServiceHub, exception: Throwable) {
+        // If that transaction was created with and after Corda 4 then just fail.
+        // The lenient dependency verification is only supported for Corda 3 transactions.
+        // To detect if the transaction was created before Corda 4 we check if the transaction has the NetworkParameters component group.
+        if (this.networkParametersHash != null) {
+            throw exception
+        }
+
+        val attachment = requireNotNull(services.attachments.internalFindTrustedAttachmentForClass(missingClass)) {
+            """Transaction $ltx is incorrectly formed. Most likely it was created during version 3 of Corda when the verification logic was more lenient.
+                |Attempted to find local dependency for class: $missingClass, but could not find one.
+                |If you wish to verify this transaction, please contact the originator of the transaction and install the provided missing JAR.
+                |You can install it using the RPC command: `uploadAttachment` without restarting the node.
+                |""".trimMargin()
+        }
+
+        log.warn("""Detected that transaction ${this.id} does not contain all cordapp dependencies.
+                    |This may be the result of a bug in a previous version of Corda.
+                    |Attempting to verify using the additional trusted dependency: $attachment for class $missingClass.
+                    |Please check with the originator that this is a valid transaction.""".trimMargin())
+
+        (services.transactionVerifierService as TransactionVerifierServiceInternal).verify(ltx, listOf(attachment)).getOrThrow()
+    }
+
     /**
      * Resolves the underlying base transaction and then returns it, handling any special case transactions such as
      * [NotaryChangeWireTransaction].
@@ -319,7 +347,6 @@ data class SignedTransaction(val txBits: SerializedBytes<CoreTransaction>,
     }
 
     @KeepForDJVM
-    @CordaSerializable
     class SignaturesMissingException(val missing: Set<PublicKey>, val descriptions: List<String>, override val id: SecureHash)
         : NamedByHash, SignatureException(missingSignatureMsg(missing, descriptions, id)), CordaThrowable by CordaException(missingSignatureMsg(missing, descriptions, id))
 

core/src/main/kotlin/net/corda/core/transactions/TransactionBuilder.kt

@@ -7,8 +7,6 @@ import net.corda.core.contracts.*
 import net.corda.core.crypto.*
 import net.corda.core.identity.Party
 import net.corda.core.internal.*
-import net.corda.core.internal.cordapp.CordappImpl.Companion.DEFAULT_CORDAPP_VERSION
-import net.corda.core.internal.cordapp.CordappResolver
 import net.corda.core.node.NetworkParameters
 import net.corda.core.node.ServiceHub
 import net.corda.core.node.ServicesForResolution
@@ -18,6 +16,7 @@ import net.corda.core.node.services.KeyManagementService
 import net.corda.core.serialization.SerializationContext
 import net.corda.core.serialization.SerializationFactory
 import net.corda.core.utilities.contextLogger
+import java.io.NotSerializableException
 import java.security.PublicKey
 import java.time.Duration
 import java.time.Instant
@@ -172,21 +171,25 @@ open class TransactionBuilder(
         try {
             wireTx.toLedgerTransaction(services).verify()
         } catch (e: NoClassDefFoundError) {
-            val missingClass = e.message
+            val missingClass = e.message ?: throw e
-            requireNotNull(missingClass) { "Transaction is incorrectly formed." }
+            addMissingAttachment(missingClass, services)
-
-            val attachment = services.attachments.internalFindTrustedAttachmentForClass(missingClass!!)
-                    ?: throw IllegalArgumentException("Attempted to find dependent attachment for class $missingClass, but could not find a suitable candidate.")
-
-            log.warnOnce("""The transaction currently built is missing an attachment for class: $missingClass.
-                    Automatically attaching contract dependency $attachment.
-                    It is strongly recommended to check that this is the desired attachment, and to manually add it to the transaction builder.
-                """.trimIndent())
-
-            addAttachment(attachment.id)
             return true
-            // Ignore these exceptions as they will break unit tests.
+        } catch (e: TransactionDeserialisationException) {
-            //  The point here is only to detect missing dependencies. The other exceptions are irrelevant.
+            if (e.cause is NotSerializableException && e.cause.cause is ClassNotFoundException) {
+                val missingClass = e.cause.cause!!.message ?: throw e
+                addMissingAttachment(missingClass.replace(".", "/"), services)
+                return true
+            }
+            return false
+        } catch (e: NotSerializableException) {
+            if (e.cause is ClassNotFoundException) {
+                val missingClass = e.cause!!.message ?: throw e
+                addMissingAttachment(missingClass.replace(".", "/"), services)
+                return true
+            }
+            return false
+        // Ignore these exceptions as they will break unit tests.
+        // The point here is only to detect missing dependencies. The other exceptions are irrelevant.
         } catch (tve: TransactionVerificationException) {
         } catch (tre: TransactionResolutionException) {
         } catch (ise: IllegalStateException) {
@@ -195,6 +198,21 @@ open class TransactionBuilder(
         return false
     }
 
+    private fun addMissingAttachment(missingClass: String, services: ServicesForResolution) {
+        val attachment = services.attachments.internalFindTrustedAttachmentForClass(missingClass)
+                ?: throw IllegalArgumentException("""The transaction currently built is missing an attachment for class: $missingClass.
+                        Attempted to find a suitable attachment but could not find any in the storage.
+                        Please contact the developer of the CorDapp for further instructions.
+                    """.trimIndent())
+
+        log.warnOnce("""The transaction currently built is missing an attachment for class: $missingClass.
+                        Automatically attaching contract dependency $attachment.
+                        Please contact the developer of the CorDapp and install the latest version, as this approach might be insecure.
+                    """.trimIndent())
+
+        addAttachment(attachment.id)
+    }
+
     /**
      * This method is responsible for selecting the contract versions to be used for the current transaction and resolve the output state [AutomaticPlaceholderConstraint]s.
      * The contract attachments are used to create a deterministic Classloader to deserialise the transaction and to run the contract verification.
@@ -653,15 +671,8 @@ with @BelongsToContract, or supply an explicit contract parameter to addOutputSt
     /** Returns an immutable list of output [TransactionState]s. */
     fun outputStates(): List<TransactionState<*>> = ArrayList(outputs)
 
-    /** Returns an immutable list of [Command]s, grouping by [CommandData] and joining signers (from v4, v3 and below return all commands with duplicates for different signers). */
+    /** Returns an immutable list of [Command]s. */
-    fun commands(): List<Command<*>> {
+    fun commands(): List<Command<*>> = ArrayList(commands)
-        return if (CordappResolver.currentTargetVersion >= CORDA_VERSION_THAT_INTRODUCED_FLATTENED_COMMANDS) {
-            commands.groupBy { cmd -> cmd.value }
-                    .entries.map { (data, cmds) -> Command(data, cmds.flatMap(Command<*>::signers).toSet().toList()) }
-        } else {
-            ArrayList(commands)
-        }
-    }
 
     /**
      * Sign the built transaction and return it. This is an internal function for use by the service hub, please use

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -9,7 +9,6 @@ import net.corda.core.contracts.ComponentGroupEnum.OUTPUTS_GROUP
 import net.corda.core.crypto.*
 import net.corda.core.identity.Party
 import net.corda.core.internal.*
-import net.corda.core.internal.cordapp.CordappImpl.Companion.DEFAULT_CORDAPP_VERSION
 import net.corda.core.node.NetworkParameters
 import net.corda.core.node.ServiceHub
 import net.corda.core.node.ServicesForResolution

core/src/main/kotlin/net/corda/core/utilities/ByteArrays.kt

@@ -20,7 +20,6 @@ import javax.xml.bind.DatatypeConverter
  * @property offset The start position of the sequence within the byte array.
  * @property size The number of bytes this sequence represents.
  */
-@CordaSerializable
 @KeepForDJVM
 sealed class ByteSequence(private val _bytes: ByteArray, val offset: Int, val size: Int) : Comparable<ByteSequence> {
     /**
@@ -145,6 +144,7 @@ sealed class ByteSequence(private val _bytes: ByteArray, val offset: Int, val si
  * functionality to Java, but it won't arrive for a few years yet!
  */
 @KeepForDJVM
+@CordaSerializable
 open class OpaqueBytes(bytes: ByteArray) : ByteSequence(bytes, 0, bytes.size) {
     companion object {
         /**

core/src/main/kotlin/net/corda/core/utilities/KotlinUtils.kt

@@ -4,15 +4,12 @@ package net.corda.core.utilities
 
 import net.corda.core.DeleteForDJVM
 import net.corda.core.KeepForDJVM
-import net.corda.core.internal.LazyMappedList
 import net.corda.core.internal.concurrent.get
-import net.corda.core.internal.createSimpleCache
 import net.corda.core.internal.uncheckedCast
 import net.corda.core.serialization.CordaSerializable
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.time.Duration
-import java.util.*
 import java.util.concurrent.ExecutionException
 import java.util.concurrent.Future
 import kotlin.reflect.KProperty

core/src/main/kotlin/net/corda/core/utilities/ProgressTracker.kt

@@ -5,7 +5,6 @@ import net.corda.core.internal.STRUCTURAL_STEP_PREFIX
 import net.corda.core.serialization.CordaSerializable
 import rx.Observable
 import rx.Subscription
-import rx.subjects.PublishSubject
 import rx.subjects.ReplaySubject
 import java.util.*
 
@@ -51,7 +50,9 @@ class ProgressTracker(vararg inputSteps: Step) {
         }
     }
 
-    /** The superclass of all step objects. */
+    /**
+     * The superclass of all step objects.
+     */
     @CordaSerializable
     open class Step(open val label: String) {
         open val changes: Observable<Change> get() = Observable.empty()
@@ -85,16 +86,22 @@ class ProgressTracker(vararg inputSteps: Step) {
 
     private val childProgressTrackers = mutableMapOf<Step, Child>()
 
-    /** The steps in this tracker, same as the steps passed to the constructor but with UNSTARTED and DONE inserted. */
+    /**
+     * The steps in this tracker, same as the steps passed to the constructor but with UNSTARTED and DONE inserted.
+     */
     val steps = arrayOf(UNSTARTED, STARTING, *inputSteps, DONE)
 
     private var _allStepsCache: List<Pair<Int, Step>> = _allSteps()
 
     // This field won't be serialized.
-    private val _changes by transient { PublishSubject.create<Change>() }
+    private val _changes by transient { ReplaySubject.create<Change>() }
-    private val _stepsTreeChanges by transient { PublishSubject.create<List<Pair<Int, String>>>() }
+    private val _stepsTreeChanges by transient { ReplaySubject.create<List<Pair<Int, String>>>() }
     private val _stepsTreeIndexChanges by transient { ReplaySubject.create<Int>() }
 
+    /**
+     * Reading returns the value of steps[stepIndex], writing moves the position of the current tracker. Once moved to
+     * the [DONE] state, this tracker is finished and the current step cannot be moved again.
+     */
     var currentStep: Step
         get() = steps[stepIndex]
         set(value) {
@@ -135,6 +142,9 @@ class ProgressTracker(vararg inputSteps: Step) {
         steps.forEach {
             configureChildTrackerForStep(it)
         }
+        // Immediately update the step tree observable to ensure the first update the client receives is the initial state of the progress
+        // tracker.
+        _stepsTreeChanges.onNext(allStepsLabels)
         this.currentStep = UNSTARTED
     }
 
@@ -145,13 +155,17 @@ class ProgressTracker(vararg inputSteps: Step) {
         }
     }
 
-    /** The zero-based index of the current step in the [steps] array (i.e. with UNSTARTED and DONE) */
+    /**
+     * The zero-based index of the current step in the [steps] array (i.e. with UNSTARTED and DONE)
+     */
     var stepIndex: Int = 0
         private set(value) {
             field = value
         }
 
-    /** The zero-bases index of the current step in a [allStepsLabels] list */
+    /**
+     * The zero-bases index of the current step in a [allStepsLabels] list
+     */
     var stepsTreeIndex: Int = -1
         private set(value) {
             if (value != field) {
@@ -161,26 +175,12 @@ class ProgressTracker(vararg inputSteps: Step) {
         }
 
     /**
-     * Reading returns the value of steps[stepIndex], writing moves the position of the current tracker. Once moved to
+     * Returns the current step, descending into children to find the deepest step we are up to.
-     * the [DONE] state, this tracker is finished and the current step cannot be moved again.
      */
-
+    @Suppress("unused")
-    /** Returns the current step, descending into children to find the deepest step we are up to. */
     val currentStepRecursive: Step
         get() = getChildProgressTracker(currentStep)?.currentStepRecursive ?: currentStep
 
-    /** Returns the current step, descending into children to find the deepest started step we are up to. */
-    private val currentStartedStepRecursive: Step
-        get() {
-            val step = getChildProgressTracker(currentStep)?.currentStartedStepRecursive ?: currentStep
-            return if (step == UNSTARTED) currentStep else step
-        }
-
-    private fun currentStepRecursiveWithoutUnstarted(): Step {
-        val stepRecursive = getChildProgressTracker(currentStep)?.currentStartedStepRecursive
-        return if (stepRecursive == null || stepRecursive == UNSTARTED) currentStep else stepRecursive
-    }
-
     fun getChildProgressTracker(step: Step): ProgressTracker? = childProgressTrackers[step]?.tracker
 
     fun setChildProgressTracker(step: ProgressTracker.Step, childProgressTracker: ProgressTracker) {
@@ -214,12 +214,17 @@ class ProgressTracker(vararg inputSteps: Step) {
         _stepsTreeChanges.onError(error)
     }
 
-    /** The parent of this tracker: set automatically by the parent when a tracker is added as a child */
+    /**
+     * The parent of this tracker: set automatically by the parent when a tracker is added as a child
+     */
     var parent: ProgressTracker? = null
         private set
 
-    /** Walks up the tree to find the top level tracker. If this is the top level tracker, returns 'this' */
+    /**
-    @Suppress("unused") // TODO: Review by EOY2016 if this property is useful anywhere.
+     * Walks up the tree to find the top level tracker. If this is the top level tracker, returns 'this'.
+     * Required for API compatibility.
+     */
+    @Suppress("unused")
     val topLevelTracker: ProgressTracker
         get() {
             var cursor: ProgressTracker = this
@@ -234,9 +239,21 @@ class ProgressTracker(vararg inputSteps: Step) {
         recalculateStepsTreeIndex()
     }
 
+    private fun getStepIndexAtLevel(): Int {
+        // This gets the index of the current step in the context of this progress tracker, so it will always be at the top level in
+        // the allStepsCache.
+        val index = _allStepsCache.indexOf(Pair(0, currentStep))
+        return if (index >= 0) index else 0
+    }
+
+    private fun getCurrentStepTreeIndex(): Int {
+        val indexAtLevel = getStepIndexAtLevel()
+        val additionalIndex = getChildProgressTracker(currentStep)?.getCurrentStepTreeIndex() ?: 0
+        return indexAtLevel + additionalIndex
+    }
+
     private fun recalculateStepsTreeIndex() {
-        val step = currentStepRecursiveWithoutUnstarted()
+        stepsTreeIndex = getCurrentStepTreeIndex()
-        stepsTreeIndex = _allStepsCache.indexOfFirst { it.second == step }
     }
 
     private fun _allSteps(level: Int = 0): List<Pair<Int, Step>> {
@@ -291,7 +308,9 @@ class ProgressTracker(vararg inputSteps: Step) {
      */
     val stepsTreeIndexChanges: Observable<Int> get() = _stepsTreeIndexChanges
 
-    /** Returns true if the progress tracker has ended, either by reaching the [DONE] step or prematurely with an error */
+    /**
+     * Returns true if the progress tracker has ended, either by reaching the [DONE] step or prematurely with an error
+     */
     val hasEnded: Boolean get() = _changes.hasCompleted() || _changes.hasThrowable()
 }
 // TODO: Expose the concept of errors.

core/src/smoke-test/kotlin/net/corda/core/NodeVersioningTest.kt

@@ -21,7 +21,7 @@ import kotlin.streams.toList
 
 class NodeVersioningTest {
     private companion object {
-        val user = User("user1", "test", permissions = setOf("ALL"))
+        val superUser = User("superUser", "test", permissions = setOf("ALL"))
         val port = AtomicInteger(15100)
     }
 
@@ -33,7 +33,7 @@ class NodeVersioningTest {
             rpcPort = port.andIncrement,
             rpcAdminPort = port.andIncrement,
             isNotary = true,
-            users = listOf(user)
+            users = listOf(superUser)
     )
 
     private val aliceConfig = NodeConfig(
@@ -42,7 +42,7 @@ class NodeVersioningTest {
             rpcPort = port.andIncrement,
             rpcAdminPort = port.andIncrement,
             isNotary = false,
-            users = listOf(user)
+            users = listOf(superUser)
     )
 
     private lateinit var notary: NodeProcess
@@ -73,7 +73,7 @@ class NodeVersioningTest {
         selfCordapp.copyToDirectory(cordappsDir)
 
         factory.create(aliceConfig).use { alice ->
-            alice.connect().use {
+            alice.connect(superUser).use {
                 val rpc = it.proxy
                 assertThat(rpc.protocolVersion).isEqualTo(PLATFORM_VERSION)
                 assertThat(rpc.nodeInfo().platformVersion).isEqualTo(PLATFORM_VERSION)

core/src/smoke-test/kotlin/net/corda/core/cordapp/CordappSmokeTest.kt

@@ -38,7 +38,7 @@ import kotlin.streams.toList
 
 class CordappSmokeTest {
     private companion object {
-        val user = User("user1", "test", permissions = setOf("ALL"))
+        val superUser = User("superUser", "test", permissions = setOf("ALL"))
         val port = AtomicInteger(15100)
     }
 
@@ -50,7 +50,7 @@ class CordappSmokeTest {
             rpcPort = port.andIncrement,
             rpcAdminPort = port.andIncrement,
             isNotary = true,
-            users = listOf(user)
+            users = listOf(superUser)
     )
 
     private val aliceConfig = NodeConfig(
@@ -59,7 +59,7 @@ class CordappSmokeTest {
             rpcPort = port.andIncrement,
             rpcAdminPort = port.andIncrement,
             isNotary = false,
-            users = listOf(user)
+            users = listOf(superUser)
     )
 
     private lateinit var notary: NodeProcess
@@ -92,7 +92,7 @@ class CordappSmokeTest {
         createDummyNodeInfo(additionalNodeInfoDir)
 
         factory.create(aliceConfig).use { alice ->
-            alice.connect().use { connectionToAlice ->
+            alice.connect(superUser).use { connectionToAlice ->
                 val aliceIdentity = connectionToAlice.proxy.nodeInfo().legalIdentitiesAndCerts.first().party
                 val future = connectionToAlice.proxy.startFlow(::GatherContextsFlow, aliceIdentity).returnValue
                 val (sessionInitContext, sessionConfirmContext) = future.getOrThrow()

core/src/test/kotlin/net/corda/core/contracts/TransactionVerificationExceptionSerialisationTests.kt

@@ -8,6 +8,7 @@ import net.corda.serialization.internal.amqp.DeserializationInput
 import net.corda.serialization.internal.amqp.SerializationOutput
 import net.corda.serialization.internal.amqp.SerializerFactoryBuilder
 import net.corda.serialization.internal.amqp.custom.PublicKeySerializer
+import net.corda.serialization.internal.amqp.custom.ThrowableSerializer
 import net.corda.testing.core.DUMMY_BANK_A_NAME
 import net.corda.testing.core.DUMMY_NOTARY_NAME
 import net.corda.testing.core.TestIdentity
@@ -18,11 +19,12 @@ class TransactionVerificationExceptionSerialisationTests {
     private fun defaultFactory() = SerializerFactoryBuilder.build(
             AllWhitelist,
             ClassLoader.getSystemClassLoader()
-    )
+    ).apply { register(ThrowableSerializer(this)) }
 
     private val context get() = AMQP_RPC_CLIENT_CONTEXT
 
     private val txid = SecureHash.allOnesHash
+    private val attachmentHash = SecureHash.allOnesHash
     private val factory = defaultFactory()
 
     @Test
@@ -52,7 +54,7 @@ class TransactionVerificationExceptionSerialisationTests {
                 context)
 
         assertEquals(exception.message, exception2.message)
-        assertEquals(exception.cause?.message, exception2.cause?.message)
+        assertEquals("java.lang.Throwable: ${exception.cause?.message}", exception2.cause?.message)
         assertEquals(exception.txId, exception2.txId)
     }
 
@@ -89,7 +91,7 @@ class TransactionVerificationExceptionSerialisationTests {
                 context)
 
         assertEquals(exception.message, exception2.message)
-        assertEquals(exception.cause?.message, exception2.cause?.message)
+        assertEquals("java.lang.Throwable: ${exception.cause?.message}", exception2.cause?.message)
         assertEquals(exception.txId, exception2.txId)
     }
 
@@ -122,4 +124,55 @@ class TransactionVerificationExceptionSerialisationTests {
         assertEquals(exception.cause?.message, exception2.cause?.message)
         assertEquals(exception.txId, exception2.txId)
     }
+
+    @Test
+    fun overlappingAttachmentsExceptionTest() {
+        val exc = TransactionVerificationException.OverlappingAttachmentsException(txid, "foo/bar/baz")
+        val exc2 = DeserializationInput(factory).deserialize(
+                SerializationOutput(factory).serialize(exc, context),
+                context)
+
+        assertEquals(exc.message, exc2.message)
+    }
+
+    @Test
+    fun packageOwnershipExceptionTest() {
+        val exc = TransactionVerificationException.PackageOwnershipException(
+                txid,
+                attachmentHash,
+                "InvalidClass",
+                "com.invalid")
+
+        val exc2 = DeserializationInput(factory).deserialize(
+                SerializationOutput(factory).serialize(exc, context),
+                context)
+
+        assertEquals(exc.message, exc2.message)
+    }
+
+    @Test
+    fun invalidAttachmentExceptionTest() {
+        val exc = TransactionVerificationException.InvalidAttachmentException(
+                txid,
+                attachmentHash)
+
+        val exc2 = DeserializationInput(factory).deserialize(
+                SerializationOutput(factory).serialize(exc, context),
+                context)
+
+        assertEquals(exc.message, exc2.message)
+    }
+
+    @Test
+    fun untrustedAttachmentsExceptionTest() {
+        val exc = TransactionVerificationException.UntrustedAttachmentsException(
+                txid,
+                listOf(attachmentHash))
+
+        val exc2 = DeserializationInput(factory).deserialize(
+                SerializationOutput(factory).serialize(exc, context),
+                context)
+
+        assertEquals(exc.message, exc2.message)
+    }
 }

core/src/test/kotlin/net/corda/core/flows/FinalityFlowTests.kt

@@ -57,19 +57,6 @@ class FinalityFlowTests : WithFinality {
                 willThrow<IllegalArgumentException>())
     }
 
-    @Test
-    fun `prevent use of the old API if the CorDapp target version is 4`() {
-        val bob = createBob()
-        val stx = aliceNode.issuesCashTo(bob)
-        val resultFuture = CordappResolver.withCordapp(targetPlatformVersion = 4) {
-            @Suppress("DEPRECATION")
-            aliceNode.startFlowAndRunNetwork(FinalityFlow(stx)).resultFuture
-        }
-        assertThatIllegalArgumentException().isThrownBy {
-            resultFuture.getOrThrow()
-        }.withMessageContaining("A flow session for each external participant to the transaction must be provided.")
-    }
-
     @Test
     fun `allow use of the old API if the CorDapp target version is 3`() {
         val oldBob = createBob(cordapps = listOf(tokenOldCordapp()))

core/src/test/kotlin/net/corda/core/flows/ReferencedStatesFlowTests.kt

@@ -15,15 +15,9 @@ import net.corda.core.transactions.TransactionBuilder
 import net.corda.core.utilities.getOrThrow
 import net.corda.node.VersionInfo
 import net.corda.testing.common.internal.testNetworkParameters
-import net.corda.testing.core.SerializationEnvironmentRule
-import net.corda.testing.internal.vault.DUMMY_LINEAR_CONTRACT_PROGRAM_ID
-import net.corda.testing.internal.vault.DummyLinearContract
-import net.corda.testing.node.StartedMockNode
 import net.corda.testing.node.internal.*
-import net.corda.testing.node.transaction
 import org.junit.After
 import org.junit.Before
-import org.junit.Rule
 import org.junit.Test
 import kotlin.test.assertEquals
 
@@ -121,6 +115,58 @@ class ReferencedStatesFlowTests {
         assertEquals(2, allRefStates.states.size)
     }
 
+    @Test
+    fun `check old ref state is consumed when update used in tx with relevant states`() {
+        // 1. Create a state to be used as a reference state. Don't share it.
+        val newRefTx = nodes[0].services.startFlow(CreateRefState()).resultFuture.getOrThrow()
+        val newRefState = newRefTx.tx.outRefsOfType<RefState.State>().single()
+
+        // 2. Use the "newRefState" in a transaction involving another party (nodes[1]) which creates a new state. They should store the new state and the reference state.
+        val newTx = nodes[0].services.startFlow(UseRefState(nodes[1].info.legalIdentities.first(), newRefState.state.data.linearId))
+                .resultFuture.getOrThrow()
+        // Wait until node 1 stores the new tx.
+        nodes[1].services.validatedTransactions.updates.filter { it.id == newTx.id }.toFuture().getOrThrow()
+        // Check that nodes[1] has finished recording the transaction (and updating the vault.. hopefully!).
+        // nodes[1] should have two states. The newly created output of type "Regular.State" and the reference state created by nodes[0].
+        assertEquals(2, nodes[1].services.vaultService.queryBy<LinearState>().states.size)
+        // Now let's find the specific reference state on nodes[1].
+        val refStateLinearId = newRefState.state.data.linearId
+        val query = QueryCriteria.LinearStateQueryCriteria(linearId = listOf(refStateLinearId))
+        val theReferencedState = nodes[1].services.vaultService.queryBy<RefState.State>(query)
+        // There should be one result - the reference state.
+        assertEquals(newRefState, theReferencedState.states.single())
+        // The reference state should not be consumed.
+        assertEquals(Vault.StateStatus.UNCONSUMED, theReferencedState.statesMetadata.single().status)
+        // nodes[0] should also have the same state.
+        val nodeZeroQuery = QueryCriteria.LinearStateQueryCriteria(linearId = listOf(refStateLinearId))
+        val theReferencedStateOnNodeZero = nodes[0].services.vaultService.queryBy<RefState.State>(nodeZeroQuery)
+        assertEquals(newRefState, theReferencedStateOnNodeZero.states.single())
+        assertEquals(Vault.StateStatus.UNCONSUMED, theReferencedStateOnNodeZero.statesMetadata.single().status)
+
+        // 3. Update the reference state but don't share the update.
+        val updatedRefTx = nodes[0].services.startFlow(UpdateRefState(newRefState)).resultFuture.getOrThrow()
+
+        // 4. Use the evolved state as a reference state.
+        val updatedTx = nodes[0].services.startFlow(UseRefState(nodes[1].info.legalIdentities.first(), newRefState.state.data.linearId))
+                .resultFuture.getOrThrow()
+        // Wait until node 1 stores the new tx.
+        nodes[1].services.validatedTransactions.updates.filter { it.id == updatedTx.id }.toFuture().getOrThrow()
+        // Check that nodes[1] has finished recording the transaction (and updating the vault.. hopefully!).
+        // nodes[1] should have four states. The originals, plus the newly created output of type "Regular.State" and the reference state created by nodes[0].
+        assertEquals(4, nodes[1].services.vaultService.queryBy<LinearState>(QueryCriteria.VaultQueryCriteria(status = Vault.StateStatus.ALL)).states.size)
+        // Now let's find the original reference state on nodes[1].
+        val updatedQuery = QueryCriteria.VaultQueryCriteria(stateRefs = listOf(newRefState.ref), status = Vault.StateStatus.ALL)
+        val theOriginalReferencedState = nodes[1].services.vaultService.queryBy<RefState.State>(updatedQuery)
+        // There should be one result - the original reference state.
+        assertEquals(newRefState, theOriginalReferencedState.states.single())
+        // The reference state should be consumed.
+        assertEquals(Vault.StateStatus.CONSUMED, theOriginalReferencedState.statesMetadata.single().status)
+        // nodes[0] should also have the same state.
+        val theOriginalReferencedStateOnNodeZero = nodes[0].services.vaultService.queryBy<RefState.State>(updatedQuery)
+        assertEquals(newRefState, theOriginalReferencedStateOnNodeZero.states.single())
+        assertEquals(Vault.StateStatus.CONSUMED, theOriginalReferencedStateOnNodeZero.statesMetadata.single().status)
+    }
+
     // A dummy reference state contract.
     class RefState : Contract {
         companion object {

core/src/test/kotlin/net/corda/core/internal/TopologicalSortTest.kt

@@ -1,9 +1,7 @@
 package net.corda.core.internal
 
 import net.corda.client.mock.Generator
-import net.corda.core.contracts.ContractState
+import net.corda.core.contracts.*
-import net.corda.core.contracts.StateRef
-import net.corda.core.contracts.TransactionState
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.SignatureMetadata
 import net.corda.core.crypto.TransactionSignature
@@ -30,11 +28,12 @@ class TopologicalSortTest {
             override val references: List<StateRef> = emptyList()
     ) : CoreTransaction() {
         override val outputs: List<TransactionState<ContractState>> = (1..numberOfOutputs).map {
-            TransactionState(DummyState(), "", notary)
+            TransactionState(DummyState(), Contract::class.java.name, notary)
         }
         override val networkParametersHash: SecureHash? = testNetworkParameters().serialize().hash
     }
 
+    @BelongsToContract(Contract::class)
     class DummyState : ContractState {
         override val participants: List<AbstractParty> = emptyList()
     }

core/src/test/kotlin/net/corda/core/node/NetworkParametersTest.kt

@@ -4,6 +4,7 @@ import com.nhaarman.mockito_kotlin.doReturn
 import com.nhaarman.mockito_kotlin.whenever
 import net.corda.core.crypto.generateKeyPair
 import net.corda.core.internal.getPackageOwnerOf
+import net.corda.core.node.services.AttachmentId
 import net.corda.core.utilities.OpaqueBytes
 import net.corda.core.utilities.days
 import net.corda.core.utilities.getOrThrow
@@ -12,10 +13,7 @@ import net.corda.finance.flows.CashIssueFlow
 import net.corda.node.services.config.NotaryConfig
 import net.corda.nodeapi.internal.network.NetworkParametersCopier
 import net.corda.testing.common.internal.testNetworkParameters
-import net.corda.testing.core.ALICE_NAME
+import net.corda.testing.core.*
-import net.corda.testing.core.BOB_NAME
-import net.corda.testing.core.DUMMY_NOTARY_NAME
-import net.corda.testing.core.singleIdentity
 import net.corda.testing.node.MockNetworkNotarySpec
 import net.corda.testing.node.MockNetworkParameters
 import net.corda.testing.node.internal.InternalMockNetwork
@@ -26,6 +24,7 @@ import org.assertj.core.api.Assertions.*
 import org.junit.After
 import org.junit.Test
 import java.nio.file.Path
+import java.time.Duration
 import java.time.Instant
 import kotlin.test.assertEquals
 import kotlin.test.assertFails
@@ -64,6 +63,34 @@ class NetworkParametersTest {
         alice.start()
     }
 
+    @Test
+    fun `that we can copy while preserving the event horizon`() {
+        // this is defensive tests in response to CORDA-2769
+        val aliceNotaryParty = TestIdentity(ALICE_NAME).party
+        val aliceNotaryInfo = NotaryInfo(aliceNotaryParty, false)
+        val nm1 = NetworkParameters(
+                minimumPlatformVersion = 1,
+                notaries = listOf(aliceNotaryInfo),
+                maxMessageSize = Int.MAX_VALUE,
+                maxTransactionSize = Int.MAX_VALUE,
+                modifiedTime = Instant.now(),
+                epoch = 1,
+                whitelistedContractImplementations = mapOf("MyClass" to listOf(AttachmentId.allOnesHash)),
+                eventHorizon = Duration.ofDays(1)
+        )
+        val twoDays = Duration.ofDays(2)
+        val nm2 = nm1.copy(minimumPlatformVersion = 2, eventHorizon = twoDays)
+
+        assertEquals(2, nm2.minimumPlatformVersion)
+        assertEquals(nm1.notaries, nm2.notaries)
+        assertEquals(nm1.maxMessageSize, nm2.maxMessageSize)
+        assertEquals(nm1.maxTransactionSize, nm2.maxTransactionSize)
+        assertEquals(nm1.modifiedTime, nm2.modifiedTime)
+        assertEquals(nm1.epoch, nm2.epoch)
+        assertEquals(nm1.whitelistedContractImplementations, nm2.whitelistedContractImplementations)
+        assertEquals(twoDays, nm2.eventHorizon)
+    }
+
     // Notaries tests
     @Test
     fun `choosing notary not specified in network parameters will fail`() {

core/src/test/kotlin/net/corda/core/node/VaultUpdateTests.kt

@@ -14,7 +14,7 @@ import kotlin.test.assertFailsWith
 
 class VaultUpdateTests {
     private companion object {
-        const val DUMMY_PROGRAM_ID = "net.corda.core.node.VaultUpdateTests.DummyContract"
+        const val DUMMY_PROGRAM_ID = "net.corda.core.node.VaultUpdateTests\$DummyContract"
         val DUMMY_NOTARY = TestIdentity(DUMMY_NOTARY_NAME, 20).party
         val emptyUpdate = Vault.Update(emptySet(), emptySet(), type = Vault.UpdateType.GENERAL, references = emptySet())
     }
@@ -25,6 +25,7 @@ class VaultUpdateTests {
         }
     }
 
+    @BelongsToContract(DummyContract::class)
     private class DummyState : ContractState {
         override val participants: List<AbstractParty> = emptyList()
     }

core/src/test/kotlin/net/corda/core/transactions/TransactionBuilderTest.kt

@@ -10,8 +10,6 @@ import net.corda.core.crypto.SecureHash
 import net.corda.core.identity.Party
 import net.corda.core.internal.AbstractAttachment
 import net.corda.core.internal.PLATFORM_VERSION
-import net.corda.core.internal.cordapp.CordappImpl.Companion.DEFAULT_CORDAPP_VERSION
-import net.corda.core.internal.cordapp.CordappResolver
 import net.corda.core.node.ServicesForResolution
 import net.corda.core.node.ZoneVersionTooLowException
 import net.corda.core.node.services.AttachmentStorage
@@ -115,27 +113,6 @@ class TransactionBuilderTest {
         assertThat(wtx.references).containsOnly(referenceStateRef)
     }
 
-    @Test
-    fun `multiple commands with same data are joined without duplicates in terms of signers`() {
-        // This behaviour is only activated for platform version 4 onwards.
-        CordappResolver.withCordapp(targetPlatformVersion = 4) {
-            val aliceParty = TestIdentity(ALICE_NAME).party
-            val bobParty = TestIdentity(BOB_NAME).party
-            val tx = TransactionBuilder(notary)
-            tx.addCommand(DummyCommandData, notary.owningKey, aliceParty.owningKey)
-            tx.addCommand(DummyCommandData, aliceParty.owningKey, bobParty.owningKey)
-
-            val commands = tx.commands()
-
-            assertThat(commands).hasSize(1)
-            assertThat(commands.single()).satisfies { cmd ->
-                assertThat(cmd.value).isEqualTo(DummyCommandData)
-                assertThat(cmd.signers).hasSize(3)
-                assertThat(cmd.signers).contains(notary.owningKey, bobParty.owningKey, aliceParty.owningKey)
-            }
-        }
-    }
-
     @Test
     fun `automatic signature constraint`() {
         val aliceParty = TestIdentity(ALICE_NAME).party

core/src/test/kotlin/net/corda/core/utilities/LazyMappedListTest.kt

@@ -1,11 +1,20 @@
 package net.corda.core.utilities
 
+import net.corda.core.contracts.ComponentGroupEnum.*
 import net.corda.core.internal.lazyMapped
+import net.corda.core.internal.TransactionDeserialisationException
+import net.corda.core.internal.eagerDeserialise
+import net.corda.core.serialization.MissingAttachmentsException
+import org.junit.Rule
 import org.junit.Test
+import org.junit.rules.ExpectedException
 import kotlin.test.assertEquals
 
 class LazyMappedListTest {
 
+    @get:Rule
+    val exception: ExpectedException = ExpectedException.none()
+
     @Test
     fun `LazyMappedList works`() {
         val originalList = (1 until 10).toList()
@@ -33,4 +42,29 @@ class LazyMappedListTest {
         assertEquals(1, callCounter)
     }
 
+    @Test
+    fun testMissingAttachments() {
+        exception.expect(MissingAttachmentsException::class.java)
+        exception.expectMessage("Uncatchable!")
+
+        val lazyList = (0 until 5).toList().lazyMapped<Int, Int> { _, _ ->
+            throw MissingAttachmentsException(emptyList(), "Uncatchable!")
+        }
+
+        lazyList.eagerDeserialise { _, _ -> -999 }
+    }
+
+    @Test
+    fun testDeserialisationExceptions() {
+        val lazyList = (0 until 5).toList().lazyMapped<Int, Int> { _, index ->
+            throw TransactionDeserialisationException(
+                OUTPUTS_GROUP, index, IllegalStateException("Catch this!"))
+        }
+
+        lazyList.eagerDeserialise { _, _ -> -999 }
+        assertEquals(5, lazyList.size)
+        lazyList.forEachIndexed { idx, item ->
+            assertEquals(-999, item, "Item[$idx] mismatch")
+        }
+    }
 }

core/src/test/kotlin/net/corda/core/utilities/ProgressTrackerTest.kt

@@ -36,12 +36,14 @@ class ProgressTrackerTest {
     lateinit var pt: ProgressTracker
     lateinit var pt2: ProgressTracker
     lateinit var pt3: ProgressTracker
+    lateinit var pt4: ProgressTracker
 
     @Before
     fun before() {
         pt = SimpleSteps.tracker()
         pt2 = ChildSteps.tracker()
         pt3 = BabySteps.tracker()
+        pt4 = ChildSteps.tracker()
     }
 
     @Test
@@ -129,8 +131,8 @@ class ProgressTrackerTest {
         assertCurrentStepsTree(6, SimpleSteps.THREE)
 
         // Assert no structure changes and proper steps propagation.
-        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(1, 2, 4, 6))
+        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(0, 1, 2, 4, 6))
-        assertThat(stepsTreeNotification).isEmpty()
+        assertThat(stepsTreeNotification).hasSize(2) // The initial tree state, plus one per tree update
     }
 
     @Test
@@ -164,8 +166,8 @@ class ProgressTrackerTest {
         assertCurrentStepsTree(7, ChildSteps.SEA)
 
         // Assert no structure changes and proper steps propagation.
-        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(1, 4, 7))
+        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(0, 1, 4, 7))
-        assertThat(stepsTreeNotification).isEmpty()
+        assertThat(stepsTreeNotification).hasSize(3) // The initial tree state, plus one per update
     }
     
     @Test
@@ -179,7 +181,7 @@ class ProgressTrackerTest {
         }
 
         // Put current state as a first change for simplicity when asserting.
-        val stepsTreeNotification = mutableListOf(pt.allStepsLabels)
+        val stepsTreeNotification = mutableListOf<List<Pair<Int, String>>>()
         pt.stepsTreeChanges.subscribe {
             stepsTreeNotification += it
         }
@@ -201,8 +203,8 @@ class ProgressTrackerTest {
         assertCurrentStepsTree(10, SimpleSteps.FOUR)
 
         // Assert no structure changes and proper steps propagation.
-        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(2, 7, 10))
+        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(0, 2, 7, 10))
-        assertThat(stepsTreeNotification).hasSize(2) // 1 change + 1 our initial state
+        assertThat(stepsTreeNotification).hasSize(3) // The initial tree state, plus one per update.
     }
 
     @Test
@@ -216,7 +218,7 @@ class ProgressTrackerTest {
         }
 
         // Put current state as a first change for simplicity when asserting.
-        val stepsTreeNotification = mutableListOf(pt.allStepsLabels)
+        val stepsTreeNotification = mutableListOf<List<Pair<Int, String>>>()
         pt.stepsTreeChanges.subscribe {
             stepsTreeNotification += it
         }
@@ -236,8 +238,8 @@ class ProgressTrackerTest {
         assertCurrentStepsTree(3, BabySteps.UNOS)
 
         // Assert no structure changes and proper steps propagation.
-        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(2, 5, 3))
+        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(0, 2, 5, 3))
-        assertThat(stepsTreeNotification).hasSize(2) // 1 change + 1 our initial state.
+        assertThat(stepsTreeNotification).hasSize(3) // The initial tree state, plus one per update
     }
 
     @Test
@@ -256,12 +258,66 @@ class ProgressTrackerTest {
         pt.currentStep = SimpleSteps.TWO
 
         val stepsIndexNotifications = LinkedList<Int>()
-        pt.stepsTreeIndexChanges.subscribe() {
+        pt.stepsTreeIndexChanges.subscribe {
             stepsIndexNotifications += it
         }
 
         pt2.currentStep = ChildSteps.AYY
 
-        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(1, 2, 3))
+        assertThat(stepsIndexNotifications).containsExactlyElementsOf(listOf(0, 1, 2, 3))
+    }
+
+    @Test
+    fun `all step changes seen if subscribed mid flow`() {
+        val steps = mutableListOf<String>()
+        pt.nextStep()
+        pt.nextStep()
+        pt.nextStep()
+        pt.changes.subscribe { steps.add(it.toString())}
+        pt.nextStep()
+        pt.nextStep()
+        pt.nextStep()
+        assertEquals(listOf("Starting", "one", "two", "three", "four", "Done"), steps)
+    }
+
+    @Test
+    fun `all tree changes seen if subscribed mid flow`() {
+        val stepTreeNotifications = mutableListOf<List<Pair<Int, String>>>()
+        val firstStepLabels = pt.allStepsLabels
+
+        pt.setChildProgressTracker(SimpleSteps.TWO, pt2)
+        val secondStepLabels = pt.allStepsLabels
+
+        pt.setChildProgressTracker(SimpleSteps.TWO, pt3)
+        val thirdStepLabels = pt.allStepsLabels
+        pt.stepsTreeChanges.subscribe { stepTreeNotifications.add(it)}
+
+        // Should have one notification for original tree, then one for each time it changed.
+        assertEquals(3, stepTreeNotifications.size)
+        assertEquals(listOf(firstStepLabels, secondStepLabels, thirdStepLabels), stepTreeNotifications)
+    }
+
+    @Test
+    fun `trees with child trackers with duplicate steps reported correctly`() {
+        val stepTreeNotifications = mutableListOf<List<Pair<Int, String>>>()
+        val stepIndexNotifications = mutableListOf<Int>()
+        pt.stepsTreeChanges.subscribe { stepTreeNotifications += it }
+        pt.stepsTreeIndexChanges.subscribe { stepIndexNotifications += it }
+        pt.setChildProgressTracker(SimpleSteps.ONE, pt2)
+        pt.setChildProgressTracker(SimpleSteps.TWO, pt4)
+
+        pt.currentStep = SimpleSteps.ONE
+        pt2.currentStep = ChildSteps.AYY
+        pt2.nextStep()
+        pt2.nextStep()
+        pt.nextStep()
+        pt4.currentStep = ChildSteps.AYY
+
+        assertEquals(listOf(0, 1, 2, 3, 4, 5, 6), stepIndexNotifications)
+    }
+
+    @Test
+    fun `cannot assign step not belonging to this progress tracker`() {
+        assertFails { pt.currentStep = BabySteps.UNOS }
     }
 }

djvm/.gitignore

@@ -1,3 +1,13 @@
-tmp/
+# DJVM-specific files
+**/tmp/
 *.log
 *.log.gz
+
+# IntelliJ
+*.iml
+*.ipr
+*.iws
+.idea/
+
+**/out/
+

djvm/build.gradle

@@ -1,75 +1,120 @@
-plugins {
+buildscript {
-    id 'com.github.johnrengelman.shadow'
+    ext {
+        corda_djvm_version = '5.0-SNAPSHOT'
+        artifactory_contextUrl = 'https://ci-artifactory.corda.r3cev.com/artifactory'
+    }
+
+    repositories {
+        mavenCentral()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
 }
+
+plugins {
+    id 'net.corda.plugins.publish-utils' version '4.0.42' apply false
+    id 'com.github.johnrengelman.shadow' version '5.0.0' apply false
+    id 'com.jfrog.artifactory' version '4.7.3' apply false
+    id 'com.jfrog.bintray' version '1.4' apply false
+    id 'com.gradle.build-scan' version '2.2.1'
+}
+
+import static org.gradle.api.JavaVersion.*
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+subprojects {
+    group 'net.corda'
+    version corda_djvm_version
+
+    repositories {
+        mavenCentral()
+        jcenter()
+    }
+
+    tasks.withType(JavaCompile) {
+        sourceCompatibility = VERSION_1_8
+        targetCompatibility = VERSION_1_8
+        options.encoding = 'UTF-8'
+    }
+
+    tasks.withType(KotlinCompile) {
+        kotlinOptions {
+            languageVersion = '1.2'
+            apiVersion = '1.2'
+            jvmTarget = VERSION_1_8
+            javaParameters = true   // Useful for reflection.
+            freeCompilerArgs = ['-Xjvm-default=enable']
+        }
+    }
+
+    tasks.withType(Jar) { task ->
+        manifest {
+            attributes('Corda-Vendor': 'Corda Open Source')
+            attributes('Automatic-Module-Name': "net.corda.${task.project.name.replaceAll('-', '.')}")
+        }
+    }
+
+    tasks.withType(Test) {
+        // Prevent the project from creating temporary files outside of the build directory.
+        systemProperty 'java.io.tmpdir', buildDir.absolutePath
+    }
+}
+
 apply plugin: 'net.corda.plugins.publish-utils'
 apply plugin: 'com.jfrog.artifactory'
-apply plugin: 'idea'
 
-description 'Corda deterministic JVM sandbox'
+bintrayConfig {
-
+    user = System.getenv('CORDA_BINTRAY_USER')
-ext {
+    key = System.getenv('CORDA_BINTRAY_KEY')
-    // Shaded version of ASM to avoid conflict with root project.
+    repo = 'corda'
-    asm_version = '6.2.1'
+    org = 'r3'
-}
+    licenses = ['Apache-2.0']
-
+    vcsUrl = 'https://github.com/corda/corda'
-repositories {
+    projectUrl = 'https://github.com/corda/corda'
-    maven {
+    gpgSign = true
-        url "$artifactory_contextUrl/corda-dev"
+    gpgPassphrase = System.getenv('CORDA_BINTRAY_GPG_PASSPHRASE')
+    publications = [
+        'corda-djvm',
+        'corda-djvm-cli'
+    ]
+    license {
+        name = 'Apache-2.0'
+        url = 'https://www.apache.org/licenses/LICENSE-2.0'
+        distribution = 'repo'
+    }
+    developer {
+        id = 'R3'
+        name = 'R3'
+        email = 'dev@corda.net'
     }
 }
 
-configurations {
+artifactory {
-    testCompile.extendsFrom shadow
+    publish {
-    jdkRt.resolutionStrategy {
+        contextUrl = artifactory_contextUrl
-        // Always check the repository for a newer SNAPSHOT.
+        repository {
-        cacheChangingModulesFor 0, 'seconds'
+            repoKey = 'corda-dev'
+            username = System.getenv('CORDA_ARTIFACTORY_USERNAME')
+            password = System.getenv('CORDA_ARTIFACTORY_PASSWORD')
+        }
+
+        defaults {
+            // The root project has applied 'publish-utils' but has nothing to publish.
+            if (project != rootProject) {
+                publications(project.extensions.publish.name())
+            }
+        }
     }
 }
 
-dependencies {
+wrapper {
-    shadow "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    gradleVersion = "5.2.1"
-    shadow "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+    distributionType = Wrapper.DistributionType.ALL
-    shadow "org.slf4j:slf4j-api:$slf4j_version"
-
-    // ASM: byte code manipulation library
-    compile "org.ow2.asm:asm:$asm_version"
-    compile "org.ow2.asm:asm-commons:$asm_version"
-
-    // ClassGraph: classpath scanning
-    shadow "io.github.classgraph:classgraph:$class_graph_version"
-
-    // Test utilities
-    testCompile "junit:junit:$junit_version"
-    testCompile "org.assertj:assertj-core:$assertj_version"
-    testCompile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
-    jdkRt "net.corda:deterministic-rt:latest.integration"
 }
 
-jar.enabled = false
+buildScan {
-
+    termsOfServiceUrl = 'https://gradle.com/terms-of-service'
-shadowJar {
+    termsOfServiceAgree = 'yes'
-    baseName 'corda-djvm'
-    classifier ''
-    relocate 'org.objectweb.asm', 'djvm.org.objectweb.asm'
-}
-assemble.dependsOn shadowJar
-
-tasks.withType(Test) {
-    systemProperty 'deterministic-rt.path', configurations.jdkRt.asPath
-}
-
-artifacts {
-    publish shadowJar
-}
-
-publish {
-    dependenciesFrom configurations.shadow
-    name shadowJar.baseName
-}
-
-idea {
-    module {
-        downloadJavadoc = true
-        downloadSources = true
-    }
 }

djvm/cli/build.gradle

@@ -1,48 +0,0 @@
-plugins {
-    id 'com.github.johnrengelman.shadow'
-}
-
-repositories {
-    maven {
-        url "$artifactory_contextUrl/corda-dev"
-    }
-}
-
-configurations {
-    deterministicRt
-}
-
-dependencies {
-    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
-    compile "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
-    compile "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
-    compile "com.jcabi:jcabi-manifests:$jcabi_manifests_version"
-
-    compile "info.picocli:picocli:$picocli_version"
-    compile project(path: ":djvm", configuration: "shadow")
-
-    // Deterministic runtime - used in whitelist generation
-    deterministicRt project(path: ':jdk8u-deterministic', configuration: 'jdk')
-}
-
-jar.enabled = false
-
-shadowJar {
-    baseName = "corda-djvm"
-    classifier = 'cli'
-    manifest {
-        attributes(
-            'Automatic-Module-Name': 'net.corda.djvm',
-            'Main-Class': 'net.corda.djvm.tools.cli.Program',
-            'Build-Date': new Date().format("yyyy-MM-dd'T'HH:mm:ssZ")
-        )
-    }
-}
-assemble.dependsOn shadowJar
-
-task generateWhitelist(type: JavaExec, dependsOn: shadowJar) {
-    // This is an example of how a whitelist can be generated from a JAR. In most applications though, it is recommended
-    // that the minimal set whitelist is used.
-    main = '-jar'
-    args = [shadowJar.outputs.files.singleFile, 'whitelist', 'generate', '-o', "$buildDir/jdk8-deterministic.dat.gz", configurations.deterministicRt.files[0] ]
-}

djvm/djvm/build.gradle

@@ -0,0 +1,87 @@
+plugins {
+    id 'org.jetbrains.kotlin.jvm'
+    id 'com.github.johnrengelman.shadow'
+    id 'net.corda.plugins.publish-utils'
+    id 'com.jfrog.artifactory'
+    id 'idea'
+}
+
+description 'Corda deterministic JVM sandbox'
+
+repositories {
+    maven {
+        url "$artifactory_contextUrl/corda-dev"
+    }
+}
+
+configurations {
+    testImplementation.extendsFrom shadow
+    jdkRt.resolutionStrategy {
+        // Always check the repository for a newer SNAPSHOT.
+        cacheChangingModulesFor 0, 'seconds'
+    }
+}
+
+dependencies {
+    shadow "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    shadow "org.jetbrains.kotlin:kotlin-reflect:$kotlin_version"
+    shadow "org.slf4j:slf4j-api:$slf4j_version"
+
+    // ASM: byte code manipulation library
+    implementation "org.ow2.asm:asm:$asm_version"
+    implementation "org.ow2.asm:asm-commons:$asm_version"
+
+    // ClassGraph: classpath scanning
+    shadow "io.github.classgraph:classgraph:$class_graph_version"
+
+    // Test utilities
+    testImplementation "junit:junit:$junit_version"
+    testImplementation "org.assertj:assertj-core:$assertj_version"
+    testImplementation "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
+    jdkRt "net.corda:deterministic-rt:latest.integration"
+}
+
+jar.enabled = false
+
+shadowJar {
+    baseName 'corda-djvm'
+    classifier ''
+    relocate 'org.objectweb.asm', 'djvm.org.objectweb.asm'
+
+    // These particular classes are only needed to "bootstrap"
+    // the compilation of the other sandbox classes. At runtime,
+    // we will generate better versions from deterministic-rt.jar.
+    exclude 'sandbox/java/lang/Appendable.class'
+    exclude 'sandbox/java/lang/CharSequence.class'
+    exclude 'sandbox/java/lang/Character\$Subset.class'
+    exclude 'sandbox/java/lang/Character\$Unicode*.class'
+    exclude 'sandbox/java/lang/Comparable.class'
+    exclude 'sandbox/java/lang/Enum.class'
+    exclude 'sandbox/java/lang/Iterable.class'
+    exclude 'sandbox/java/lang/StackTraceElement.class'
+    exclude 'sandbox/java/lang/StringBuffer.class'
+    exclude 'sandbox/java/lang/StringBuilder.class'
+    exclude 'sandbox/java/nio/**'
+    exclude 'sandbox/java/util/**'
+}
+assemble.dependsOn shadowJar
+
+tasks.withType(Test) {
+    systemProperty 'deterministic-rt.path', configurations.jdkRt.asPath
+}
+
+artifacts {
+    publish shadowJar
+}
+
+publish {
+    dependenciesFrom configurations.shadow
+    name shadowJar.baseName
+}
+
+idea {
+    module {
+        downloadJavadoc = true
+        downloadSources = true
+    }
+}

djvm/djvm/cli/build.gradle

@@ -0,0 +1,64 @@
+plugins {
+    id 'org.jetbrains.kotlin.jvm'
+    id 'com.github.johnrengelman.shadow'
+    id 'net.corda.plugins.publish-utils'
+    id 'com.jfrog.artifactory'
+}
+
+description 'Corda deterministic JVM sandbox command-line tool'
+
+ext {
+    djvmName = 'corda-djvm-cli'
+}
+
+dependencies {
+    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
+    implementation "org.jetbrains.kotlin:kotlin-reflect"
+    implementation "org.apache.logging.log4j:log4j-slf4j-impl:$log4j_version"
+    implementation "org.apache.logging.log4j:log4j-core:$log4j_version"
+    implementation "com.jcabi:jcabi-manifests:$jcabi_manifests_version"
+
+    implementation "info.picocli:picocli:$picocli_version"
+    implementation project(path: ':djvm', configuration: 'shadow')
+}
+
+jar.enabled = false
+
+shadowJar {
+    baseName djvmName
+    classifier ''
+    manifest {
+        attributes(
+            'Automatic-Module-Name': 'net.corda.djvm.cli',
+            'Main-Class': 'net.corda.djvm.tools.cli.Program',
+            'Build-Date': new Date().format("yyyy-MM-dd'T'HH:mm:ssZ"),
+            'Class-Path': 'tmp/'
+        )
+    }
+}
+
+task shadowZip(type: Zip) {
+    archiveBaseName = djvmName
+    archiveClassifier = ''
+
+    from(shadowJar) {
+        rename "$djvmName-(.*).jar", "${djvmName}.jar"
+    }
+    from('src/shell/') {
+        fileMode = 0755
+    }
+    zip64 true
+}
+
+assemble.dependsOn shadowZip
+
+artifacts {
+    publish shadowZip
+}
+
+publish {
+    dependenciesFrom configurations.shadow
+    publishSources = false
+    publishJavadoc = false
+    name shadowZip.baseName
+}

djvm/djvm/cli/src/main/kotlin/net/corda/djvm/tools/cli/CommandBase.kt

@@ -8,6 +8,8 @@ import net.corda.djvm.references.ClassReference
 import net.corda.djvm.references.EntityReference
 import net.corda.djvm.references.MemberReference
 import net.corda.djvm.rewiring.SandboxClassLoadingException
+import org.apache.logging.log4j.Level
+import org.apache.logging.log4j.core.config.Configurator
 import picocli.CommandLine
 import picocli.CommandLine.Help.Ansi
 import picocli.CommandLine.Option
@@ -19,7 +21,7 @@ abstract class CommandBase : Callable<Boolean> {
 
     @Option(
             names = ["-l", "--level"],
-            description = ["The minimum severity level to log (TRACE, INFO, WARNING or ERROR."],
+            description = ["The minimum severity level to log (TRACE, DEBUG, INFO, WARNING or ERROR."],
             converter = [SeverityConverter::class]
     )
     protected var level: Severity = Severity.WARNING
@@ -102,6 +104,7 @@ abstract class CommandBase : Callable<Boolean> {
             printError("Error: Cannot set verbose and quiet modes at the same time")
             return false
         }
+        configureLogging()
         return try {
             handleCommand()
         } catch (exception: Throwable) {
@@ -110,6 +113,17 @@ abstract class CommandBase : Callable<Boolean> {
         }
     }
 
+    private fun configureLogging() {
+        val logLevel = when(level) {
+            Severity.ERROR -> Level.ERROR
+            Severity.WARNING -> Level.WARN
+            Severity.INFORMATIONAL -> Level.INFO
+            Severity.DEBUG -> Level.DEBUG
+            Severity.TRACE -> Level.TRACE
+        }
+        Configurator.setRootLevel(logLevel)
+    }
+
     protected fun printException(exception: Throwable) = when (exception) {
         is SandboxClassLoadingException -> {
             printMessages(exception.messages, exception.classOrigins)
@@ -262,4 +276,4 @@ abstract class CommandBase : Callable<Boolean> {
         }
     }
 
-}
+}

djvm/djvm/cli/src/main/resources/log4j2.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="info">
 
-    <ThresholdFilter level="info"/>
+    <ThresholdFilter level="trace"/>
     <Appenders>
         <!-- Will generate up to 10 log files for a given day. During every rollover it will delete
              those that are older than 60 days, but keep the most recent 10 GB -->

djvm/djvm/cli/src/shell/djvm

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(dirname $(readlink -f ${BASH_SOURCE[0]}))
+
+CLASSPATH="${CLASSPATH:-}"
+
+DEBUG=`echo "${DEBUG:-0}" | sed 's/^[Nn][Oo]*$/0/g'`
+DEBUG_PORT=5005
+DEBUG_AGENT=""
+
+if [ "$DEBUG" != 0 ]; then
+    echo "Opening remote debugging session on port $DEBUG_PORT"
+    DEBUG_AGENT="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=$DEBUG_PORT"
+fi
+
+exec java $DEBUG_AGENT -cp "$CLASSPATH:.:tmp:$SCRIPT_DIR/corda-djvm-cli.jar" net.corda.djvm.tools.cli.Program "$@"

djvm/djvm/cli/src/shell/djvm.bat

@@ -0,0 +1,15 @@
+@ECHO off
+
+SETLOCAL ENABLEEXTENSIONS
+
+IF NOT DEFINED CLASSPATH (SET CLASSPATH=)
+
+IF DEFINED DEBUG (
+    SET DEBUG_PORT=5005
+    SET DEBUG_AGENT=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=%DEBUG_PORT%
+    ECHO Opening remote debugging session on port %DEBUG_PORT%
+) ELSE (
+    SET DEBUG_AGENT=
+)
+
+CALL java %DEBUG_AGENT% -cp "%CLASSPATH%;.;tmp;%~dp0\corda-djvm-cli.jar" net.corda.djvm.tools.cli.Program %*

djvm/djvm/cli/src/shell/install

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(dirname $(readlink -f ${BASH_SOURCE[0]}))
+
+# Generate auto-completion file for Bash and ZSH
+java -cp ${SCRIPT_DIR}/corda-djvm-cli.jar \
+    picocli.AutoComplete -n djvm net.corda.djvm.tools.cli.Commands -f

djvm/djvm/shell/djvm

@@ -3,8 +3,7 @@
 file="${BASH_SOURCE[0]}"
 linked_file="$(test -L "$file" && readlink "$file" || echo "$file")"
 base_dir="$(cd "$(dirname "$linked_file")/../" && pwd)"
-version="$(cat $base_dir/../build.gradle | sed -n 's/^[ ]*ext\.corda_release_version[ =]*"\([^"]*\)".*$/\1/p')"
+djvm_cli_jar=$(ls -1 $base_dir/cli/build/libs/corda-djvm-cli-*.jar)
-jar_file="$base_dir/cli/build/libs/corda-djvm-$version-cli.jar"
 
 CLASSPATH="${CLASSPATH:-}"
 
@@ -17,4 +16,4 @@ if [ "$DEBUG" != 0 ]; then
     DEBUG_AGENT="-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=$DEBUG_PORT"
 fi
 
-java $DEBUG_AGENT -cp "$CLASSPATH:.:tmp:$jar_file" net.corda.djvm.tools.cli.Program "$@"
+java $DEBUG_AGENT -cp "$CLASSPATH:.:tmp:$djvm_cli_jar" net.corda.djvm.tools.cli.Program "$@"

djvm/djvm/shell/install

@@ -2,16 +2,23 @@
 
 file="${BASH_SOURCE[0]}"
 base_dir="$(cd "$(dirname "$file")/" && pwd)"
-version="$(cat $base_dir/../../build.gradle | sed -n 's/^[ ]*ext\.corda_release_version[ =]*"\([^"]*\)".*$/\1/p')"
 
 # Build DJVM module and CLI
 cd "$base_dir/.."
-../gradlew shadowJar
+if !(../gradlew shadowJar); then
+    echo "Failed to build DJVM"
+    exit 1
+fi
+
+djvm_cli_jar=$(ls -1 $base_dir/../cli/build/libs/corda-djvm-cli-*.jar)
 
 # Generate auto-completion file for Bash and ZSH
 cd "$base_dir"
-java -cp "$base_dir/../cli/build/libs/corda-djvm-$version-cli.jar" \
+if !(java -cp $djvm_cli_jar \
-    picocli.AutoComplete -n djvm net.corda.djvm.tools.cli.Commands -f
+    picocli.AutoComplete -n djvm net.corda.djvm.tools.cli.Commands -f); then
+    echo "Failed to generate auto-completion file"
+    exit 1
+fi
 
 # Create a symbolic link to the `djvm` utility
 sudo ln -sf "$base_dir/djvm" /usr/local/bin/djvm