Merge remote-tracking branch 'origin/release/os/4.4' into jzd/os4.4-to-os4.5-merge-2020-03-17

build.gradle

@@ -635,67 +635,83 @@ buildScan {
     termsOfServiceAgree = 'yes'
 }
 
+ext.generalPurpose  = [
+        numberOfShards: 10,
+        streamOutput: false,
+        coresPerFork: 2,
+        memoryInGbPerFork: 12,
+        nodeTaints: "small"
+]
+
+ext.largeScaleSet  = [
+        numberOfShards: 15,
+        streamOutput: false,
+        coresPerFork: 6,
+        memoryInGbPerFork: 10,
+        nodeTaints: "big"
+]
+
 task allParallelIntegrationTest(type: ParallelTestGroup) {
     dependsOn dependxiesModule
     podLogLevel PodLogLevel.INFO
     testGroups "integrationTest"
-    numberOfShards 10
-    streamOutput false
-    coresPerFork 2
-    memoryInGbPerFork 12
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
     distribute DistributeTestsBy.METHOD
-    nodeTaints "big"
 }
 task allParallelUnitTest(type: ParallelTestGroup) {
     dependsOn dependxiesModule
     podLogLevel PodLogLevel.INFO
     testGroups "test"
-    numberOfShards 10
-    streamOutput false
-    coresPerFork 2
-    memoryInGbPerFork 12
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
     distribute DistributeTestsBy.CLASS
-    nodeTaints "small"
 }
 task allParallelUnitAndIntegrationTest(type: ParallelTestGroup) {
     dependsOn dependxiesModule
     testGroups "test", "integrationTest"
-    numberOfShards 15
-    streamOutput false
-    coresPerFork 6
-    memoryInGbPerFork 10
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
     distribute DistributeTestsBy.METHOD
-    nodeTaints "big"
 }
 task parallelRegressionTest(type: ParallelTestGroup) {
     testGroups "test", "integrationTest", "smokeTest"
     dependsOn dependxiesModule
-    numberOfShards 15
-    streamOutput false
-    coresPerFork 2
-    memoryInGbPerFork 10
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
     distribute DistributeTestsBy.METHOD
-    nodeTaints "big"
 }
 task allParallelSmokeTest(type: ParallelTestGroup) {
     testGroups "smokeTest"
     dependsOn dependxiesModule
-    numberOfShards 4
-    streamOutput false
-    coresPerFork 6
-    memoryInGbPerFork 10
-    distribute DistributeTestsBy.CLASS
-    nodeTaints "big"
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
+    distribute DistributeTestsBy.METHOD
 }
 task allParallelSlowIntegrationTest(type: ParallelTestGroup) {
     testGroups "slowIntegrationTest"
     dependsOn dependxiesModule
-    numberOfShards 4
-    streamOutput false
-    coresPerFork 6
-    memoryInGbPerFork 10
-    distribute DistributeTestsBy.CLASS
-    nodeTaints "big"
+    numberOfShards generalPurpose.numberOfShards
+    streamOutput generalPurpose.streamOutput
+    coresPerFork generalPurpose.coresPerFork
+    memoryInGbPerFork generalPurpose.memoryInGbPerFork
+    nodeTaints generalPurpose.nodeTaints
+    distribute DistributeTestsBy.METHOD
 }
 apply plugin: 'com.r3.testing.distributed-testing'
 apply plugin: 'com.r3.testing.image-building'

constants.properties

@@ -30,7 +30,7 @@ snakeYamlVersion=1.19
 caffeineVersion=2.7.0
 metricsVersion=4.1.0
 metricsNewRelicVersion=1.1.1
-djvmVersion=1.0-RC10
+djvmVersion=1.0
 deterministicRtVersion=1.0-RC02
 openSourceBranch=https://github.com/corda/corda/blob/release/os/4.4
 openSourceSamplesBranch=https://github.com/corda/samples/blob/release-V4

core/src/main/kotlin/net/corda/core/node/services/vault/QueryCriteriaUtils.kt

@@ -302,7 +302,7 @@ object Builder {
 
     @JvmStatic
     @JvmOverloads
-    fun <R> FieldInfo.notEqual(value: R, exactMatch: Boolean = true) = predicate(Builder.equal(value, exactMatch))
+    fun <R> FieldInfo.notEqual(value: R, exactMatch: Boolean = true) = predicate(Builder.notEqual(value, exactMatch))
 
     @JvmStatic
     @Deprecated("Does not support fields from a MappedSuperclass. Use equivalent on a FieldInfo.")

core/src/test/kotlin/net/corda/core/node/services/vault/QueryCriteriaUtilsBuilderTest.kt

@@ -0,0 +1,165 @@
+package net.corda.core.node.services.vault
+
+import net.corda.core.node.services.vault.BinaryComparisonOperator
+import net.corda.core.node.services.vault.Builder.`in`
+import net.corda.core.node.services.vault.Builder.equal
+import net.corda.core.node.services.vault.Builder.greaterThan
+import net.corda.core.node.services.vault.Builder.greaterThanOrEqual
+import net.corda.core.node.services.vault.Builder.isNull
+import net.corda.core.node.services.vault.Builder.lessThan
+import net.corda.core.node.services.vault.Builder.lessThanOrEqual
+import net.corda.core.node.services.vault.Builder.like
+import net.corda.core.node.services.vault.Builder.notEqual
+import net.corda.core.node.services.vault.Builder.notIn
+import net.corda.core.node.services.vault.Builder.notLike
+import net.corda.core.node.services.vault.Builder.notNull
+import net.corda.core.node.services.vault.CollectionOperator
+import net.corda.core.node.services.vault.ColumnPredicate
+import net.corda.core.node.services.vault.ColumnPredicate.AggregateFunction
+import net.corda.core.node.services.vault.ColumnPredicate.Between
+import net.corda.core.node.services.vault.ColumnPredicate.BinaryComparison
+import net.corda.core.node.services.vault.ColumnPredicate.CollectionExpression
+import net.corda.core.node.services.vault.ColumnPredicate.EqualityComparison
+import net.corda.core.node.services.vault.ColumnPredicate.Likeness
+import net.corda.core.node.services.vault.ColumnPredicate.NullExpression
+import net.corda.core.node.services.vault.CriteriaExpression.ColumnPredicateExpression
+import net.corda.core.node.services.vault.EqualityComparisonOperator
+import net.corda.core.node.services.vault.FieldInfo
+import net.corda.core.node.services.vault.LikenessOperator
+import net.corda.core.node.services.vault.NullOperator
+import net.corda.core.node.services.vault.Operator
+import net.corda.core.node.services.vault.getField
+import org.assertj.core.api.Assertions.assertThat
+import org.assertj.core.api.ObjectAssert
+import org.junit.Test
+import javax.persistence.Entity
+
+class QueryCriteriaUtilsBuilderTest {
+
+    /** JPA Entity class needed by `getField` */
+    @Entity
+    private class TestEntity(val field: String)
+
+    /** Returns a `FieldInfo` object to work on */
+    private val fieldInfo: FieldInfo get() = getField("field", TestEntity::class.java)
+
+    /** Thrown for the `ColumnPredicate` types that have no `operator` field */
+    private class ColumnPredicateHasNoOperatorFieldException : Exception("This ColumnPredicate has no operator field")
+
+    /** Returns the `operator` for the given `ColumnPredicate` */
+    private fun ColumnPredicate<out Any?>.getOperator(): Operator = when (this) {
+        is AggregateFunction -> throw ColumnPredicateHasNoOperatorFieldException()
+        is Between -> throw ColumnPredicateHasNoOperatorFieldException()
+        is BinaryComparison<*> -> operator
+        is CollectionExpression -> operator
+        is EqualityComparison<*> -> operator
+        is Likeness -> operator
+        is NullExpression -> operator
+    }
+
+    /** Returns the `operator` for the given `ColumnPredicateExpression` */
+    private fun ColumnPredicateExpression<Any, *>.getOperator(): Operator = this.predicate.getOperator()
+
+    /** Assert that the `ColumnPredicateExpression` uses the given `Operator`. */
+    private fun <T : ColumnPredicateExpression<Any, C>, C> ObjectAssert<T>.usesOperator(operator: Operator) {
+        extracting {
+            assertThat(it.getOperator()).isEqualTo(operator)
+        }
+    }
+
+    /** Sample `String` value to pass to the predicate expression */
+    private val stringValue = ""
+
+    /** Sample `List` value to pass to the predicate expression */
+    private val listValue = emptyList<String>()
+
+    @Test(timeout = 500)
+    fun `equal predicate uses EQUAL operator`() {
+        assertThat(fieldInfo.equal(stringValue)).usesOperator(EqualityComparisonOperator.EQUAL)
+    }
+
+    @Test(timeout = 500)
+    fun `equal predicate (exactMatch=false) uses EQUAL_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.equal(stringValue, exactMatch = false)).usesOperator(EqualityComparisonOperator.EQUAL_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `notEqual predicate uses NOT_EQUAL operator`() {
+        assertThat(fieldInfo.notEqual(stringValue)).usesOperator(EqualityComparisonOperator.NOT_EQUAL)
+    }
+
+    @Test(timeout = 500)
+    fun `notEqual predicate (exactMatch=false) uses NOT_EQUAL_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.notEqual(stringValue, exactMatch = false)).usesOperator(EqualityComparisonOperator.NOT_EQUAL_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `lessThan predicate uses LESS_THAN operator`() {
+        assertThat(fieldInfo.lessThan(stringValue)).usesOperator(BinaryComparisonOperator.LESS_THAN)
+    }
+
+    @Test(timeout = 500)
+    fun `lessThanOrEqual predicate uses LESS_THAN_OR_EQUAL operator`() {
+        assertThat(fieldInfo.lessThanOrEqual(stringValue)).usesOperator(BinaryComparisonOperator.LESS_THAN_OR_EQUAL)
+    }
+
+    @Test(timeout = 500)
+    fun `greaterThan predicate uses GREATER_THAN operator`() {
+        assertThat(fieldInfo.greaterThan(stringValue)).usesOperator(BinaryComparisonOperator.GREATER_THAN)
+    }
+
+    @Test(timeout = 500)
+    fun `greaterThanOrEqual predicate uses GREATER_THAN_OR_EQUAL operator`() {
+        assertThat(fieldInfo.greaterThanOrEqual(stringValue)).usesOperator(BinaryComparisonOperator.GREATER_THAN_OR_EQUAL)
+    }
+
+    @Test(timeout = 500)
+    fun `in predicate uses IN operator`() {
+        assertThat(fieldInfo.`in`(listValue)).usesOperator(CollectionOperator.IN)
+    }
+
+    @Test(timeout = 500)
+    fun `in predicate (exactMatch=false) uses IN_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.`in`(listValue, exactMatch = false)).usesOperator(CollectionOperator.IN_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `notIn predicate uses NOT_IN operator`() {
+        assertThat(fieldInfo.notIn(listValue)).usesOperator(CollectionOperator.NOT_IN)
+    }
+
+    @Test(timeout = 500)
+    fun `notIn predicate (exactMatch=false) uses NOT_IN_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.notIn(listValue, exactMatch = false)).usesOperator(CollectionOperator.NOT_IN_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `like predicate uses LIKE operator`() {
+        assertThat(fieldInfo.like(stringValue)).usesOperator(LikenessOperator.LIKE)
+    }
+
+    @Test(timeout = 500)
+    fun `like predicate (exactMatch=false) uses LIKE_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.like(stringValue, exactMatch = false)).usesOperator(LikenessOperator.LIKE_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `notLike predicate uses NOT_LIKE operator`() {
+        assertThat(fieldInfo.notLike(stringValue)).usesOperator(LikenessOperator.NOT_LIKE)
+    }
+
+    @Test(timeout = 500)
+    fun `notLike predicate (exactMatch=false) uses NOT_LIKE_IGNORE_CASE operator`() {
+        assertThat(fieldInfo.notLike(stringValue, exactMatch = false)).usesOperator(LikenessOperator.NOT_LIKE_IGNORE_CASE)
+    }
+
+    @Test(timeout = 500)
+    fun `isNull predicate uses IS_NULL operator`() {
+        assertThat(fieldInfo.isNull()).usesOperator(NullOperator.IS_NULL)
+    }
+
+    @Test(timeout = 500)
+    fun `notNull predicate uses NOT_NULL operator`() {
+        assertThat(fieldInfo.notNull()).usesOperator(NullOperator.NOT_NULL)
+    }
+}

docs/source/changelog.rst

@@ -111,6 +111,8 @@ Unreleased
 
 * Added new node configuration option to exclude packages from Quasar instrumentation.
 
+* Fixed the operator used by the ``notEqual`` predicate
+
 .. _changelog_v4.1:
 
 Version 4.1

docs/source/contributing-philosophy.rst

@@ -11,8 +11,7 @@ modern business transactions.  It is unique in its aim to build a platform for b
 retaining strict privacy. Corda provides an implementation of this vision in a code base which others are free to build on, contribute to
 or innovate around. The mission of Corda is further detailed in the `Corda introductory white paper`_.
 
-The project is supported and maintained by the `R3 Alliance <https://www.r3.com>`_, or R3 for short, which consists of over two hundred firms
-working together to build and maintain this open source enterprise-grade blockchain platform.
+The project is supported and maintained by `R3 <https://www.r3.com>`_.
 
 Community Locations
 -------------------
@@ -44,12 +43,12 @@ Community maintainers
 ^^^^^^^^^^^^^^^^^^^^^
 Current community maintainers:
 
-* `Joel Dudley <https://github.com/joeldudleyr3>`_ - Contact me:
+* `Rick Parker <https://github.com/rick-r3>`_ - Contact me:
 
     * On the `Corda Slack team <http://slack.corda.net/>`_, either in the ``#community`` channel or by direct message using the handle
-      ``@joel``
+      ``@parkri``
 
-    * By email: joel.dudley at r3.com
+    * By email: rick.parker at r3.com
 
 We anticipate additional maintainers joining the project in the future from across the community.
 
@@ -60,13 +59,12 @@ Over two hundred individuals have contributed to the development of Corda. You c
 
 Transparency and Conflict Policy
 --------------------------------
-The project is supported and maintained by the `R3 Alliance <https://www.r3.com>`_, which consists of over two hundred firms working together
-to build and maintain this open source enterprise-grade blockchain platform. We develop in the open and publish our
+The project is supported and maintained by `R3 <https://www.r3.com>`_. We develop in the open and publish our
 `Jira <https://r3-cev.atlassian.net/projects/CORDA/summary>`_ to give everyone visibility. R3 also maintains and distributes a commercial
 distribution of Corda. Our vision is that distributions of Corda be compatible and interoperable, and our contribution and code review
 guidelines are designed in part to enable this.
 
-As the R3 Alliance is maintainer of the project and also develops a commercial distribution of Corda, what happens if a member of the
+As R3 is maintainer of the project and also develops a commercial distribution of Corda, what happens if a member of the
 community contributes a feature which the R3 team have implemented only in their commercial product? How is this apparent conflict managed?
 Our approach is simple: if the contribution meets the standards for the project (see above), then the existence of a competing commercial
 implementation will not be used as a reason to reject it. In other words, it is our policy that should a community feature be contributed

docs/source/key-concepts-djvm.rst

@@ -16,14 +16,6 @@ So, what does it mean for a piece of code to be fully deterministic?  Ultimately
 as a function, is pure. In other words, given the same set of inputs, it will always produce the same set of outputs
 without inflicting any side-effects that might later affect the computation.
 
-.. important:: The code in the DJVM module has not yet been integrated with the rest of the platform.  It will eventually become a
-   part of the node and enforce deterministic and secure execution of smart contract code, which is mobile and may
-   propagate around the network without human intervention.
-
-   Currently, it stands alone as an evaluation version. We want to give developers the ability to start trying it out and
-   get used to developing deterministic code under the set of constraints that we envision will be placed on contract code
-   in the future.
-
 Non-Determinism
 ~~~~~~~~~~~~~~~
 
@@ -272,34 +264,73 @@ The DJVM doesn't support multi-threading and so synchronised methods and code bl
 use in sandboxed code. Consequently, we automatically transform them into ordinary methods and code blocks instead.
 
 
-Future Work
-~~~~~~~~~~~
+Trying out the DJVM
+~~~~~~~~~~~~~~~~~~~
 
-Further work is planned:
+.. warning:: The code in the DJVM module is still a beta release. It has been partially integrated with Corda to allow contract
+  verification. However, DJVM-enabled nodes cannot yet participate in a general Corda network containing nodes that do not use the DJVM. It
+  is provided to allow developers to try out the DJVM and experiment with developing deterministic code under the set of constraints that
+  we envision will be placed on contract code in the future.
 
- * To enable controlled use of reflection APIs.
+Tweaking Your Contract Code
+...........................
 
- * Currently, dynamic invocation is disallowed. Allow specific lambda and
-   string concatenation meta-factories used by Java code itself.
+CorDapp developers may need to tweak their contract CorDapps for use inside the DJVM. This is because not every class, constructor or
+method defined in the ``corda-core`` and ``corda-serialization`` modules is available when running inside the sandbox.
 
- * Map more mathematical operations to use their 'exact' counterparts.
+During development, you can choose to compile individual CorDapp modules against the DJVM by defining the following
+``deterministic.gradle`` script plugin:
 
- * General tightening of the enforced constraints.
+.. code-block:: shell
 
- * Cost accounting of runtime metrics such as memory allocation, branching and
-   exception handling. More specifically defining sensible runtime thresholds
-   and make further improvements to the instrumentation.
+  configurations {
+      compileClasspath { Configuration c -> deterministic(c) }
+  }
 
- * More sophisticated runtime accounting as discussed in `Runtime Costing`_.
+  private final void deterministic(Configuration configuration) {
+      if (configuration.state == Configuration.State.UNRESOLVED) {
+          // Ensure that this module uses the deterministic Corda artifacts.
+          configuration.resolutionStrategy.dependencySubstitution {
+              substitute module("$corda_release_group:corda-serialization") with module("$corda_release_group:corda-serialization-deterministic:$corda_release_version")
+              substitute module("$corda_release_group:corda-core") with module("$corda_release_group:corda-core-deterministic:$corda_release_version")
+          }
+      }
+  }
 
+And applying it to individual modules of your CorDapp using:
 
-Command-line Tool
-~~~~~~~~~~~~~~~~~
+.. code-block:: shell
+
+  apply from: "${rootProject.projectDir}/deterministic.gradle"
+
+Uses of Corda's core or serialization APIs that are unavailable inside the sandbox will then cause compilation errors.
+
+Note however that successful compilation against ``corda-core-deterministic`` and ``corda-serialization-deterministic`` is 
+not sufficient. The only way to be sure that a piece of code is deterministic is to actually run it inside a DJVM sandbox, 
+as described below.
+
+Enabling Use of the DJVM for a Node
+...................................
+
+You can enable the DJVM for your node by adding the following line to your node's ``node.conf`` file:
+
+.. code-block:: shell
+
+  systemProperties = { "net.corda.djvm" = true }
+
+This will cause your node to sandbox every call to ``Contract.verify``. If your transaction contains a source of non-determinism,
+transaction verification will fail.
+
+Alternatively, you can enable the DJVM when creating nodes via DemoBench by ticking the ``Deterministic Contract Verification`` checkbox
+when creating the initial notary node.
+
+Using the Command-line Tool
+...........................
 
 You can download and unpack ``corda-djvm-cli.zip`` from the R3 Artifactory.
 Alternatively, you can build it yourself from the source as follows.
 
-Open your terminial and clone the DJVM repository from GitHub:
+Open your terminal and clone the DJVM repository from GitHub:
 
 .. code-block:: shell
 

docs/source/release-notes.rst

@@ -51,6 +51,26 @@ Security enhancements
 * The ability to SSH into the standalone shell has been removed
 * A new read-only RPC user role template has been documented in :doc:`shell`
 
+Changes to integration testing
++++++++++++++++++++++++
+
+The "out-of-process" nodes spawned through Driver DSL (see :doc:`tutorial-integration-testing`) will no longer accidentally contain your CorDapps on their application classpath. The list of items that will be automatically filtered out include:
+
+* Directories (only regular files are allowed)
+* Jars with Maven classifiers ``tests`` or ``test``
+* Jars with any Cordapp attributes in their manifests (any of those listed in :doc:`cordapp-build-systems` or ``Target-Platform-Version`` and ``Min-Platform-Version`` if both are present)
+* Jars with the ``Corda-Testing`` attribute in their manifests. The manifest of the following artifacts has been updated to include the ``Corda-Testing`` attribute:
+
+	* ``corda-node-driver``
+	* ``corda-test-utils``
+	* ``corda-test-common``
+	* ``corda-test-db``
+	* ``corda-mock``
+
+* Files whose names start with ``corda-mock``, ``junit``, ``testng`` or ``mockito``
+
+Some of your existing integration tests might implicitly be relying on the presence of the above files, so please keep this in mind when upgrading your version of Corda.
+
 
 Platform version change
 ~~~~~~~~~~~~~~~~~~~~~~~
@@ -59,6 +79,10 @@ Given the addition of new APIs, the platform version of Corda 4.4 has been bumpe
 
 For more information on platform version, please see :doc:`versioning`. For more details on upgrading a CorDapp to use platform version 5, please see :doc:`app-upgrade-notes`.
 
+Known Issues
+~~~~~~~~~~~~
+
+Changes introduced in Corda 4.4 to increase ledger integrity have highlighted limitations regarding database transactions. To prevent flows from continuing to process after a database transaction has failed to commit or suffered from a pre-commit persistence exception, extra database flushes have been added. These extra flushes can cause exceptions to be thrown where they were not before (or cause different exception types to be raised compared to Corda 4.3 or previous versions). In general, CorDapp developers should not expect to be able to catch exceptions thrown during a database transaction and then continue with further DB operations as part of the same flow. A safer pattern involves allowing the flow to fail and be retried
 
 Issues Fixed
 ~~~~~~~~~~~~

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 #Wed Aug 21 10:48:19 BST 2019
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
+distributionUrl=https\://gradleproxy:gradleproxy@software.r3.com/artifactory/gradle-proxy/gradle-5.4.1-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists