CORDA-2405 Update versions of whitelisted attachments (#4549)

The version of contract attachments that are whitelisted should be read from NetworkParameters.whitelistedContractImplementations.
It use the lattes network map from db with the highest epoch.

docs/source/api-contract-constraints.rst

@@ -240,10 +240,15 @@ The following controls are enforced for these different types of jars within the
 - Signed contract JARs must be uniquely versioned per contract class (or group of).
   At runtime the node will throw a `DuplicateContractClassException`` exception if this condition is violated.
 
-- Unsigned contract JARs: there may exist multiple instances of the same contract jar.
+- Unsigned contract JARs: there should not exist multiple instances of the same contract jar.
+  When a whitelisted JARs is imported and it doesn't contain a version number, the version will be copied from the position (counting from 1)
+  of this JAR in the whilelist. The same JAR can be present in many lists (if it contains many contracts),
+  in such case the version will be equal to the highest position of the JAR in all lists.
+  The new whitelist needs to be distributed to the node before the JAR is imported, otherwise it will receive default version.
   At run-time the node will warn of duplicates encountered.
   The most recent version given by insertionDate into the attachment storage will be used upon transaction building/resolution.
 
+
 Issues when using the HashAttachmentConstraint
 ----------------------------------------------
 

node-api/src/main/kotlin/net/corda/nodeapi/internal/persistence/AttachmentVersionNumberMigration.kt

@@ -0,0 +1,102 @@
+package net.corda.nodeapi.internal.persistence
+
+import  liquibase.change.custom.CustomTaskChange
+import liquibase.database.Database
+import liquibase.database.jvm.JdbcConnection
+import liquibase.exception.ValidationErrors
+import liquibase.resource.ResourceAccessor
+import net.corda.core.node.NetworkParameters
+import net.corda.core.node.services.AttachmentId
+import net.corda.core.serialization.deserialize
+import net.corda.core.utilities.contextLogger
+
+class AttachmentVersionNumberMigration : CustomTaskChange {
+    companion object {
+        private val logger = contextLogger()
+    }
+
+    override fun execute(database: Database?) {
+        val connection = database?.connection as JdbcConnection
+        try {
+            logger.debug("Start executing...")
+            val networkParameters = getNetworkParameters(connection)
+            if (networkParameters == null) {
+                logger.debug("Network parameters not found.")
+                return
+            } else {
+                logger.debug("Network parameters epoch: ${networkParameters.epoch}, whitelistedContractImplementations: ${networkParameters.whitelistedContractImplementations}.")
+            }
+            val availableAttachments = getAttachmentsWithDefaultVersion(connection)
+            if (availableAttachments.isEmpty()) {
+                logger.debug("Attachments not found.")
+                return
+            } else {
+                logger.debug("Attachments with version '1': $availableAttachments")
+            }
+
+            availableAttachments.forEach { attachmentId ->
+                val versions = networkParameters?.whitelistedContractImplementations?.values.mapNotNull { it.indexOfFirst { it.toString() == attachmentId} }.filter { it >= 0 }
+                val maxPosition = versions.max() ?: 0
+                if (maxPosition > 0) {
+                    val version = maxPosition + 1
+                    val msg = "Updating version of attachment $attachmentId to '$version'"
+                    if (versions.toSet().size > 1)
+                        logger.warn("Several versions based on whitelistedContractImplementations position are available: ${versions.toSet()}. $msg")
+                    else
+                        logger.debug(msg)
+                    updateVersion(connection, attachmentId, version)
+                }
+            }
+
+            logger.debug("Done")
+        } catch (e: Exception) {
+            logger.error("Exception while retrieving network parameters ${e.message}", e)
+        }
+    }
+
+    override fun validate(database: Database?): ValidationErrors? {
+        return null
+    }
+
+    override fun getConfirmationMessage(): String? {
+        return null
+    }
+
+    override fun setFileOpener(resourceAccessor: ResourceAccessor?) {
+    }
+
+    override fun setUp() {
+    }
+
+    private fun getNetworkParameters(connection: JdbcConnection): NetworkParameters? =
+            connection.createStatement().use {
+                val rs = it.executeQuery("SELECT PARAMETERS_BYTES FROM NODE_NETWORK_PARAMETERS ORDER BY EPOCH DESC")
+                if (rs.next()) {
+                    val networkParametersBytes = rs.getBytes(1) as ByteArray
+                    val networkParameters: NetworkParameters = networkParametersBytes.deserialize()
+                    rs.close()
+                    networkParameters
+                } else
+                    null
+            }
+
+    private fun getAttachmentsWithDefaultVersion(connection: JdbcConnection): List<String> =
+            connection.createStatement().use {
+                val attachments = mutableListOf<String>()
+                val rs = it.executeQuery("SELECT ATT_ID FROM NODE_ATTACHMENTS WHERE VERSION = 1")
+                while (rs.next()) {
+                    val elem = rs.getString(1)
+                    attachments.add(elem)
+                }
+                rs.close()
+                attachments
+            }
+
+    private fun updateVersion(connection: JdbcConnection, attachmentId: String, version: Int) {
+        connection.prepareStatement("UPDATE NODE_ATTACHMENTS SET VERSION = ? WHERE ATT_ID = ?").use {
+            it.setInt(1, version)
+            it.setString(2, attachmentId)
+            it.executeUpdate()
+        }
+    }
+}

node/src/main/kotlin/net/corda/node/services/persistence/NodeAttachmentService.kt

@@ -322,6 +322,21 @@ class NodeAttachmentService(
         }
     }
 
+    private fun increaseDefaultVersionIfWhitelistedAttachment(contractClassNames: List<ContractClassName>, contractVersionFromFile: Int, attachmentId : AttachmentId) =
+        if (contractVersionFromFile == DEFAULT_CORDAPP_VERSION) {
+            val versions = contractClassNames.mapNotNull { servicesForResolution.networkParameters.whitelistedContractImplementations[it]?.indexOf(attachmentId) }.filter { it >= 0 }.map { it + 1 } // +1 as versions starts from 1 not 0
+            val max = versions.max()
+            if (max != null && max > contractVersionFromFile) {
+                val msg = "Updating version of attachment $attachmentId from '$contractVersionFromFile' to '$max'"
+                if (versions.toSet().size > 1)
+                    log.warn("Several versions based on whitelistedContractImplementations position are available: ${versions.toSet()}. $msg")
+                else
+                    log.debug(msg)
+                max
+            } else contractVersionFromFile
+        }
+        else contractVersionFromFile
+
     // TODO: PLT-147: The attachment should be randomised to prevent brute force guessing and thus privacy leaks.
     private fun import(jar: InputStream, uploader: String?, filename: String?): AttachmentId {
         return database.transaction {
@@ -339,7 +354,7 @@ class NodeAttachmentService(
                 if (!hasAttachment(id)) {
                     checkIsAValidJAR(bytes.inputStream())
                     val jarSigners = getSigners(bytes)
-                    val contractVersion = getVersion(bytes)
+                    val contractVersion = increaseDefaultVersionIfWhitelistedAttachment(contractClassNames, getVersion(bytes), id)
                     val session = currentDBSession()
 
                     verifyVersionUniquenessForSignedAttachments(contractClassNames, contractVersion, jarSigners)
@@ -477,7 +492,7 @@ class NodeAttachmentService(
         val signed = it.value.filter { it.signers?.isNotEmpty() ?: false }.map { AttachmentId.parse(it.attId) }
         check (signed.size <= 1) //sanity check
         val unsigned = it.value.filter { it.signers?.isEmpty() ?: true }.map { AttachmentId.parse(it.attId) }
-        if (unsigned.size > 1) //TODO cater better for whiltelisted JARs - CORDA-2405
+        if (unsigned.size > 1)
             log.warn("Selecting attachment ${unsigned.first()} from duplicated, unsigned attachments ${unsigned.map { it.toString() }} for contract $contractClassName version '${it.key}'.")
         return it.key to AttachmentIds(signed.singleOrNull(), unsigned.firstOrNull())
     }

node/src/main/resources/migration/node-core.changelog-master.xml

@@ -14,5 +14,5 @@
     <include file="migration/node-core.changelog-tx-mapping.xml"/>
     <include file="migration/node-core.changelog-v9.xml"/>
     <include file="migration/node-core.changelog-v10.xml"/>
-
+    <include file="migration/node-core.changelog-v11.xml"/>
 </databaseChangeLog>

node/src/main/resources/migration/node-core.changelog-v11.xml

@@ -0,0 +1,11 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd"
+                   logicalFilePath="migration/node-services.changelog-init.xml">
+
+    <changeSet author="R3.Corda" id="update-version-of-whitelisted-jars">
+        <customChange class="net.corda.nodeapi.internal.persistence.AttachmentVersionNumberMigration">
+        </customChange>
+    </changeSet>
+</databaseChangeLog>

node/src/test/java/net/corda/node/services/vault/VaultQueryJavaTests.java

@@ -11,6 +11,7 @@ import net.corda.core.identity.AbstractParty;
 import net.corda.core.identity.CordaX500Name;
 import net.corda.core.identity.Party;
 import net.corda.core.messaging.DataFeed;
+import net.corda.core.node.ServicesForResolution;
 import net.corda.core.node.services.AttachmentStorage;
 import net.corda.core.node.services.IdentityService;
 import net.corda.core.node.services.Vault;
@@ -40,7 +41,6 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
-import org.mockito.Mockito;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -59,12 +59,13 @@ import static net.corda.core.node.services.vault.Builder.equal;
 import static net.corda.core.node.services.vault.Builder.sum;
 import static net.corda.core.node.services.vault.QueryCriteriaUtils.*;
 import static net.corda.core.utilities.ByteArrays.toHexString;
+import static net.corda.testing.common.internal.ParametersUtilitiesKt.testNetworkParameters;
 import static net.corda.testing.core.internal.ContractJarTestUtils.INSTANCE;
 import static net.corda.testing.core.TestConstants.*;
-import static net.corda.testing.internal.RigorousMockKt.rigorousMock;
 import static net.corda.testing.node.MockServices.makeTestDatabaseAndMockServices;
 import static net.corda.testing.node.MockServicesKt.makeTestIdentityService;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.mock;
 
 public class VaultQueryJavaTests {
@@ -97,6 +98,9 @@ public class VaultQueryJavaTests {
         vaultFiller = new VaultFiller(services, DUMMY_NOTARY);
         vaultService = services.getVaultService();
         storage = new NodeAttachmentService(new MetricRegistry(), new TestingNamedCacheFactory(100), database);
+        ServicesForResolution serviceForResolution = mock(ServicesForResolution.class);
+        ((NodeAttachmentService) storage).servicesForResolution = serviceForResolution;
+        doReturn(testNetworkParameters()).when(serviceForResolution).getNetworkParameters();
     }
 
     @After

node/src/test/kotlin/net/corda/node/services/persistence/NodeAttachmentServiceTest.kt

@@ -4,6 +4,8 @@ import co.paralleluniverse.fibers.Suspendable
 import com.codahale.metrics.MetricRegistry
 import com.google.common.jimfs.Configuration
 import com.google.common.jimfs.Jimfs
+import com.nhaarman.mockito_kotlin.doReturn
+import com.nhaarman.mockito_kotlin.whenever
 import net.corda.core.contracts.ContractAttachment
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.sha256
@@ -21,14 +23,12 @@ import net.corda.nodeapi.exceptions.DuplicateAttachmentException
 import net.corda.nodeapi.exceptions.DuplicateContractClassException
 import net.corda.nodeapi.internal.persistence.CordaPersistence
 import net.corda.nodeapi.internal.persistence.DatabaseConfig
+import net.corda.testing.common.internal.testNetworkParameters
 import net.corda.testing.core.internal.ContractJarTestUtils.makeTestContractJar
 import net.corda.testing.core.internal.ContractJarTestUtils.makeTestJar
 import net.corda.testing.core.internal.ContractJarTestUtils.makeTestSignedContractJar
 import net.corda.testing.core.internal.SelfCleaningDir
-import net.corda.testing.internal.LogHelper
-import net.corda.testing.internal.TestingNamedCacheFactory
-import net.corda.testing.internal.configureDatabase
-import net.corda.testing.internal.rigorousMock
+import net.corda.testing.internal.*
 import net.corda.testing.node.MockServices.Companion.makeTestDataSourceProperties
 import net.corda.testing.node.internal.InternalMockNetwork
 import net.corda.testing.node.internal.startFlow
@@ -54,7 +54,9 @@ class NodeAttachmentServiceTest {
     private lateinit var fs: FileSystem
     private lateinit var database: CordaPersistence
     private lateinit var storage: NodeAttachmentService
-    private val services = rigorousMock<ServicesForResolution>()
+    private val services = rigorousMock<ServicesForResolution>().also {
+        doReturn(testNetworkParameters()).whenever(it).networkParameters
+    }
 
     @Before
     fun setUp() {

testing/test-common/src/main/kotlin/net/corda/testing/common/internal/ParametersUtilities.kt

@@ -9,6 +9,7 @@ import java.security.PublicKey
 import java.time.Duration
 import java.time.Instant
 
+@JvmOverloads
 fun testNetworkParameters(
         notaries: List<NotaryInfo> = emptyList(),
         minimumPlatformVersion: Int = 1,