CORDA-3823: hash agility updates for rc03 (#6800)

* wip

* wip

* wip (need to review IEE comments)

* wip

* wip

* Small refactoring, fixed network-verifier's TestNotaryFlow

* Added command line option to explicitly enable hash agility support

* wip-do-not-push

* wip

* wip

* wip

* aligned merkletree/transaction hash algorithms

* wip

* Added mixed algorithm support for nodes vs leaves and corrected mixed algorithm tests

* moved global computeNonce and componentHash to DigestService

* added comment for failing test to fix

* wip

* Minor cleanups, added deprecated componentHash/computeNonce

* restored exploratory changes to failing SignedTransaction test

* cleaned up and minor rafactoring

* Fixed some tests with hardcoded hash algorithm

* some changes and cleanups following code review

* WIP commit before large change

* WIP Fixed 3 tests

* WIP removed direct references to randomSHA256() and sha256()

* Updated/added liquibase migrations to support larger hash algorithms

* Reviewed, cleanups, comments, fixes

* removing direct references to sha256()

* WIP verifying obligations test errors

* reviewing obligation/attachment issues with sha3_256

* Full review before PR - intermediate commits

* Reviewed and cleaned up

* Futher cleanup

* Fixed partial tree backward compatible json and cleanups

* all tests passing

* Removed couple of unused imports

* Reworked global componentHash function to avoid deprecated warnings

* replaced SHA3s with some alternate SHA2s

* Removed SHA3-256 and SHA3-512 references

* fixed some tests using non ubiquitous hash algorithms

* Fixed ABI compatibility (not for TransactionBuilder)

* Fixed ABI compatibility to TransactionBuilder

* couple of fixes

* fixed DigestService's randomHash

* Removed constructor with loosely typed args for private constructor of LedgerTransaction class (API removal)

* re-introduced LedgerTransaction deprecated ctor for deserialization

* Add possibility to load CustomMessageDigest bypassing JCA (#6798)

* Change api-current for DigestAlgorithm

* disable flaky tests

* addressed liquibase migration script versions

* Removed TODOs and cleanups

* relaxed privacy salt validation

* Fixed privacy salt test to comply with relaxed validation

* detekt and privacySalt validation

* diff cleanup

* diff cleanup

* removed unused import

* removed PrivacySalt's validateFor method and references

* removed invalid character

Co-authored-by: Denis Rekalov <denis.rekalov@r3.com>

core-deterministic/testing/src/test/kotlin/net/corda/deterministic/contracts/PrivacySaltTest.kt

@@ -25,10 +25,4 @@ class PrivacySaltTest {
         val ex = assertFailsWith<IllegalArgumentException> { PrivacySalt(ByteArray(SALT_SIZE - 1) { 0x7f }) }
         assertEquals("Privacy salt should be at least 32 bytes.", ex.message)
     }
-
-    @Test(timeout=300_000)
-	fun testTooShortPrivacySaltForSHA512() {
-        val ex = assertFailsWith<IllegalArgumentException> { PrivacySalt(ByteArray(SALT_SIZE) { 0x7f }).apply { validateFor("SHA-512") } }
-        assertEquals("Privacy salt should be at least 64 bytes for SHA-512.", ex.message)
-    }
 }

core-tests/src/test/kotlin/net/corda/coretests/transactions/CompatibleTransactionTests.kt

@@ -417,8 +417,6 @@ class CompatibleTransactionTests {
     @Test(timeout=300_000)
 	fun `FilteredTransaction signer manipulation tests`() {
         // Required to call the private constructor.
-        //val ftxConstructor = FilteredTransaction::class.constructors.first()
-        // TODO(iee): verify if it was a hard requirement that the constructor must be the first()
         val ftxConstructor = FilteredTransaction::class.constructors.last()
 
         // 1st and 3rd commands require a signature from KEY_1.

core/src/main/kotlin/net/corda/core/contracts/Structures.kt

@@ -340,11 +340,6 @@ class PrivacySalt(bytes: ByteArray) : OpaqueBytes(bytes) {
         require(bytes.size >= MINIMUM_SIZE) { "Privacy salt should be at least $MINIMUM_SIZE bytes." }
     }
 
-    fun validateFor(algorithm: String) {
-        val digestLength = SecureHash.digestLengthFor(algorithm)
-        require(bytes.size >= digestLength) { "Privacy salt should be at least $digestLength bytes for $algorithm." }
-    }
-
     companion object {
         private const val MINIMUM_SIZE = 32
 

core/src/main/kotlin/net/corda/core/contracts/TransactionVerificationException.kt

@@ -342,7 +342,6 @@ abstract class TransactionVerificationException(val txId: SecureHash, message: S
                     "At this time these are not loadable because the DJVM sandbox has not yet been integrated. " +
                     "You will need to manually install the CorDapp to whitelist it for use.")
 
-    // TODO(iee): verify if this is an acceptable exception
     @KeepForDJVM
     class UnsupportedHashTypeException(txId: SecureHash) : TransactionVerificationException(txId, "The transaction Id is defined by an unsupported hash type", null);
 

core/src/main/kotlin/net/corda/core/internal/TransactionUtils.kt

@@ -5,7 +5,6 @@ import net.corda.core.contracts.*
 import net.corda.core.crypto.DigestService
 import net.corda.core.crypto.SecureHash
 import net.corda.core.crypto.algorithm
-import net.corda.core.crypto.hashAs
 import net.corda.core.crypto.internal.DigestAlgorithmFactory
 import net.corda.core.flows.FlowLogic
 import net.corda.core.identity.Party
@@ -50,19 +49,12 @@ class ContractUpgradeTransactionBuilder(
 }
 
 /** Concatenates the hash components into a single [ByteArray] and returns its hash. */
-fun combinedHash(components: Iterable<SecureHash>/*, digestService: DigestService = DigestService.default*/): SecureHash {
+fun combinedHash(components: Iterable<SecureHash>, digestService: DigestService): SecureHash {
     val stream = ByteArrayOutputStream()
     components.forEach {
         stream.write(it.bytes)
     }
-    // TODO(iee): need to re-visit and review this code to understand which is the right
+    return digestService.hash(stream.toByteArray());
-    //            way to combine the hashes. Is this meant to match a pre-existing tx id,
-    //            or create a new tx id with the [default] hash algorithm from whatever
-    //            components are passed in and independently from their algorithm?
-    //            This is used to build the tx id of ContractUpgradeFilteredTransaction and
-    //            ContractUpgradeWireTransaction
-    return stream.toByteArray().hashAs(components.first().algorithm)
-    //return digestService.hash(stream.toByteArray());
 }
 
 /**

core/src/main/kotlin/net/corda/core/transactions/ContractUpgradeTransactions.kt

@@ -88,7 +88,6 @@ data class ContractUpgradeWireTransaction(
     init {
         check(inputs.isNotEmpty()) { "A contract upgrade transaction must have inputs" }
         checkBaseInvariants()
-        // privacySalt.validateFor(digestService.hashAlgorithm)
     }
 
     /**
@@ -114,7 +113,7 @@ data class ContractUpgradeWireTransaction(
         val componentHashes = serializedComponents.mapIndexed { index, component ->
             digestService.componentHash(nonces[index], component)
         }
-        combinedHash(componentHashes/*, digestService*/)
+        combinedHash(componentHashes, digestService)
     }
 
     /** Required for filtering transaction components. */
@@ -211,7 +210,7 @@ data class ContractUpgradeFilteredTransaction(
          * Required for computing the transaction id.
          */
         val hiddenComponents: Map<Int, SecureHash>,
-        val digestService: DigestService = DigestService.sha2_256
+        val digestService: DigestService
 ) : CoreTransaction() {
 
     /**
@@ -250,7 +249,7 @@ data class ContractUpgradeFilteredTransaction(
                 else -> throw IllegalStateException("Missing component hashes")
             }
         }
-        combinedHash(hashList/*, digestService*/)
+        combinedHash(hashList, digestService)
     }
     override val outputs: List<TransactionState<ContractState>> get() = emptyList()
     override val references: List<StateRef> get() = emptyList()

core/src/main/kotlin/net/corda/core/transactions/LedgerTransaction.kt

@@ -92,7 +92,7 @@ private constructor(
         private val isAttachmentTrusted: (Attachment) -> Boolean,
         private val verifierFactory: (LedgerTransaction, ClassLoader) -> Verifier,
         private val attachmentsClassLoaderCache: AttachmentsClassLoaderCache?,
-        val digestService: DigestService = DigestService.sha2_256
+        val digestService: DigestService
 ) : FullTransaction() {
 
     /**
@@ -120,13 +120,6 @@ private constructor(
             networkParameters, references, componentGroups, serializedInputs, serializedReferences,
             isAttachmentTrusted, verifierFactory, attachmentsClassLoaderCache, DigestService.sha2_256)
 
-    // TODO(iee): add missing => Removed from API txt
-    //  public <init>(java.util.List, java.util.List, java.util.List, java.util.List, net.corda.core.crypto.SecureHash,
-    //      net.corda.core.identity.Party, net.corda.core.contracts.TimeWindow, net.corda.core.contracts.PrivacySalt,
-    //      net.corda.core.node.NetworkParameters, java.util.List, java.util.List, java.util.List, java.util.List,
-    //      kotlin.jvm.functions.Function1, kotlin.jvm.functions.Function2,
-    //      net.corda.core.serialization.internal.AttachmentsClassLoaderCache, kotlin.jvm.internal.DefaultConstructorMarker)
-
     init {
         if (timeWindow != null) check(notary != null) { "Transactions with time-windows must be notarised" }
         checkNotaryWhitelisted()

core/src/main/kotlin/net/corda/core/transactions/NotaryChangeTransactions.kt

@@ -50,14 +50,6 @@ data class NotaryChangeWireTransaction(
         return NotaryChangeWireTransaction(serializedComponents, DigestService.sha2_256)
     }
 
-    // TODO(iee): add missing:
-    //  public <init>(net.corda.core.identity.Party, java.util.UUID, java.util.List<net.corda.core.contracts.StateRef>,
-    //      java.util.List<net.corda.core.crypto.SecureHash>,
-    //      java.util.List<net.corda.core.contracts.TransactionState<net.corda.core.contracts.ContractState>>,
-    //      java.util.List<net.corda.core.contracts.Command<?>>, net.corda.core.contracts.TimeWindow,
-    //      net.corda.core.contracts.PrivacySalt, java.util.List<net.corda.core.contracts.StateRef>,
-    //      net.corda.core.node.ServiceHub)
-
     override val inputs: List<StateRef> = serializedComponents[INPUTS.ordinal].deserialize()
     override val references: List<StateRef> = emptyList()
     override val notary: Party = serializedComponents[NOTARY.ordinal].deserialize()

core/src/main/kotlin/net/corda/core/transactions/WireTransaction.kt

@@ -80,8 +80,6 @@ class WireTransaction(componentGroups: List<ComponentGroup>, val privacySalt: Pr
         check(inputs.isNotEmpty() || outputs.isNotEmpty()) { "A transaction must contain at least one input or output state" }
         check(commands.isNotEmpty()) { "A transaction must contain at least one command" }
         if (timeWindow != null) check(notary != null) { "Transactions with time-windows must be notarised" }
-        // TODO(iee): review salt validation - Should we just warn when privacy bytes already >= 32 but <= digestLength?
-        // privacySalt.validateFor(digestService.hashAlgorithm)
     }
 
     /** The transaction id is represented by the root hash of Merkle tree over the transaction components. */

finance/contracts/src/test/kotlin/net/corda/finance/contracts/asset/ObligationTests.kt

@@ -58,8 +58,6 @@ class ObligationTests {
         val MEGA_CORP_PUBKEY get() = megaCorp.publicKey
         val MINI_CORP get() = miniCorp.party
         val MINI_CORP_PUBKEY get() = miniCorp.publicKey
-        // TODO(iee): Obligations fail to find attachments if we use anything else than sha2_256. All attachment logic is still
-        //            using direct calls to .sha256() which still needs to be replaced with DigestService.default
     }
 
     @Rule

node-api/src/main/kotlin/net/corda/nodeapi/internal/serialization/kryo/Kryo.kt

@@ -221,7 +221,6 @@ object WireTransactionSerializer : Serializer<WireTransaction>() {
     override fun read(kryo: Kryo, input: Input, type: Class<WireTransaction>): WireTransaction {
         val componentGroups: List<ComponentGroup> = uncheckedCast(kryo.readClassAndObject(input))
         val privacySalt = kryo.readClassAndObject(input) as PrivacySalt
-        // TODO(iee): handle backward compatibility when deserializing old version of WTX
         val digestService = kryo.readClassAndObject(input) as? DigestService
         return WireTransaction(componentGroups, privacySalt, digestService ?: DigestService.sha2_256)
     }
@@ -236,7 +235,6 @@ object NotaryChangeWireTransactionSerializer : Serializer<NotaryChangeWireTransa
 
     override fun read(kryo: Kryo, input: Input, type: Class<NotaryChangeWireTransaction>): NotaryChangeWireTransaction {
         val components: List<OpaqueBytes> = uncheckedCast(kryo.readClassAndObject(input))
-        // TODO(iee): handle backward compatibility when deserializing old version of NCWTX
         val digestService = kryo.readClassAndObject(input) as? DigestService
         return NotaryChangeWireTransaction(components, digestService ?: DigestService.sha2_256)
     }
@@ -253,7 +251,6 @@ object ContractUpgradeWireTransactionSerializer : Serializer<ContractUpgradeWire
     override fun read(kryo: Kryo, input: Input, type: Class<ContractUpgradeWireTransaction>): ContractUpgradeWireTransaction {
         val components: List<OpaqueBytes> = uncheckedCast(kryo.readClassAndObject(input))
         val privacySalt = kryo.readClassAndObject(input) as PrivacySalt
-        // TODO(iee): handle backward compatibility when deserializing old version of WTX
         val digestService = kryo.readClassAndObject(input) as? DigestService
         return ContractUpgradeWireTransaction(components, privacySalt, digestService ?: DigestService.sha2_256)
     }

node/src/main/kotlin/net/corda/node/internal/schemas/NodeInfoSchema.kt

@@ -29,11 +29,6 @@ object NodeInfoSchemaV1 : MappedSchema(
             @Column(name = "node_info_id", nullable = false)
             var id: Int,
 
-            // TODO(iee): this field receives a hardcoded SHA2_256 string that comes from SerializationAPI's
-            // SerializedBytes' hash method that calls directly to .sha256(). It appears to be used
-            // only for serialized NodeInfo instances. Requires review and discussion to determine if
-            // it should follow the node/system hash algorithm or remain sha256 (in case it does change,
-            // it will be necessary to change length from 64 to 144.
             @Suppress("MagicNumber") // database column width
             @Column(name = "node_info_hash", length = 64, nullable = false)
             val hash: String,

node/src/main/kotlin/net/corda/node/services/transactions/PersistentUniquenessProvider.kt

@@ -265,7 +265,6 @@ class PersistentUniquenessProvider(val clock: Clock, val database: CordaPersiste
     }
 
     private fun handleConflicts(txId: SecureHash, conflictingStates: LinkedHashMap<StateRef, StateConsumptionDetails>) {
-        // TODO(iee): is the use of reHash correct here? Check backward/forward compatibility
         if (isConsumedByTheSameTx(txId.reHash(), conflictingStates)) {
             log.info("Transaction $txId already notarised. TxId: $txId")
             return

node/src/main/kotlin/net/corda/notary/common/BatchSigning.kt

@@ -25,12 +25,9 @@ fun signBatch(
     require(algorithms.size > 0) {
         "Cannot sign an empty batch"
     }
-    // TODO(iee): too strict? will be valid in the future?
     require(algorithms.size == 1) {
         "Cannot sign a batch with multiple hash algorithms: $algorithms"
     }
-    // TODO(iee): assuming this is running on a notary node, and therefore using only the default
-    //            hash algorithm. Review and discuss.
     val merkleTree = MerkleTree.getMerkleTree(txIds.map { it.reHash() }, services.digestService)
     val merkleTreeRoot = merkleTree.hash
     val signableData = SignableData(