Cleaned up DistributedNotaryTests, including addressing overlap with RaftValidatingNotaryServiceTests

node/src/integration-test/kotlin/net/corda/node/services/DistributedNotaryTests.kt

@@ -1,181 +0,0 @@
-package net.corda.node.services
-
-import com.google.common.net.HostAndPort
-import net.corda.core.contracts.DummyContract
-import net.corda.core.contracts.StateAndRef
-import net.corda.core.contracts.StateRef
-import net.corda.core.contracts.TransactionType
-import net.corda.core.crypto.CompositeKey
-import net.corda.core.crypto.Party
-import net.corda.core.crypto.composite
-import net.corda.core.crypto.generateKeyPair
-import net.corda.core.getOrThrow
-import net.corda.core.messaging.SingleMessageRecipient
-import net.corda.core.node.services.ServiceInfo
-import net.corda.core.random63BitValue
-import net.corda.core.serialization.serialize
-import net.corda.core.utilities.LogHelper
-import net.corda.flows.NotaryError
-import net.corda.flows.NotaryException
-import net.corda.flows.NotaryFlow
-import net.corda.node.internal.AbstractNode
-import net.corda.node.internal.Node
-import net.corda.node.services.config.ConfigHelper
-import net.corda.node.services.config.FullNodeConfiguration
-import net.corda.node.services.network.NetworkMapService
-import net.corda.node.services.transactions.RaftValidatingNotaryService
-import net.corda.node.utilities.databaseTransaction
-import net.corda.testing.freeLocalHostAndPort
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-import java.io.File
-import java.nio.file.Files
-import java.nio.file.Path
-import java.nio.file.Paths
-import java.security.KeyPair
-import java.time.Instant
-import java.time.ZoneOffset
-import java.time.format.DateTimeFormatter
-import java.util.*
-import kotlin.concurrent.thread
-import kotlin.test.assertEquals
-import kotlin.test.assertFailsWith
-
-// TODO: clean up and rewrite this using DriverDSL
-class DistributedNotaryTests {
-    private val folderName = DateTimeFormatter
-            .ofPattern("yyyyMMddHHmmss")
-            .withZone(ZoneOffset.UTC)
-            .format(Instant.now())
-    val baseDir = "build/notaryTest/$folderName"
-    val notaryName = "Notary Service"
-    val clusterSize = 3
-
-    @Before
-    fun setup() {
-        LogHelper.setLevel("-org.apache.activemq")
-        LogHelper.setLevel(NetworkMapService::class)
-        File(baseDir).deleteRecursively()
-        File(baseDir).mkdirs()
-    }
-
-    @After
-    fun tearDown() {
-        LogHelper.reset("org.apache.activemq")
-        LogHelper.reset(NetworkMapService::class)
-        File(baseDir).deleteRecursively()
-    }
-
-    @Test
-    fun `should detect double spend`() {
-        val masterNode = createNotaryCluster()
-        val alice = createAliceNode(masterNode.net.myAddress)
-
-        val notaryParty = alice.netMapCache.getAnyNotary(RaftValidatingNotaryService.type)!!
-
-        val stx = run {
-            val notaryNodeKeyPair = databaseTransaction(masterNode.database) { masterNode.services.notaryIdentityKey }
-            val inputState = issueState(alice, notaryParty, notaryNodeKeyPair)
-            val tx = TransactionType.General.Builder(notaryParty).withItems(inputState)
-            val aliceKey = databaseTransaction(alice.database) { alice.services.legalIdentityKey }
-            tx.signWith(aliceKey)
-            tx.toSignedTransaction(false)
-        }
-
-        val buildFlow = { NotaryFlow.Client(stx) }
-
-        val firstSpend = alice.services.startFlow(buildFlow())
-        firstSpend.resultFuture.getOrThrow()
-
-        val secondSpend = alice.services.startFlow(buildFlow())
-
-        val ex = assertFailsWith(NotaryException::class) { secondSpend.resultFuture.getOrThrow() }
-        val error = ex.error as NotaryError.Conflict
-        assertEquals(error.tx, stx.tx)
-    }
-
-    private fun createNotaryCluster(): Node {
-        val notaryClusterAddress = freeLocalHostAndPort()
-        val keyPairs = (1..clusterSize).map { generateKeyPair() }
-
-        val notaryKeyTree = CompositeKey.Builder().addKeys(keyPairs.map { it.public.composite }).build(1)
-        val notaryParty = Party(notaryName, notaryKeyTree).serialize()
-
-        var networkMapAddress: SingleMessageRecipient? = null
-
-        val cluster = keyPairs.mapIndexed { i, keyPair ->
-            val dir = Paths.get(baseDir, "notaryNode$i")
-            Files.createDirectories(dir)
-
-            val privateKeyFile = RaftValidatingNotaryService.type.id + "-private-key"
-            val publicKeyFile = RaftValidatingNotaryService.type.id + "-public"
-
-            notaryParty.writeToFile(dir.resolve(publicKeyFile))
-            keyPair.serialize().writeToFile(dir.resolve(privateKeyFile))
-
-            val node: Node
-            if (networkMapAddress == null) {
-                val config = generateConfig(dir, "node" + random63BitValue(), notaryClusterAddress)
-                node = createNotaryNode(config)
-                networkMapAddress = node.net.myAddress
-            } else {
-                val config = generateConfig(dir, "node" + random63BitValue(), freeLocalHostAndPort(), notaryClusterAddress)
-                node = createNotaryNode(config, networkMapAddress)
-            }
-
-            node
-        }
-
-        return cluster.first()
-    }
-
-    private fun createNotaryNode(config: FullNodeConfiguration, networkMapAddress: SingleMessageRecipient? = null): Node {
-        val extraAdvertisedServices = if (networkMapAddress == null) setOf(ServiceInfo(NetworkMapService.type, "NMS")) else emptySet<ServiceInfo>()
-
-        val notaryNode = Node(
-                configuration = config,
-                advertisedServices = extraAdvertisedServices + ServiceInfo(RaftValidatingNotaryService.type, notaryName),
-                networkMapAddress = networkMapAddress)
-
-        notaryNode.setup().start()
-        thread { notaryNode.run() }
-        notaryNode.networkMapRegistrationFuture.getOrThrow()
-        return notaryNode
-    }
-
-    private fun createAliceNode(networkMapAddress: SingleMessageRecipient): Node {
-        val aliceDir = Paths.get(baseDir, "alice")
-        val alice = Node(
-                configuration = generateConfig(aliceDir, "Alice"),
-                advertisedServices = setOf(),
-                networkMapAddress = networkMapAddress)
-        alice.setup().start()
-        thread { alice.run() }
-        alice.networkMapRegistrationFuture.getOrThrow()
-        return alice
-    }
-
-    private fun issueState(node: AbstractNode, notary: Party, notaryKey: KeyPair): StateAndRef<*> {
-        return databaseTransaction(node.database) {
-            val tx = DummyContract.generateInitial(node.info.legalIdentity.ref(0), Random().nextInt(), notary)
-            tx.signWith(node.services.legalIdentityKey)
-            tx.signWith(notaryKey)
-            val stx = tx.toSignedTransaction()
-            node.services.recordTransactions(listOf(stx))
-            StateAndRef(tx.outputStates().first(), StateRef(stx.id, 0))
-        }
-    }
-
-    private fun generateConfig(dir: Path, name: String, notaryNodeAddress: HostAndPort? = null, notaryClusterAddress: HostAndPort? = null) = FullNodeConfiguration(
-            ConfigHelper.loadConfig(dir,
-                    allowMissingConfig = true,
-                    configOverrides = mapOf(
-                            "myLegalName" to name,
-                            "basedir" to dir,
-                            "artemisAddress" to freeLocalHostAndPort().toString(),
-                            "webAddress" to freeLocalHostAndPort().toString(),
-                            "notaryNodeAddress" to notaryNodeAddress?.toString(),
-                            "notaryClusterAddresses" to (if (notaryClusterAddress == null) emptyList<String>() else listOf(notaryClusterAddress.toString()))
-                    )))
-}

node/src/integration-test/kotlin/net/corda/node/services/DistributedServiceTests.kt

@@ -27,7 +27,7 @@ import rx.Observable
 import java.util.*
 import kotlin.test.assertEquals
 
-class RaftValidatingNotaryServiceTests : DriverBasedTest() {
+class DistributedServiceTests : DriverBasedTest() {
     lateinit var alice: NodeInfo
     lateinit var notaries: List<NodeHandle>
     lateinit var aliceProxy: CordaRPCOps
@@ -69,8 +69,10 @@ class RaftValidatingNotaryServiceTests : DriverBasedTest() {
         runTest()
     }
 
+    // TODO Use a dummy distributed service rather than a Raft Notary Service as this test is only about Artemis' ability
+    // to handle distributed services
     @Test
-    fun `notarisation requests are distributed evenly in raft cluster`() {
+    fun `requests are distributed evenly amongst the nodes`() {
         // Issue 100 pounds, then pay ourselves 50x2 pounds
         val issueHandle = aliceProxy.startFlow(::CashFlow, CashCommand.IssueCash(100.POUNDS, OpaqueBytes.of(0), alice.legalIdentity, raftNotaryIdentity))
         require(issueHandle.returnValue.toBlocking().first() is CashFlowResult.Success)
@@ -98,6 +100,7 @@ class RaftValidatingNotaryServiceTests : DriverBasedTest() {
         require(notarisationsPerNotary.values.all { it > 10 })
     }
 
+    // TODO This should be in RaftNotaryServiceTests
     @Test
     fun `cluster survives if a notary is killed`() {
         // Issue 100 pounds, then pay ourselves 10x5 pounds

node/src/integration-test/kotlin/net/corda/node/services/RaftNotaryServiceTests.kt

@@ -0,0 +1,98 @@
+package net.corda.node.services
+
+import net.corda.core.contracts.DummyContract
+import net.corda.core.contracts.StateAndRef
+import net.corda.core.contracts.StateRef
+import net.corda.core.contracts.TransactionType
+import net.corda.core.crypto.Party
+import net.corda.core.div
+import net.corda.core.getOrThrow
+import net.corda.core.node.services.ServiceInfo
+import net.corda.flows.NotaryError
+import net.corda.flows.NotaryException
+import net.corda.flows.NotaryFlow
+import net.corda.node.internal.AbstractNode
+import net.corda.node.internal.Node
+import net.corda.node.services.transactions.RaftValidatingNotaryService
+import net.corda.node.utilities.ServiceIdentityGenerator
+import net.corda.node.utilities.databaseTransaction
+import net.corda.testing.getFreeLocalPorts
+import net.corda.testing.node.NodeBasedTest
+import org.junit.Test
+import java.security.KeyPair
+import java.util.*
+import kotlin.test.assertEquals
+import kotlin.test.assertFailsWith
+
+class RaftNotaryServiceTests : NodeBasedTest() {
+    private val notaryName = "RAFT Notary Service"
+    private val clusterSize = 3
+
+    @Test
+    fun `detect double spend`() {
+        val masterNode = createNotaryCluster()
+        val alice = startNode("Alice")
+
+        val notaryParty = alice.netMapCache.getNotary(notaryName)!!
+
+        val stx = run {
+            val notaryNodeKeyPair = databaseTransaction(masterNode.database) {
+                masterNode.services.notaryIdentityKey
+            }
+            val inputState = issueState(alice, notaryParty, notaryNodeKeyPair)
+            val tx = TransactionType.General.Builder(notaryParty).withItems(inputState)
+            val aliceKey = databaseTransaction(alice.database) {
+                alice.services.legalIdentityKey
+            }
+            tx.signWith(aliceKey)
+            tx.toSignedTransaction(false)
+        }
+
+        val buildFlow = { NotaryFlow.Client(stx) }
+
+        val firstSpend = alice.services.startFlow(buildFlow())
+        firstSpend.resultFuture.getOrThrow()
+
+        val secondSpend = alice.services.startFlow(buildFlow())
+
+        val ex = assertFailsWith(NotaryException::class) { secondSpend.resultFuture.getOrThrow() }
+        val error = ex.error as NotaryError.Conflict
+        assertEquals(error.tx, stx.tx)
+    }
+
+    private fun createNotaryCluster(): Node {
+        val notaryService = ServiceInfo(RaftValidatingNotaryService.type, notaryName)
+        val notaryAddresses = getFreeLocalPorts("localhost", clusterSize).map { it.toString() }
+        ServiceIdentityGenerator.generateToDisk(
+            (0 until clusterSize).map { tempFolder.root.toPath() / "Notary$it" },
+            notaryService.type.id,
+            notaryName)
+
+        val masterNode = startNode(
+            "Notary0",
+            advertisedServices = setOf(notaryService),
+            configOverrides = mapOf("notaryNodeAddress" to notaryAddresses[0]))
+
+        for (i in 1 until clusterSize) {
+            startNode(
+                "Notary$i",
+                advertisedServices = setOf(notaryService),
+                configOverrides = mapOf(
+                    "notaryNodeAddress" to notaryAddresses[i],
+                    "notaryClusterAddresses" to listOf(notaryAddresses[0])))
+        }
+
+        return masterNode
+    }
+
+    private fun issueState(node: AbstractNode, notary: Party, notaryKey: KeyPair): StateAndRef<*> {
+        return databaseTransaction(node.database) {
+            val tx = DummyContract.generateInitial(node.info.legalIdentity.ref(0), Random().nextInt(), notary)
+            tx.signWith(node.services.legalIdentityKey)
+            tx.signWith(notaryKey)
+            val stx = tx.toSignedTransaction()
+            node.services.recordTransactions(listOf(stx))
+            StateAndRef(tx.outputStates().first(), StateRef(stx.id, 0))
+        }
+    }
+}

node/src/main/kotlin/net/corda/node/utilities/ServiceIdentityGenerator.kt

@@ -6,6 +6,7 @@ import net.corda.core.crypto.composite
 import net.corda.core.crypto.generateKeyPair
 import net.corda.core.serialization.serialize
 import net.corda.core.utilities.loggerFor
+import net.corda.core.utilities.trace
 import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.Paths
@@ -24,7 +25,7 @@ object ServiceIdentityGenerator {
      * @param threshold The threshold for the generated group [CompositeKey.Node].
      */
     fun generateToDisk(dirs: List<Path>, serviceId: String, serviceName: String, threshold: Int = 1) {
-        log.trace("Generating a group identity \"serviceName\" for nodes: ${dirs.joinToString()}")
+        log.trace { "Generating a group identity \"serviceName\" for nodes: ${dirs.joinToString()}" }
 
         val keyPairs = (1..dirs.size).map { generateKeyPair() }
         val notaryKey = CompositeKey.Builder().addKeys(keyPairs.map { it.public.composite }).build(threshold)
@@ -32,8 +33,8 @@ object ServiceIdentityGenerator {
 
         keyPairs.zip(dirs) { keyPair, dir ->
             Files.createDirectories(dir)
-            val privateKeyFile = serviceId + "-private-key"
-            val publicKeyFile = serviceId + "-public"
+            val privateKeyFile = "$serviceId-private-key"
+            val publicKeyFile = "$serviceId-public"
             notaryParty.writeToFile(dir.resolve(publicKeyFile))
             keyPair.serialize().writeToFile(dir.resolve(privateKeyFile))
         }