Overriding network parameters if they are specified in the deployment… (#136)

* Overriding network parameters if they are specified in the deployment configuration

* Addressing review comments

* Addressing review comments

* Changing the getCurrentNetworkMap return type to be nullable

docs/source/running-doorman.rst

@@ -67,9 +67,9 @@ When doorman is running it will serve the current network parameters. The first
 started it will need to know the initial value for the network parameters.
 
 The initial values for the network parameters can be specified with a file, like this:
-    .. literalinclude:: ../../network-management/initial-network-parameters.conf
+    .. literalinclude:: ../../network-management/network-parameters.conf
 
-And the location of that file can be specified with: ``--initialNetworkParameters``.
+And the location of that file can be specified with: ``--update-network-parameters``.
 Note that when reading from file:
 
 1. ``epoch`` will always be set to 1,

network-management/doorman.conf

@@ -14,7 +14,7 @@ dataSourceProperties {
 h2port = 0
 
 jiraConfig{
-  address = "https://doorman-jira-host/"
+  address = "https://doorman-jira-host.com/"
   projectCode = "TD"
   username = "username"
   password = "password"

network-management/initial-network-parameters.conf

@@ -1,12 +0,0 @@
-notaries : [{
-    name: "O=Notary A, L=Port Louis, C=MU, OU=Org Unit, CN=Service Name"
-    key: "GfHq2tTVk9z4eXgyWmExBB3JfHpeuYrk9jUc4zaVVSXpnW8FdCUNDhw6GRGN"
-    validating: true
-}, {
-    name: "O=Notary B, L=Bali, C=ID, OU=Org Unit, CN=Service Name"
-    key: "GfHq2tTVk9z4eXgyEshv6vtBDjp7n76QZH5hk6VXLhk3vRTAmKcP9F9tRfPj"
-    validating: false
-}]
-minimumPlatformVersion = 1
-maxMessageSize = 100
-maxTransactionSize = 100

network-management/src/integration-test/kotlin/com/r3/corda/networkmanage/hsm/SigningServiceIntegrationTest.kt

@@ -22,11 +22,8 @@ import net.corda.node.utilities.X509Utilities
 import net.corda.node.utilities.configureDatabase
 import net.corda.node.utilities.registration.HTTPNetworkRegistrationService
 import net.corda.node.utilities.registration.NetworkRegistrationHelper
-import net.corda.testing.ALICE
+import net.corda.testing.*
-import net.corda.testing.BOB
-import net.corda.testing.CHARLIE
 import net.corda.testing.common.internal.testNetworkParameters
-import net.corda.testing.testNodeConfiguration
 import org.bouncycastle.pkcs.jcajce.JcaPKCS10CertificationRequest
 import org.h2.tools.Server
 import org.junit.*
@@ -49,6 +46,10 @@ class SigningServiceIntegrationTest {
     @JvmField
     val tempFolder = TemporaryFolder()
 
+    @Rule
+    @JvmField
+    val testSerialization = SerializationEnvironmentRule(true)
+
     private lateinit var timer: Timer
 
     @Before
@@ -92,7 +93,7 @@ class SigningServiceIntegrationTest {
             // Identity service not needed doorman, corda persistence is not very generic.
             throw UnsupportedOperationException()
         }, SchemaService())
-        val doorman = startDoorman(NetworkHostAndPort(HOST, 0), database, approveAll = true, approveInterval = 2, signInterval = 30, initialNetworkMapParameters = testNetworkParameters(emptyList()))
+        val doorman = startDoorman(NetworkHostAndPort(HOST, 0), database, approveAll = true, approveInterval = 2, signInterval = 30, networkMapParameters = testNetworkParameters(emptyList()))
 
         // Start Corda network registration.
         val config = testNodeConfiguration(
@@ -148,7 +149,7 @@ class SigningServiceIntegrationTest {
             // Identity service not needed doorman, corda persistence is not very generic.
             throw UnsupportedOperationException()
         }, SchemaService())
-        val doorman = startDoorman(NetworkHostAndPort(HOST, 0), database, approveAll = true, approveInterval = 2, signInterval = 10, initialNetworkMapParameters = testNetworkParameters(emptyList()))
+        val doorman = startDoorman(NetworkHostAndPort(HOST, 0), database, approveAll = true, approveInterval = 2, signInterval = 10, networkMapParameters = testNetworkParameters(emptyList()))
 
         thread(start = true, isDaemon = true) {
             val h2ServerArgs = arrayOf("-tcpPort", H2_TCP_PORT, "-tcpAllowOthers")

network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/NetworkMapStorage.kt

@@ -12,7 +12,7 @@ interface NetworkMapStorage {
      * Retrieves current network map. Current in this context means the one that has been most recently signed.
      * @return current network map
      */
-    fun getCurrentNetworkMap(): SignedNetworkMap
+    fun getCurrentNetworkMap(): SignedNetworkMap?
 
     /**
      * Retrieves current map node info hashes only. Hashes are further filtered by the [certificateStatuses] parameter
@@ -45,10 +45,10 @@ interface NetworkMapStorage {
     fun getNetworkParameters(parameterHash: SecureHash): NetworkParameters
 
     /**
-     * Retrieve network map parameters that are used in the current network map.
+     * Retrieve network map parameters.
-     * @return current network map parameters
+     * @return current network map parameters or null if they don't exist
      */
-    fun getCurrentNetworkParameters(): NetworkParameters
+    fun getCurrentNetworkParameters(): NetworkParameters?
 
     /**
      *  Persists given network parameters.

network-management/src/main/kotlin/com/r3/corda/networkmanage/common/persistence/PersistentNetworkMapStorage.kt

@@ -17,22 +17,18 @@ import org.hibernate.jpa.QueryHints
  * Database implementation of the [NetworkMapStorage] interface
  */
 class PersistentNetworkMapStorage(private val database: CordaPersistence) : NetworkMapStorage {
-    override fun getCurrentNetworkMap(): SignedNetworkMap = database.transaction {
+    override fun getCurrentNetworkMap(): SignedNetworkMap? = database.transaction {
         val networkMapEntity = getCurrentNetworkMapEntity(getNetworkMapWithNodeInfoAndParametersHint(session))
-        networkMapEntity ?: throw NoSuchElementException("Current Network Map does not exist.")
+        networkMapEntity?.let {
-        val nodeInfoHashes = networkMapEntity.nodeInfoList.map { it.nodeInfoHash }
+            val nodeInfoHashes = it.nodeInfoList.map { it.nodeInfoHash }
-        val networkParameterHash = networkMapEntity.parameters.parametersHash
+            val networkParameterHash = it.parameters.parametersHash
-        val signatureAndCertPath = networkMapEntity.signatureAndCertificate()
+            val signatureAndCertPath = it.signatureAndCertificate()
-        SignedNetworkMap(NetworkMap(nodeInfoHashes, networkParameterHash), signatureAndCertPath!!)
+            SignedNetworkMap(NetworkMap(nodeInfoHashes, networkParameterHash), signatureAndCertPath!!)
+        }
     }
 
-    override fun getCurrentNetworkParameters(): NetworkParameters = database.transaction {
+    override fun getCurrentNetworkParameters(): NetworkParameters? = database.transaction {
-        val networkMapEntity = getCurrentNetworkMapEntity(getNetworkMapWithParametersHint(session))
+        getCurrentNetworkMapEntity(getNetworkMapWithParametersHint(session))?.parameters?.networkParameters()
-        if (networkMapEntity != null) {
-            networkMapEntity.parameters.networkParameters()
-        } else {
-            throw NoSuchElementException("Current Network Parameters do not exist.")
-        }
     }
 
     override fun saveNetworkMap(signedNetworkMap: SignedNetworkMap) {
@@ -40,7 +36,7 @@ class PersistentNetworkMapStorage(private val database: CordaPersistence) : Netw
             val networkMap = signedNetworkMap.networkMap
             val signatureAndCertPath = signedNetworkMap.signatureData
             val signature = signatureAndCertPath.signature
-            val networkParametersEntity = getNetworkParametersEntity(networkMap.parametersHash.toString())
+            val networkParametersEntity = getNetworkParametersEntity(networkMap.parametersHash)
             networkParametersEntity ?: throw IllegalArgumentException("Error when retrieving network parameters entity for network map signing! - Entity does not exist")
             val networkMapEntity = NetworkMapEntity(
                     parameters = networkParametersEntity,

network-management/src/main/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSigner.kt

@@ -35,7 +35,7 @@ class NetworkMapSigner(private val networkMapStorage: NetworkMapStorage,
         val nodeInfoHashes = currentNetworkMapValidNodeInfo + detachedValidNodeInfo
         val networkParameters = networkMapStorage.getLatestNetworkParameters()
         val networkMap = NetworkMap(nodeInfoHashes.map { it.toString() }, networkParameters.serialize().hash.toString())
-        if (networkMap != currentSignedNetworkMap.networkMap) {
+        if (networkMap != currentSignedNetworkMap?.networkMap) {
             val digitalSignature = signer.sign(networkMap.serialize().bytes)
             require(digitalSignature != null) { "Error while signing network map." }
             val signedHashedNetworkMap = SignedNetworkMap(networkMap, digitalSignature!!)

network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/DoormanParameters.kt

@@ -1,20 +1,13 @@
 package com.r3.corda.networkmanage.doorman
 
 import com.r3.corda.networkmanage.common.utils.ShowHelpException
-import com.r3.corda.networkmanage.common.utils.toConfigWithOptions
-import com.r3.corda.networkmanage.doorman.DoormanParameters.Companion.DEFAULT_APPROVE_INTERVAL
-import com.r3.corda.networkmanage.doorman.DoormanParameters.Companion.DEFAULT_SIGN_INTERVAL
 import com.typesafe.config.ConfigFactory
 import com.typesafe.config.ConfigParseOptions
 import joptsimple.OptionParser
 import net.corda.core.internal.isRegularFile
-import net.corda.core.internal.exists
-import net.corda.core.internal.div
-import net.corda.core.utilities.seconds
 import net.corda.nodeapi.config.parseAs
 import java.nio.file.Path
 import java.nio.file.Paths
-import java.time.Duration
 import java.util.*
 
 data class DoormanParameters(// TODO Create a localSigning sub-config and put that there
@@ -60,10 +53,12 @@ data class DoormanParameters(// TODO Create a localSigning sub-config and put th
 }
 
 data class CommandLineOptions(val configFile: Path,
-                              val initialNetworkParameters: Path) {
+                              val updateNetworkParametersFile: Path?) {
     init {
         check(configFile.isRegularFile()) { "Config file $configFile does not exist" }
-        check(initialNetworkParameters.isRegularFile()) { "Initial network parameters file $initialNetworkParameters does not exist" }
+        if (updateNetworkParametersFile != null) {
+            check(updateNetworkParametersFile.isRegularFile()) { "Update network parameters file $updateNetworkParametersFile does not exist" }
+        }
     }
 }
 
@@ -76,23 +71,26 @@ fun parseCommandLine(vararg args: String): CommandLineOptions {
             .accepts("config-file", "The path to the config file")
             .withRequiredArg()
             .describedAs("filepath")
-    val initialNetworkParametersArg = optionParser
+    val updateNetworkParametersArg = optionParser
-            .accepts("initial-network-parameters", "initial network parameters filepath")
+            .accepts("update-network-parameters", "Update network parameters filepath. Currently only network parameters initialisation is supported.")
             .withRequiredArg()
-            .describedAs("The initial network map")
+            .describedAs("The new network map")
             .describedAs("filepath")
     val helpOption = optionParser.acceptsAll(listOf("h", "?", "help"), "show help").forHelp();
 
     val optionSet = optionParser.parse(*args)
     // Print help and exit on help option or if there are missing options.
-    if (optionSet.has(helpOption) || !optionSet.has(configFileArg) || !optionSet.has(initialNetworkParametersArg)) {
+    if (optionSet.has(helpOption) || !optionSet.has(configFileArg)) {
         throw ShowHelpException(optionParser)
     }
 
     val configFile = Paths.get(optionSet.valueOf(configFileArg)).toAbsolutePath()
-    val initialNetworkParameters = Paths.get(optionSet.valueOf(initialNetworkParametersArg)).toAbsolutePath()
+    val updateNetworkParametersOptionValue = optionSet.valueOf(updateNetworkParametersArg)
+    val updateNetworkParameters = updateNetworkParametersOptionValue?.let {
+        Paths.get(it).toAbsolutePath()
+    }
 
-    return CommandLineOptions(configFile, initialNetworkParameters)
+    return CommandLineOptions(configFile, updateNetworkParameters)
 }
 
 /**

network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/Main.kt

@@ -159,7 +159,7 @@ fun generateCAKeyPair(keystorePath: Path, rootStorePath: Path, rootKeystorePass:
 fun startDoorman(hostAndPort: NetworkHostAndPort,
                  database: CordaPersistence,
                  approveAll: Boolean,
-                 initialNetworkMapParameters: NetworkParameters,
+                 networkMapParameters: NetworkParameters?,
                  signer: LocalSigner? = null,
                  approveInterval: Long,
                  signInterval: Long,
@@ -184,6 +184,16 @@ fun startDoorman(hostAndPort: NetworkHostAndPort,
     val networkMapStorage = PersistentNetworkMapStorage(database)
     val nodeInfoStorage = PersistentNodeInfoStorage(database)
 
+    if (networkMapParameters != null) {
+        // Persisting new network parameters
+        val currentNetworkParameters = networkMapStorage.getCurrentNetworkParameters()
+        if (currentNetworkParameters == null) {
+            networkMapStorage.putNetworkParameters(networkMapParameters)
+        } else {
+            throw UnsupportedOperationException("Network parameters already exist. Updating them via the file config is not supported yet.")
+        }
+    }
+
     val doorman = DoormanServer(hostAndPort, RegistrationWebService(requestProcessor, DoormanServer.serverStatus), NodeInfoWebService(nodeInfoStorage, networkMapStorage))
     doorman.start()
 
@@ -266,8 +276,9 @@ fun main(args: Array<String>) {
                 DoormanParameters.Mode.DOORMAN -> {
                     val database = configureDatabase(dataSourceProperties, databaseProperties, { throw UnsupportedOperationException() }, SchemaService())
                     val signer = buildLocalSigner(this)
-
+                    val networkParameters = commandLineOptions.updateNetworkParametersFile?.let {
-                    val networkParameters = parseNetworkParametersFrom(commandLineOptions.initialNetworkParameters)
+                        parseNetworkParametersFrom(it)
+                    }
                     startDoorman(NetworkHostAndPort(host, port), database, approveAll, networkParameters, signer, approveInterval, signInterval, jiraConfig)
                 }
             }

network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/NetworkParametersConfiguration.kt

@@ -47,21 +47,19 @@ internal data class NetworkParametersConfiguration(val minimumPlatformVersion: I
  * Parses a file and returns a [NetworkParameters] instance.
  *
  * @return a [NetworkParameters] with values read from [configFile] except:
- * an epoch of [DEFAULT_EPOCH],
+ * an epoch of [DEFAULT_EPOCH] and
- * an eventHorizon of [DEFAULT_EVENT_HORIZON], and
  * a modifiedTime initialized with [Instant.now].
- * If [configFile] is null [DEFAULT_NETWORK_PARAMETERS] is returned.
  */
-fun parseNetworkParametersFrom(configFile: Path): NetworkParameters {
+fun parseNetworkParametersFrom(configFile: Path, epoch: Int = DEFAULT_EPOCH): NetworkParameters {
     check(configFile.exists()) { "File $configFile does not exist" }
-    val initialNetworkParameters = ConfigFactory.parseFile(configFile.toFile(), ConfigParseOptions.defaults())
+    val networkParametersConfig = ConfigFactory.parseFile(configFile.toFile(), ConfigParseOptions.defaults())
             .parseAs(NetworkParametersConfiguration::class)
 
-    return NetworkParameters(initialNetworkParameters.minimumPlatformVersion,
+    return NetworkParameters(networkParametersConfig.minimumPlatformVersion,
-            initialNetworkParameters.notaries.map { it.toNotaryInfo() },
+            networkParametersConfig.notaries.map { it.toNotaryInfo() },
-            initialNetworkParameters.eventHorizonDays.days,
+            networkParametersConfig.eventHorizonDays.days,
-            initialNetworkParameters.maxMessageSize,
+            networkParametersConfig.maxMessageSize,
-            initialNetworkParameters.maxTransactionSize,
+            networkParametersConfig.maxTransactionSize,
             Instant.now(),
-            DEFAULT_EPOCH)
+            epoch)
 }

network-management/src/main/kotlin/com/r3/corda/networkmanage/doorman/webservice/NodeInfoWebService.kt

@@ -65,7 +65,12 @@ class NodeInfoWebService(private val nodeInfoStorage: NodeInfoStorage,
     @GET
     fun getNetworkMap(): Response {
         // TODO: Cache the response?
-        return ok(networkMapStorage.getCurrentNetworkMap().serialize().bytes).build()
+        val currentNetworkMap = networkMapStorage.getCurrentNetworkMap()
+        return if (currentNetworkMap != null) {
+            ok(currentNetworkMap.serialize().bytes).build()
+        } else {
+            status(Response.Status.NOT_FOUND).build()
+        }
     }
 
     @GET

network-management/src/test/kotlin/com/r3/corda/networkmanage/NetworkParametersConfigurationTest.kt

@@ -11,13 +11,11 @@ import java.time.Instant
 
 class NetworkParametersConfigurationTest {
 
-    private val validInitialNetworkConfigPath = File(javaClass.getResource("/initial-network-parameters.conf").toURI())
+    private val validOverrideNetworkConfigPath = File("network-parameters.conf").toPath()
 
     @Test
     fun `reads an existing file`() {
-        val confFile = validInitialNetworkConfigPath.toPath()
+        val networkParameters = parseNetworkParametersFrom(validOverrideNetworkConfigPath)
-
-        val networkParameters = parseNetworkParametersFrom(confFile)
         assertThat(networkParameters.minimumPlatformVersion).isEqualTo(1)
         assertThat(networkParameters.eventHorizon).isEqualTo(100.days)
         val notaries = networkParameters.notaries

network-management/src/test/kotlin/com/r3/corda/networkmanage/common/persistence/DBNetworkMapStorageTest.kt

@@ -118,7 +118,7 @@ class DBNetworkMapStorageTest : TestBase() {
         val result = networkMapStorage.getCurrentNetworkParameters()
 
         // then
-        assertEquals(1, result.minimumPlatformVersion)
+        assertEquals(1, result?.minimumPlatformVersion)
     }
 
     @Test

network-management/src/test/kotlin/com/r3/corda/networkmanage/common/signer/NetworkMapSignerTest.kt

@@ -73,4 +73,29 @@ class NetworkMapSignerTest : TestBase() {
         // Verify networkMapStorage is not called
         verify(networkMapStorage, never()).saveNetworkMap(any())
     }
+
+    @Test
+    fun `signNetworkMap creates a new network map if there is no current network map`() {
+        // given
+        val networkMapParameters = createNetworkParameters()
+        whenever(networkMapStorage.getCurrentNetworkMap()).thenReturn(null)
+        whenever(networkMapStorage.getCurrentNetworkMapNodeInfoHashes(any())).thenReturn(emptyList())
+        whenever(networkMapStorage.getDetachedAndValidNodeInfoHashes()).thenReturn(emptyList())
+        whenever(networkMapStorage.getLatestNetworkParameters()).thenReturn(networkMapParameters)
+        whenever(signer.sign(any())).thenReturn(mock())
+
+        // when
+        networkMapSigner.signNetworkMap()
+
+        // then
+        // Verify networkMapStorage calls
+        verify(networkMapStorage).getCurrentNetworkMapNodeInfoHashes(any())
+        verify(networkMapStorage).getDetachedAndValidNodeInfoHashes()
+        verify(networkMapStorage).getLatestNetworkParameters()
+        argumentCaptor<SignedNetworkMap>().apply {
+            verify(networkMapStorage).saveNetworkMap(capture())
+            val networkMap = firstValue.networkMap
+            assertEquals(networkMapParameters.serialize().hash.toString(), networkMap.parametersHash)
+        }
+    }
 }

network-management/src/test/kotlin/com/r3/corda/networkmanage/doorman/DoormanParametersTest.kt

@@ -9,23 +9,23 @@ import kotlin.test.assertEquals
 import kotlin.test.assertFailsWith
 
 class DoormanParametersTest {
-    private val validInitialNetworkConfigPath = File(javaClass.getResource("/initial-network-parameters.conf").toURI()).absolutePath
+    private val validOverrideNetworkConfigPath = File("network-parameters.conf").absolutePath
-    private val validConfigPath = File(javaClass.getResource("/doorman.conf").toURI()).absolutePath
+    private val validConfigPath = File("doorman.conf").absolutePath
     private val invalidConfigPath = File(javaClass.getResource("/doorman_fail.conf").toURI()).absolutePath
-    private val validArgs = arrayOf("--config-file", validConfigPath, "--initial-network-parameters", validInitialNetworkConfigPath)
+    private val validArgs = arrayOf("--config-file", validConfigPath, "--update-network-parameters", validOverrideNetworkConfigPath)
 
     @Test
     fun `should fail when initial network parameters file is missing`() {
         val message = assertFailsWith<IllegalStateException> {
-            parseCommandLine("--config-file", validConfigPath, "--initial-network-parameters", "not-here")
+            parseCommandLine("--config-file", validConfigPath, "--update-network-parameters", "not-here")
         }.message
-        assertThat(message).contains("Initial network parameters file ")
+        assertThat(message).contains("Update network parameters file ")
     }
 
     @Test
     fun `should fail when config file is missing`() {
         val message = assertFailsWith<IllegalStateException> {
-            parseCommandLine("--config-file", "not-existing-file", "--initial-network-parameters", validInitialNetworkConfigPath)
+            parseCommandLine("--config-file", "not-existing-file")
         }.message
         assertThat(message).contains("Config file ")
     }
@@ -40,7 +40,7 @@ class DoormanParametersTest {
     @Test
     fun `should fail when config missing`() {
         assertFailsWith<ConfigException.Missing> {
-            parseParameters(parseCommandLine("--config-file", invalidConfigPath, "--initial-network-parameters", validInitialNetworkConfigPath).configFile)
+            parseParameters(parseCommandLine("--config-file", invalidConfigPath).configFile)
         }
     }
 

network-management/src/test/resources/doorman.conf

@@ -1,22 +0,0 @@
-basedir="."
-keystorePath = "/opt/doorman/certificates/caKeystore.jks"
-keyStorePassword = "password"
-caPrivateKeyPassword = "password"
-host = "localhost"
-port = 8080
-h2port = 0
-
-dataSourceProperties {
-  "dataSourceClassName" = org.h2.jdbcx.JdbcDataSource
-  "dataSource.url" = "jdbc:h2:file:"${basedir}"/persistence;DB_CLOSE_ON_EXIT=FALSE;LOCK_TIMEOUT=10000;WRITE_DELAY=0;AUTO_SERVER_PORT="${h2port}
-  "dataSource.user" = sa
-  "dataSource.password" = ""
-}
-
-jiraConfig{
-  address = "https://doorman-jira-host.com/"
-  projectCode = "TD"
-  username = "username"
-  password = "password"
-  doneTransitionCode = 41
-}