Merge pull request #852 from corda/mnesbit-configure-bridge-async-send-ack

Set the confirmationWindowSize setting in the bridge Artemis link.

bridge/src/main/kotlin/net/corda/bridge/services/api/BridgeConfiguration.kt

@@ -118,5 +118,11 @@ interface BridgeConfiguration : NodeSSLConfiguration {
     // e.g links to the Float Outer, or Artemis sessions, before the process continues shutting down anyway.
     // Default value is 1000 ms.
     val politeShutdownPeriod: Int
+    // p2pConfirmationWindowSize determines the number of bytes buffered by the broker before flushing to disk and
+    // acking the triggering send. Setting this to -1 causes session commits to immediately return, potentially
+    // causing blowup in the broker if the rate of sends exceeds the broker's flush rate. Note also that this window
+    // causes send latency to be around [brokerConnectionTtlCheckInterval] if the window isn't saturated.
+    // This is relevant to bridges, because we push messages into the inbox and use the async acknowledgement responses to reply to sender.
+    val p2pConfirmationWindowSize: Int
     val whitelistedHeaders: List<String>
 }

bridge/src/main/kotlin/net/corda/bridge/services/artemis/BridgeArtemisConnectionServiceImpl.kt

@@ -80,6 +80,7 @@ class BridgeArtemisConnectionServiceImpl(val conf: BridgeConfiguration,
                 clientFailureCheckPeriod = -1
                 minLargeMessageSize = maxMessageSize
                 isUseGlobalPools = nodeSerializationEnv != null
+                confirmationWindowSize = conf.p2pConfirmationWindowSize
             }
             connectThread = Thread({ artemisReconnectionLoop() }, "Artemis Connector Thread").apply {
                 isDaemon = true

bridge/src/main/kotlin/net/corda/bridge/services/config/BridgeConfigurationImpl.kt

@@ -72,6 +72,7 @@ data class BridgeConfigurationImpl(
         override val artemisReconnectionIntervalMin: Int = 5000,
         override val artemisReconnectionIntervalMax: Int = 60000,
         override val politeShutdownPeriod: Int = 1000,
+        override val p2pConfirmationWindowSize: Int = 1048576,
         override val whitelistedHeaders: List<String> = ArtemisMessagingComponent.Companion.P2PMessagingHeaders.whitelistedHeaders.toList()
 ) : BridgeConfiguration {
     init {

bridge/src/main/resources/bridgedefault.conf

@@ -13,4 +13,5 @@ enableAMQPPacketTrace = false
 artemisReconnectionIntervalMin = 5000
 artemisReconnectionIntervalMax = 60000
 politeShutdownPeriod = 1000
+p2pConfirmationWindowSize = 1048576
 crlCheckSoftFail = true

docs/source/bridge-configuration-file.rst

@@ -105,6 +105,8 @@ absolute path to the bridge's base directory.
 
         :trustStoreFile: The path to the TrustStore file to use in outgoing ``TLS/AMQP 1.0`` connections.
 
+        :crlCheckSoftFail: If true (recommended setting) allows certificate checks to pass if the CRL provider is unavailable.
+
    :socksProxyConfig:  This section is optionally present if outgoing peer connections should go via a SOCKS4, or SOCKS5 proxy:
 
         :version: Either SOCKS4, or SOCKS5 to define the protocol version used in connecting to the SOCKS proxy.
@@ -131,6 +133,8 @@ absolute path to the bridge's base directory.
 
         :trustStoreFile: The path to the TrustStore file to use in inbound ``TLS/AMQP 1.0`` connections.
 
+        :crlCheckSoftFail: If true (recommended setting) allows certificate checks to pass if the CRL provider is unavailable.
+
 :bridgeInnerConfig:  This section is required for ``BridgeInner`` mode and configures the tunnel connection to the ``FloatOuter`` (s) in the DMZ. The section should be absent in ``SenderReceiver`` and ``FloatOuter`` modes:
 
         :floatAddresses: The list of host and ports to connect the available ``FloatOuter`` instances. At least one must be present.
@@ -150,6 +154,8 @@ absolute path to the bridge's base directory.
 
             :trustStoreFile: The path to the TrustStore file to use in control tunnel connections.
 
+            :crlCheckSoftFail: If true (recommended setting) allows certificate checks to pass if the CRL provider is unavailable.
+            
         :customFloatOuterSSLConfiguration: The keys and certificates for the ``FloatOuter`` are provisioned dynamically from the ``BridgeInner`` over the control tunnel and are not loaded from disk in the DMZ.
             By default, they are taken from (``<workspace>/certificates/sslkeystore.jks``)
             and validate against (``<workspace>/certificates/truststore.jks``), using the passwords defined in the root config. However, alternate sources may be defined in this section.
@@ -162,6 +168,8 @@ absolute path to the bridge's base directory.
 
             :trustStoreFile: The path to the TrustStore file to use in the ``FloatOuter`` when it activates the peer listening socket.
 
+            :crlCheckSoftFail: If true (recommended setting) allows certificate checks to pass if the CRL provider is unavailable.
+
 :floatOuterConfig:   This section is required for ``FloatOuter`` mode and configures the control tunnel listening socket. It should be absent for ``SenderReceiver`` and ``BridgeInner`` modes:
 
         :floatAddress: The host and port to bind the control tunnel listener socket to. This can be for a specific interface if used on a multi-homed machine.
@@ -180,6 +188,8 @@ absolute path to the bridge's base directory.
 
             :trustStoreFile: The path to the TrustStore file to use in control tunnel connections.
 
+            :crlCheckSoftFail: If true (recommended setting) allows certificate checks to pass if the CRL provider is unavailable.
+            
 :haConfig: Optionally the ``SenderReceiver`` and ``BridgeInner`` modes can be run in a hot-warm configuration, which determines the active instance using an external master election service.
     Currently, only Zookeeper can be used as master elector. Eventually other electors may be supported e.g. ``etcd``. This configuration section controls these options:
 
@@ -200,6 +210,9 @@ absolute path to the bridge's base directory.
 
 :artemisReconnectionIntervalMax: The worst case Artemis retry period after repeated failure to connect is [artemisReconnectionIntervalMax] ms. The default interval is 60000 ms.
 
+:p2pConfirmationWindowSize: This is a performance tuning detail within the Artemis connection setup, which controls the send acknowledgement behaviour.
+    Its value should only be modified from the default if suggested by R3 to resolve issues.
+
 :enableAMQPPacketTrace: Set this developer flag to true if very detailed logs are required for connectivity debugging. Note that the logging volume is substantial, so do not enable in production systems.
 
 Fully Worked Example
@@ -372,6 +385,7 @@ Configuration in ``bridge.conf`` for ``bridgeserver1``:
                trustStorePassword = "trustpass"
                sslKeystore = "./bridgecerts/bridge.jks"
                trustStoreFile = "./bridgecerts/trust.jks"
+               crlCheckSoftFail = true
         }
     }
     haConfig { // Enable HA pointing at Zookeeper cluster for master selection.
@@ -402,6 +416,7 @@ Configuration in ``bridge.conf`` for ``bridgeserver2``:
                trustStorePassword = "trustpass"
                sslKeystore = "./bridgecerts/bridge.jks"
                trustStoreFile = "./bridgecerts/trust.jks"
+               crlCheckSoftFail = true
         }
     }
     haConfig { // Enable HA pointing at Zookeeper cluster for master selection.
@@ -426,6 +441,7 @@ Configuration in ``bridge.conf`` for ``floatserver1``:
                trustStorePassword = "trustpass"
                sslKeystore = "./floatcerts/float.jks"
                trustStoreFile = "./floatcerts/trust.jks"
+               crlCheckSoftFail = true
         }
     }
     networkParametersPath = network-parameters // The network-parameters file is expected to be copied from the node after registration and here is expected in the workspace folder.
@@ -446,6 +462,7 @@ Configuration in ``bridge.conf`` for ``floatserver2``:
                trustStorePassword = "trustpass"
                sslKeystore = "./floatcerts/float.jks"
                trustStoreFile = "./floatcerts/trust.jks"
+               crlCheckSoftFail = true
         }
     }
     networkParametersPath = network-parameters // The network-parameters file is expected to be copied from the node after registration and here is expected in the workspace folder.