Add jitter to flow retries (#1037)

* Add jitter to flow retries

See https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/

* Add two seconds timeout to `timed flow is retried` test

* Minimum jittered delay is one second

* Adjust default retry config

Make sure the last retry fires after more than 20 minutes, which
is twice the default network map update. This way the notary should
have the latest network map when the last retry kicks in.

node/src/integration-test/kotlin/net/corda/node/services/TimedFlowMultiThreadedSMMTests.kt

@@ -182,4 +182,4 @@ class TimedFlowMultiThreadedSMMTests : IntegrationTest() {
             }
         }
     }
-}
+}

node/src/main/kotlin/net/corda/node/services/statemachine/MultiThreadedStateMachineManager.kt

@@ -602,10 +602,11 @@ class MultiThreadedStateMachineManager(
     private fun scheduleTimeoutException(flow: Flow, retryCount: Int): ScheduledFuture<*> {
         return with(serviceHub.configuration.flowTimeout) {
             val timeoutDelaySeconds = timeout.seconds * Math.pow(backoffBase, retryCount.toDouble()).toLong()
+            val jitteredDelaySeconds = maxOf(1L ,timeoutDelaySeconds/2 + (Random().nextDouble() * timeoutDelaySeconds/2).toLong())
             timeoutScheduler.schedule({
                 val event = Event.Error(FlowTimeoutException(maxRestartCount))
                 flow.fiber.scheduleEvent(event)
-            }, timeoutDelaySeconds, TimeUnit.SECONDS)
+            }, jitteredDelaySeconds, TimeUnit.SECONDS)
         }
     }
 

node/src/main/resources/reference.conf

@@ -46,6 +46,6 @@ rpcSettings = {
 }
 flowTimeout {
     timeout = 30 seconds
-    maxRestartCount = 5
+    maxRestartCount = 6
     backoffBase = 1.8
 }