Add StateMachineRunId, type for SMM Changes

client/src/main/kotlin/com/r3corda/client/model/GatheredTransactionDataModel.kt

@@ -3,6 +3,7 @@ package com.r3corda.client.model
 import com.r3corda.client.fxutils.foldToObservableList
 import com.r3corda.core.crypto.SecureHash
 import com.r3corda.core.transactions.LedgerTransaction
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.node.services.monitor.ServiceToClientEvent
 import com.r3corda.node.services.monitor.TransactionBuildResult
 import com.r3corda.node.utilities.AddOrRemove
@@ -16,7 +17,7 @@ import java.time.Instant
 import java.util.UUID
 
 interface GatheredTransactionData {
-    val fiberId: ObservableValue<Long?>
+    val stateMachineRunId: ObservableValue<StateMachineRunId?>
     val uuid: ObservableValue<UUID?>
     val protocolStatus: ObservableValue<ProtocolStatus?>
     val stateMachineStatus: ObservableValue<StateMachineStatus?>
@@ -42,7 +43,7 @@ sealed class StateMachineStatus(val stateMachineName: String) {
 }
 
 data class GatheredTransactionDataWritable(
-        override val fiberId: SimpleObjectProperty<Long?> = SimpleObjectProperty(null),
+        override val stateMachineRunId: SimpleObjectProperty<StateMachineRunId?> = SimpleObjectProperty(null),
         override val uuid: SimpleObjectProperty<UUID?> = SimpleObjectProperty(null),
         override val stateMachineStatus: SimpleObjectProperty<StateMachineStatus?> = SimpleObjectProperty(null),
         override val protocolStatus: SimpleObjectProperty<ProtocolStatus?> = SimpleObjectProperty(null),
@@ -85,7 +86,7 @@ class GatheredTransactionDataModel {
                             is ServiceToClientEvent.OutputState -> {}
                             is ServiceToClientEvent.StateMachine -> {
                                 newFiberIdTransactionStateOrModify(transactionStates, serviceToClientEvent,
-                                        fiberId = serviceToClientEvent.fiberId,
+                                        stateMachineRunId = serviceToClientEvent.stateMachineRunId,
                                         tweak = {
                                             stateMachineStatus.set(when (serviceToClientEvent.addOrRemove) {
                                                 AddOrRemove.ADD -> StateMachineStatus.Added(serviceToClientEvent.label)
@@ -96,7 +97,7 @@ class GatheredTransactionDataModel {
                             }
                             is ServiceToClientEvent.Progress -> {
                                 newFiberIdTransactionStateOrModify(transactionStates, serviceToClientEvent,
-                                        fiberId = serviceToClientEvent.fiberId,
+                                        stateMachineRunId = serviceToClientEvent.stateMachineRunId,
                                         tweak = {
                                             protocolStatus.set(ProtocolStatus(serviceToClientEvent.message))
                                         }
@@ -106,8 +107,8 @@ class GatheredTransactionDataModel {
                                 val state = serviceToClientEvent.state
                                 newUuidTransactionStateOrModify(transactionStates, serviceToClientEvent,
                                         uuid = serviceToClientEvent.id,
-                                        fiberId = when (state) {
+                                        stateMachineRunId = when (state) {
-                                            is TransactionBuildResult.ProtocolStarted -> state.fiberId
+                                            is TransactionBuildResult.ProtocolStarted -> state.stateMachineId
                                             is TransactionBuildResult.Failed -> null
                                         },
                                         transactionId = when (state) {
@@ -160,13 +161,13 @@ class GatheredTransactionDataModel {
         private fun newFiberIdTransactionStateOrModify(
                 transactionStates: ObservableList<GatheredTransactionDataWritable>,
                 event: ServiceToClientEvent,
-                fiberId: Long,
+                stateMachineRunId: StateMachineRunId,
                 tweak: GatheredTransactionDataWritable.() -> Unit
         ) {
-            val index = transactionStates.indexOfFirst { it.fiberId.value == fiberId }
+            val index = transactionStates.indexOfFirst { it.stateMachineRunId.value == stateMachineRunId }
             val state = if (index < 0) {
                 val newState = GatheredTransactionDataWritable(
-                        fiberId = SimpleObjectProperty(fiberId),
+                        stateMachineRunId = SimpleObjectProperty(stateMachineRunId),
                         lastUpdate = SimpleObjectProperty(event.time)
                 )
                 tweak(newState)
@@ -185,19 +186,19 @@ class GatheredTransactionDataModel {
                 transactionStates: ObservableList<GatheredTransactionDataWritable>,
                 event: ServiceToClientEvent,
                 uuid: UUID,
-                fiberId: Long?,
+                stateMachineRunId: StateMachineRunId?,
                 transactionId: SecureHash?,
                 tweak: GatheredTransactionDataWritable.() -> Unit
         ) {
             val index = transactionStates.indexOfFirst {
                 it.uuid.value == uuid ||
-                        (fiberId != null && it.fiberId.value == fiberId) ||
+                        (stateMachineRunId != null && it.stateMachineRunId.value == stateMachineRunId) ||
                         (transactionId != null && it.transaction.value?.id == transactionId)
             }
             val state = if (index < 0) {
                 val newState = GatheredTransactionDataWritable(
                         uuid = SimpleObjectProperty(uuid),
-                        fiberId = SimpleObjectProperty(fiberId),
+                        stateMachineRunId = SimpleObjectProperty(stateMachineRunId),
                         lastUpdate = SimpleObjectProperty(event.time)
                 )
                 tweak(newState)

contracts/src/main/kotlin/com/r3corda/contracts/testing/VaultFiller.kt

@@ -11,6 +11,7 @@ import com.r3corda.core.contracts.TransactionType
 import com.r3corda.core.crypto.Party
 import com.r3corda.core.node.ServiceHub
 import com.r3corda.core.node.services.Vault
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.core.serialization.OpaqueBytes
 import com.r3corda.core.utilities.DUMMY_NOTARY
 import java.security.PublicKey

core/src/main/kotlin/com/r3corda/core/protocols/ProtocolStateMachine.kt

@@ -6,6 +6,13 @@ import com.r3corda.core.crypto.Party
 import com.r3corda.core.node.ServiceHub
 import com.r3corda.core.utilities.UntrustworthyData
 import org.slf4j.Logger
+import java.util.*
+
+data class StateMachineRunId private constructor(val uuid: UUID) {
+    companion object {
+        fun createRandom() = StateMachineRunId(UUID.randomUUID())
+    }
+}
 
 /**
  * The interface of [ProtocolStateMachineImpl] exposing methods and properties required by ProtocolLogic for compilation.
@@ -30,6 +37,8 @@ interface ProtocolStateMachine<R> {
 
     /** Unique ID for this machine, valid only while it is in memory. */
     val machineId: Long
+    /** Unique ID for this machine run, valid across restarts */
+    val stateMachineRunId: StateMachineRunId
     /** This future will complete when the call method returns. */
     val resultFuture: ListenableFuture<R>
 }

explorer/src/main/kotlin/com/r3corda/explorer/views/TransactionViewer.kt

@@ -11,6 +11,7 @@ import com.r3corda.core.crypto.Party
 import com.r3corda.core.crypto.SecureHash
 import com.r3corda.core.crypto.toStringShort
 import com.r3corda.core.transactions.LedgerTransaction
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.explorer.AmountDiff
 import com.r3corda.explorer.formatters.AmountFormatter
 import com.r3corda.explorer.formatters.Formatter
@@ -46,7 +47,7 @@ class TransactionViewer: View() {
     // Top half (transactions table)
     private val transactionViewTable: TableView<ViewerNode> by fxid()
     private val transactionViewTransactionId: TableColumn<ViewerNode, String> by fxid()
-    private val transactionViewFiberId: TableColumn<ViewerNode, String> by fxid()
+    private val transactionViewStateMachineId: TableColumn<ViewerNode, String> by fxid()
     private val transactionViewClientUuid: TableColumn<ViewerNode, String> by fxid()
     private val transactionViewTransactionStatus: TableColumn<ViewerNode, TransactionCreateStatus?> by fxid()
     private val transactionViewProtocolStatus: TableColumn<ViewerNode, String> by fxid()
@@ -98,7 +99,7 @@ class TransactionViewer: View() {
      */
     data class ViewerNode(
             val transactionId: ObservableValue<SecureHash?>,
-            val fiberId: ObservableValue<Long?>,
+            val stateMachineRunId: ObservableValue<StateMachineRunId?>,
             val clientUuid: ObservableValue<UUID?>,
             val originator: ObservableValue<String>,
             val transactionStatus: ObservableValue<TransactionCreateStatus?>,
@@ -125,7 +126,7 @@ class TransactionViewer: View() {
     private val viewerNodes = gatheredTransactionDataList.map {
         ViewerNode(
                 transactionId = it.transaction.map { it?.id },
-                fiberId = it.fiberId,
+                stateMachineRunId = it.stateMachineRunId,
                 clientUuid = it.uuid,
                 /**
                  * We can't really do any better based on uuid, we need to store explicit data for this TODO
@@ -288,7 +289,7 @@ class TransactionViewer: View() {
         }
 
         transactionViewTransactionId.setCellValueFactory { it.value.transactionId.map { "${it ?: ""}" } }
-        transactionViewFiberId.setCellValueFactory { it.value.fiberId.map { "${it?: ""}" } }
+        transactionViewStateMachineId.setCellValueFactory { it.value.stateMachineRunId.map { "${it?.uuid ?: ""}" } }
         transactionViewClientUuid.setCellValueFactory { it.value.clientUuid.map { "${it ?: ""}" } }
         transactionViewProtocolStatus.setCellValueFactory { it.value.protocolStatus.map { "${it ?: ""}" } }
         transactionViewTransactionStatus.setCellValueFactory { it.value.transactionStatus }

explorer/src/main/resources/com/r3corda/explorer/views/TransactionViewer.fxml

@@ -45,7 +45,7 @@
             <TableView fx:id="transactionViewTable" prefHeight="200.0" prefWidth="200.0">
               <columns>
                   <TableColumn fx:id="transactionViewTransactionId" prefWidth="75.0" text="Transaction ID" />
-                <TableColumn fx:id="transactionViewFiberId" prefWidth="187.0" text="Fiber ID" />
+                <TableColumn fx:id="transactionViewStateMachineId" prefWidth="187.0" text="StateMachine ID" />
                   <TableColumn fx:id="transactionViewClientUuid" prefWidth="75.0" text="Client UUID" />
                   <TableColumn fx:id="transactionViewTransactionStatus" prefWidth="75.0" text="Transaction status" />
                   <TableColumn fx:id="transactionViewProtocolStatus" prefWidth="75.0" text="Protocol status" />

node/src/main/kotlin/com/r3corda/node/internal/AbstractNode.kt

@@ -36,10 +36,7 @@ import com.r3corda.node.services.network.NetworkMapService
 import com.r3corda.node.services.network.NetworkMapService.Companion.REGISTER_PROTOCOL_TOPIC
 import com.r3corda.node.services.network.NodeRegistration
 import com.r3corda.node.services.network.PersistentNetworkMapService
-import com.r3corda.node.services.persistence.NodeAttachmentService
+import com.r3corda.node.services.persistence.*
-import com.r3corda.node.services.persistence.PerFileCheckpointStorage
-import com.r3corda.node.services.persistence.PerFileTransactionStorage
-import com.r3corda.node.services.persistence.StorageServiceImpl
 import com.r3corda.node.services.statemachine.StateMachineManager
 import com.r3corda.node.services.transactions.NotaryService
 import com.r3corda.node.services.transactions.SimpleNotaryService

node/src/main/kotlin/com/r3corda/node/services/api/ServiceHubInternal.kt

@@ -7,6 +7,7 @@ import com.r3corda.core.node.ServiceHub
 import com.r3corda.core.node.services.TxWritableStorageService
 import com.r3corda.core.protocols.ProtocolLogic
 import com.r3corda.core.protocols.ProtocolLogicRefFactory
+import com.r3corda.core.protocols.StateMachineRunId
 
 interface MessagingServiceInternal : MessagingService {
     /**

node/src/main/kotlin/com/r3corda/node/services/monitor/Events.kt

@@ -2,6 +2,7 @@ package com.r3corda.node.services.monitor
 
 import com.r3corda.core.contracts.*
 import com.r3corda.core.transactions.LedgerTransaction
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.node.utilities.AddOrRemove
 import java.time.Instant
 import java.util.*
@@ -22,13 +23,13 @@ sealed class ServiceToClientEvent(val time: Instant) {
     }
     class StateMachine(
             time: Instant,
-            val fiberId: Long,
+            val stateMachineRunId: StateMachineRunId,
             val label: String,
             val addOrRemove: AddOrRemove
     ) : ServiceToClientEvent(time) {
         override fun toString() = "StateMachine($label, ${addOrRemove.name})"
     }
-    class Progress(time: Instant, val fiberId: Long, val message: String) : ServiceToClientEvent(time) {
+    class Progress(time: Instant, val stateMachineRunId: StateMachineRunId, val message: String) : ServiceToClientEvent(time) {
         override fun toString() = "Progress($message)"
     }
     class TransactionBuild(time: Instant, val id: UUID, val state: TransactionBuildResult) : ServiceToClientEvent(time) {
@@ -46,7 +47,7 @@ sealed class TransactionBuildResult {
      *
      * @param transaction the transaction created as a result, in the case where the protocol has completed.
      */
-    class ProtocolStarted(val fiberId: Long, val transaction: LedgerTransaction?, val message: String?) : TransactionBuildResult() {
+    class ProtocolStarted(val stateMachineId: StateMachineRunId, val transaction: LedgerTransaction?, val message: String?) : TransactionBuildResult() {
         override fun toString() = "Started($message)"
     }
 

node/src/main/kotlin/com/r3corda/node/services/monitor/NodeMonitorService.kt

@@ -11,6 +11,7 @@ import com.r3corda.core.messaging.createMessage
 import com.r3corda.core.node.services.DEFAULT_SESSION_ID
 import com.r3corda.core.node.services.Vault
 import com.r3corda.core.protocols.ProtocolLogic
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.core.serialization.serialize
 import com.r3corda.core.transactions.LedgerTransaction
 import com.r3corda.core.transactions.TransactionBuilder
@@ -63,15 +64,15 @@ class NodeMonitorService(services: ServiceHubInternal, val smm: StateMachineMana
         services.storageService.validatedTransactions.updates.subscribe { tx -> notifyTransaction(tx.tx.toLedgerTransaction(services)) }
         services.vaultService.updates.subscribe { update -> notifyVaultUpdate(update) }
         smm.changes.subscribe { change ->
-            val fiberId: Long = change.third
+            val stateMachineRunId: StateMachineRunId = change.stateMachineRunId
-            val logic: ProtocolLogic<*> = change.first
+            val logic: ProtocolLogic<*> = change.logic
             val progressTracker = logic.progressTracker
 
-            notifyEvent(ServiceToClientEvent.StateMachine(Instant.now(), fiberId, logic.javaClass.name, change.second))
+            notifyEvent(ServiceToClientEvent.StateMachine(Instant.now(), stateMachineRunId, logic.javaClass.name, change.addOrRemove))
             if (progressTracker != null) {
-                when (change.second) {
+                when (change.addOrRemove) {
                     AddOrRemove.ADD -> progressTracker.changes.subscribe { progress ->
-                        notifyEvent(ServiceToClientEvent.Progress(Instant.now(), fiberId, progress.toString()))
+                        notifyEvent(ServiceToClientEvent.Progress(Instant.now(), stateMachineRunId, progress.toString()))
                     }
                     AddOrRemove.REMOVE -> {
                         // Nothing to do
@@ -168,7 +169,7 @@ class NodeMonitorService(services: ServiceHubInternal, val smm: StateMachineMana
             val tx = builder.toSignedTransaction(checkSufficientSignatures = false)
             val protocol = FinalityProtocol(tx, setOf(req), setOf(req.recipient))
             return TransactionBuildResult.ProtocolStarted(
-                    smm.add(BroadcastTransactionProtocol.TOPIC, protocol).machineId,
+                    smm.add(BroadcastTransactionProtocol.TOPIC, protocol).stateMachineRunId,
                     tx.tx.toLedgerTransaction(services),
                     "Cash payment transaction generated"
             )
@@ -202,7 +203,7 @@ class NodeMonitorService(services: ServiceHubInternal, val smm: StateMachineMana
             val tx = builder.toSignedTransaction(checkSufficientSignatures = false)
             val protocol = FinalityProtocol(tx, setOf(req), participants)
             return TransactionBuildResult.ProtocolStarted(
-                    smm.add(BroadcastTransactionProtocol.TOPIC, protocol).machineId,
+                    smm.add(BroadcastTransactionProtocol.TOPIC, protocol).stateMachineRunId,
                     tx.tx.toLedgerTransaction(services),
                     "Cash destruction transaction generated"
             )
@@ -221,7 +222,7 @@ class NodeMonitorService(services: ServiceHubInternal, val smm: StateMachineMana
         // Issuance transactions do not need to be notarised, so we can skip directly to broadcasting it
         val protocol = BroadcastTransactionProtocol(tx, setOf(req), setOf(req.recipient))
         return TransactionBuildResult.ProtocolStarted(
-                smm.add(BroadcastTransactionProtocol.TOPIC, protocol).machineId,
+                smm.add(BroadcastTransactionProtocol.TOPIC, protocol).stateMachineRunId,
                 tx.tx.toLedgerTransaction(services),
                 "Cash issuance completed"
         )

node/src/main/kotlin/com/r3corda/node/services/statemachine/ProtocolStateMachineImpl.kt

@@ -9,6 +9,7 @@ import com.google.common.util.concurrent.SettableFuture
 import com.r3corda.core.crypto.Party
 import com.r3corda.core.protocols.ProtocolLogic
 import com.r3corda.core.protocols.ProtocolStateMachine
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.core.utilities.UntrustworthyData
 import com.r3corda.core.utilities.trace
 import com.r3corda.node.services.api.ServiceHubInternal
@@ -28,7 +29,8 @@ import java.util.concurrent.ExecutionException
  * a protocol invokes a sub-protocol, then it will pass along the PSM to the child. The call method of the topmost
  * logic element gets to return the value that the entire state machine resolves to.
  */
-class ProtocolStateMachineImpl<R>(val logic: ProtocolLogic<R>,
+class ProtocolStateMachineImpl<R>(override val stateMachineRunId: StateMachineRunId,
+                                  val logic: ProtocolLogic<R>,
                                   scheduler: FiberScheduler,
                                   private val loggerName: String)
 : Fiber<R>("protocol", scheduler), ProtocolStateMachine<R> {

node/src/main/kotlin/com/r3corda/node/services/statemachine/StateMachineManager.kt

@@ -7,12 +7,14 @@ import com.codahale.metrics.Gauge
 import com.esotericsoftware.kryo.Kryo
 import com.google.common.base.Throwables
 import com.google.common.util.concurrent.ListenableFuture
+import com.r3corda.core.ThreadBox
 import com.r3corda.core.abbreviate
 import com.r3corda.core.messaging.TopicSession
 import com.r3corda.core.messaging.runOnNextMessage
 import com.r3corda.core.messaging.send
 import com.r3corda.core.protocols.ProtocolLogic
 import com.r3corda.core.protocols.ProtocolStateMachine
+import com.r3corda.core.protocols.StateMachineRunId
 import com.r3corda.core.serialization.*
 import com.r3corda.core.then
 import com.r3corda.core.utilities.ProgressTracker
@@ -24,10 +26,10 @@ import com.r3corda.node.utilities.AddOrRemove
 import com.r3corda.node.utilities.AffinityExecutor
 import rx.Observable
 import rx.subjects.PublishSubject
+import rx.subjects.UnicastSubject
 import java.io.PrintWriter
 import java.io.StringWriter
 import java.util.*
-import java.util.Collections.synchronizedMap
 import java.util.concurrent.ExecutionException
 import javax.annotation.concurrent.ThreadSafe
 
@@ -60,30 +62,43 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
 
     val scheduler = FiberScheduler()
 
+    data class Change(
+            val logic: ProtocolLogic<*>,
+            val addOrRemove: AddOrRemove,
+            val stateMachineRunId: StateMachineRunId
+    )
+
     // A list of all the state machines being managed by this class. We expose snapshots of it via the stateMachines
     // property.
-    private val stateMachines = synchronizedMap(LinkedHashMap<ProtocolStateMachineImpl<*>, Checkpoint>())
+    private val mutex = ThreadBox(object {
+        var started = false
+        val stateMachines = LinkedHashMap<ProtocolStateMachineImpl<*>, Checkpoint>()
+        val changesPublisher = PublishSubject.create<Change>()
+
+        fun notifyChangeObservers(psm: ProtocolStateMachineImpl<*>, addOrRemove: AddOrRemove) {
+            changesPublisher.onNext(Change(psm.logic, addOrRemove, psm.stateMachineRunId))
+        }
+    })
 
     // Monitoring support.
     private val metrics = serviceHub.monitoringService.metrics
 
     init {
-        metrics.register("Protocols.InFlight", Gauge<Int> { stateMachines.size })
+        metrics.register("Protocols.InFlight", Gauge<Int> { mutex.content.stateMachines.size })
     }
 
     private val checkpointingMeter = metrics.meter("Protocols.Checkpointing Rate")
     private val totalStartedProtocols = metrics.counter("Protocols.Started")
     private val totalFinishedProtocols = metrics.counter("Protocols.Finished")
-    private var started = false
 
     // Context for tokenized services in checkpoints
     private val serializationContext = SerializeAsTokenContext(tokenizableServices, quasarKryo())
 
     /** Returns a list of all state machines executing the given protocol logic at the top level (subprotocols do not count) */
     fun <P : ProtocolLogic<T>, T> findStateMachines(protocolClass: Class<P>): List<Pair<P, ListenableFuture<T>>> {
-        synchronized(stateMachines) {
         @Suppress("UNCHECKED_CAST")
-            return stateMachines.keys
+        return mutex.locked {
+            stateMachines.keys
                     .map { it.logic }
                     .filterIsInstance(protocolClass)
                     .map { it to (it.psm as ProtocolStateMachineImpl<T>).resultFuture }
@@ -91,19 +106,25 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
     }
 
     val allStateMachines: List<ProtocolLogic<*>>
-        get() = stateMachines.keys.map { it.logic }
+        get() = mutex.locked { stateMachines.keys.map { it.logic } }
-
-    private val _changesPublisher = PublishSubject.create<Triple<ProtocolLogic<*>, AddOrRemove, Long>>()
 
     /**
      * An observable that emits triples of the changing protocol, the type of change, and a process-specific ID number
      * which may change across restarts.
      */
-    val changes: Observable<Triple<ProtocolLogic<*>, AddOrRemove, Long>>
+    val changes: Observable<Change>
-        get() = _changesPublisher
+        get() = mutex.content.changesPublisher
 
-    private fun notifyChangeObservers(psm: ProtocolStateMachineImpl<*>, change: AddOrRemove) {
+    /**
-        _changesPublisher.onNext(Triple(psm.logic, change, psm.id))
+     * Atomic get snapshot + subscribe. This is needed so we don't miss updates between subscriptions to [changes] and
+     * calls to [allStateMachines]
+     */
+    fun getAllStateMachinesAndChanges(): Pair<List<ProtocolStateMachineImpl<*>>, Observable<Change>> {
+        return mutex.locked {
+            val bufferedChanges = UnicastSubject.create<Change>()
+            changesPublisher.subscribe(bufferedChanges)
+            Pair(stateMachines.keys.toList(), bufferedChanges)
+        }
     }
 
     // Used to work around a small limitation in Quasar.
@@ -122,7 +143,7 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
     fun start() {
         checkpointStorage.checkpoints.forEach { createFiberForCheckpoint(it) }
         serviceHub.networkMapCache.mapServiceRegistered.then(executor) {
-            synchronized(started) {
+            mutex.locked {
                 started = true
                 stateMachines.forEach { restartFiber(it.key, it.value) }
             }
@@ -205,6 +226,7 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
         }
         psm.actionOnEnd = {
             psm.logic.progressTracker?.currentStep = ProgressTracker.DONE
+            mutex.locked {
                 val finalCheckpoint = stateMachines.remove(psm)
                 if (finalCheckpoint != null) {
                     checkpointStorage.removeCheckpoint(finalCheckpoint)
@@ -212,11 +234,14 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
                 totalFinishedProtocols.inc()
                 notifyChangeObservers(psm, AddOrRemove.REMOVE)
             }
+        }
         val checkpoint = startingCheckpoint()
         checkpoint.fiberCreated = true
         totalStartedProtocols.inc()
+        mutex.locked {
             stateMachines[psm] = checkpoint
             notifyChangeObservers(psm, AddOrRemove.ADD)
+        }
         return checkpoint
     }
 
@@ -226,14 +251,15 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
      * restarted with checkpointed state machines in the storage service.
      */
     fun <T> add(loggerName: String, logic: ProtocolLogic<T>): ProtocolStateMachine<T> {
-        val fiber = ProtocolStateMachineImpl(logic, scheduler, loggerName)
+        val id = StateMachineRunId.createRandom()
+        val fiber = ProtocolStateMachineImpl(id, logic, scheduler, loggerName)
         // Need to add before iterating in case of immediate completion
         val checkpoint = initFiber(fiber) {
             val checkpoint = Checkpoint(serializeFiber(fiber), null, null)
             checkpoint
         }
         checkpointStorage.addCheckpoint(checkpoint)
-        synchronized(started) { // If we are not started then our checkpoint will be picked up during start
+        mutex.locked { // If we are not started then our checkpoint will be picked up during start
             if (!started) {
                 return fiber
             }
@@ -267,7 +293,9 @@ class StateMachineManager(val serviceHub: ServiceHubInternal, tokenizableService
                                  request: ProtocolIORequest?,
                                  receivedPayload: Any?) {
         val newCheckpoint = Checkpoint(serialisedFiber, request, receivedPayload)
-        val previousCheckpoint = stateMachines.put(psm, newCheckpoint)
+        val previousCheckpoint = mutex.locked {
+            stateMachines.put(psm, newCheckpoint)
+        }
         if (previousCheckpoint != null) {
             checkpointStorage.removeCheckpoint(previousCheckpoint)
         }

node/src/main/kotlin/com/r3corda/node/utilities/ANSIProgressObserver.kt

@@ -13,10 +13,10 @@ import java.util.*
 class ANSIProgressObserver(val smm: StateMachineManager) {
 
     init {
-        smm.changes.subscribe { change: Triple<ProtocolLogic<*>, AddOrRemove, Long> ->
+        smm.changes.subscribe { change ->
-            when (change.second) {
+            when (change.addOrRemove) {
-                AddOrRemove.ADD -> addProtocolLogic(change.first)
+                AddOrRemove.ADD -> addProtocolLogic(change.logic)
-                AddOrRemove.REMOVE -> removeProtocolLogic(change.first)
+                AddOrRemove.REMOVE -> removeProtocolLogic(change.logic)
             }
         }
     }

node/src/test/kotlin/com/r3corda/node/services/NodeSchedulerServiceTest.kt

@@ -86,8 +86,8 @@ class NodeSchedulerServiceTest : SingletonSerializeAsToken() {
         scheduler = NodeSchedulerService(services, factory, schedulerGatedExecutor)
         smmExecutor = AffinityExecutor.ServiceAffinityExecutor("test", 1)
         val mockSMM = StateMachineManager(services, listOf(services), PerFileCheckpointStorage(fs.getPath("checkpoints")), smmExecutor)
-        mockSMM.changes.subscribe { change:Triple<ProtocolLogic<*>, AddOrRemove, Long> ->
+        mockSMM.changes.subscribe { change ->
-            if(change.second==AddOrRemove.REMOVE && mockSMM.allStateMachines.size==0) {
+            if (change.addOrRemove == AddOrRemove.REMOVE && mockSMM.allStateMachines.isEmpty()) {
                 smmHasRemovedAllProtocols.countDown()
             }
         }

src/main/kotlin/com/r3corda/simulation/Simulation.kt

@@ -14,6 +14,7 @@ import com.r3corda.demos.api.NodeInterestRates
 import com.r3corda.node.services.config.NodeConfiguration
 import com.r3corda.node.services.network.NetworkMapService
 import com.r3corda.node.services.transactions.SimpleNotaryService
+import com.r3corda.node.utilities.AddOrRemove
 import com.r3corda.testing.node.InMemoryMessagingNetwork
 import com.r3corda.testing.node.MockNetwork
 import com.r3corda.testing.node.TestClock
@@ -259,8 +260,8 @@ abstract class Simulation(val networkSendManuallyPumped: Boolean,
 
     protected fun showProgressFor(nodes: List<SimulatedNode>) {
         nodes.forEach { node ->
-            node.smm.changes.filter { it.second == com.r3corda.node.utilities.AddOrRemove.ADD }.first().subscribe {
+            node.smm.changes.filter { it.addOrRemove == AddOrRemove.ADD }.first().subscribe {
-                linkProtocolProgress(node, it.first)
+                linkProtocolProgress(node, it.logic)
             }
         }
     }
@@ -278,8 +279,8 @@ abstract class Simulation(val networkSendManuallyPumped: Boolean,
 
     protected fun showConsensusFor(nodes: List<SimulatedNode>) {
         val node = nodes.first()
-        node.smm.changes.filter { it.second == com.r3corda.node.utilities.AddOrRemove.ADD }.first().subscribe {
+        node.smm.changes.filter { it.addOrRemove == com.r3corda.node.utilities.AddOrRemove.ADD }.first().subscribe {
-            linkConsensus(nodes, it.first)
+            linkConsensus(nodes, it.logic)
         }
     }