Add simulation of the avalanche consensus protocol to experimental (#3283)

.idea/compiler.xml

@@ -7,6 +7,8 @@
       <module name="attachment-demo_integrationTest" target="1.8" />
       <module name="attachment-demo_main" target="1.8" />
       <module name="attachment-demo_test" target="1.8" />
+      <module name="avalanche_main" target="1.8" />
+      <module name="avalanche_test" target="1.8" />
       <module name="bank-of-corda-demo_integrationTest" target="1.8" />
       <module name="bank-of-corda-demo_main" target="1.8" />
       <module name="bank-of-corda-demo_test" target="1.8" />

experimental/avalanche/Readme.md

@@ -0,0 +1,24 @@
+# Avalanche Simulation
+
+Experimental simulation of the Avalanche protocol by Team Rocket. This
+implementation is incomplete.
+
+The paper: [Snowflake to Avalanche: A Novel Metastable Consensus Protocol Family for
+           Cryptocurrencies](https://ipfs.io/ipfs/QmUy4jh5mGNZvLkjies1RWM4YuvJh5o2FYopNPVYwrRVGV).
+
+## Running the Simulation
+```
+./gradlew shadowJar
+java -jar build/libs/avalanche-all.jar --dump-dags
+```
+
+### Visualising the DAGs
+```
+for f in node-0-*.dot; do dot -Tpng -O $f; done
+```
+The above command generates a number of PNG files `node-0-*.png`, showing the
+evolution of the DAG. The nodes are labeled with the ID of the spent state,
+the chit and confidence values. The prefered transaction of a conflict set is
+labelled with a star. Accepted transactions are blue.
+
+![DAG](./images/node-0-003.dot.png)

experimental/avalanche/build.gradle

@@ -0,0 +1,42 @@
+buildscript {
+    ext.kotlin_version = '1.2.40'
+
+    repositories {
+        mavenCentral()
+        jcenter()
+    }
+
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.3'
+    }
+}
+
+plugins {
+    id "org.jetbrains.kotlin.jvm"
+    id 'com.github.johnrengelman.shadow' version '2.0.3'
+    id 'java'
+    id 'application'
+}
+repositories {
+    mavenCentral()
+}
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
+    compile group: 'info.picocli', name: 'picocli', version: '3.0.1'
+    testCompile group: 'junit', name: 'junit', version: '4.12'
+}
+compileKotlin {
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+}
+compileTestKotlin {
+    kotlinOptions {
+        jvmTarget = "1.8"
+    }
+}
+mainClassName = "net.corda.avalanche.MainKt"
+shadowJar {
+    baseName = "avalanche"
+}

experimental/avalanche/src/main/kotlin/net/corda/avalanche/Main.kt

@@ -0,0 +1,231 @@
+package net.corda.avalanche
+
+import picocli.CommandLine
+import java.io.File
+import java.util.*
+import kotlin.collections.LinkedHashMap
+
+fun main(args: Array<String>) {
+
+    val parameters = Parameters()
+    CommandLine(parameters).parse(*args)
+    if (parameters.helpRequested) {
+        CommandLine.usage(Parameters(), System.out)
+        return
+    }
+
+    val network = Network(parameters)
+    val n1 = network.nodes[0]
+    val c1 = mutableListOf<Transaction>()
+    val c2 = mutableListOf<Transaction>()
+
+    repeat(parameters.nrTransactions) {
+        val n = network.nodes.shuffled(network.rng).first()
+        c1.add(n.onGenerateTx(it))
+        if (network.rng.nextDouble() < parameters.doubleSpendRatio) {
+            val d = network.rng.nextInt(it)
+            println("double spend of $d")
+            val n2 = network.nodes.shuffled(network.rng).first()
+            c2.add(n2.onGenerateTx(d))
+        }
+
+        network.run()
+
+        if (parameters.dumpDags) {
+            n1.dumpDag(File("node-0-${String.format("%03d", it)}.dot"))
+        }
+        println("$it: " + String.format("%.3f", fractionAccepted(n1)))
+    }
+
+    val conflictSets = (c1 + c2).groupBy { it.data }.filterValues { it.size > 1 }
+    conflictSets.forEach { v, txs ->
+        val acceptance = txs.map { t -> network.nodes.map { it.isAccepted(t) }.any { it } }
+        require(acceptance.filter { it }.size < 2) { "More than one transaction of the conflict set of $v got accepted." }
+    }
+}
+
+fun fractionAccepted(n: Node): Double {
+    val accepted = n.transactions.values.filter { n.isAccepted(it) }.size
+    return accepted.toDouble() / n.transactions.size
+}
+
+data class Transaction(
+        val id: UUID,
+        val data: Int,
+        val parents: List<UUID>,
+        var chit: Int = 0,
+        var confidence: Int = 0) {
+    override fun toString(): String {
+        return "T(id=${id.toString().take(5)}, data=$data, parents=[${parents.map {it.toString().take(5) }}, chit=$chit, confidence=$confidence)"
+    }
+}
+
+data class ConflictSet(
+        var pref: Transaction,
+        var last: Transaction,
+        var count: Int,
+        var size: Int
+)
+
+class Network(val parameters: Parameters) {
+    val rng = Random(parameters.seed)
+    val tx = Transaction(UUID.randomUUID(), -1, emptyList(), 1)
+    val nodes = (0..parameters.nrNodes).map { Node(it, parameters, tx.copy(),this, rng) }
+    fun run() {
+        nodes.forEach { it.avalancheLoop() }
+    }
+}
+
+class Node(val id: Int, parameters: Parameters, val genesisTx: Transaction, val network: Network, val rng: Random) {
+
+    val alpha = parameters.alpha
+    val k  = parameters.k
+    val beta1 = parameters.beta1
+    val beta2 = parameters.beta2
+
+    val transactions = LinkedHashMap<UUID, Transaction>(mapOf(genesisTx.id to genesisTx))
+    val queried = mutableSetOf<UUID>(genesisTx.id)
+    val conflicts = mutableMapOf<Int, ConflictSet>(genesisTx.data to ConflictSet(genesisTx, genesisTx, 0, 1))
+
+    val accepted = mutableSetOf<UUID>(genesisTx.id)
+    val parentSets = mutableMapOf<UUID, Set<Transaction>>()
+
+    fun onGenerateTx(data: Int): Transaction {
+       val edges = parentSelection()
+       val t = Transaction(UUID.randomUUID(), data, edges.map { it.id })
+        onReceiveTx(this, t)
+        return t
+    }
+
+    fun onReceiveTx(sender: Node, tx: Transaction) {
+        if (transactions.contains(tx.id)) return
+        tx.chit = 0
+        tx.confidence = 0
+
+        tx.parents.forEach {
+            if (!transactions.contains(it)) {
+                val t = sender.onSendTx(it)
+                onReceiveTx(sender, t)
+            }
+        }
+
+        if (!conflicts.contains(tx.data)) {
+            conflicts[tx.data] = ConflictSet(tx, tx, 0, 1)
+        } else {
+            conflicts[tx.data]!!.size++
+        }
+
+        transactions[tx.id] = tx
+    }
+
+    fun onSendTx(id: UUID): Transaction {
+        return transactions[id]!!.copy()
+    }
+
+    fun onQuery(sender: Node, tx: Transaction): Int {
+        onReceiveTx(sender, tx)
+        return if (isStronglyPreferred(tx)) 1
+               else 0
+    }
+
+    fun avalancheLoop() {
+        val txs = transactions.values.filterNot { queried.contains(it.id) }
+        txs.forEach { tx ->
+            val sample = network.nodes.filterNot { it == this }.shuffled(rng).take(k)
+            val res = sample.map {
+                val txCopy = tx.copy()
+                it.onQuery(this, txCopy)
+            }.sum()
+            if (res >= alpha * k) {
+                tx.chit = 1
+                // Update the preference for ancestors.
+                parentSet(tx).forEach { p ->
+                    p.confidence += 1
+                }
+                parentSet(tx).forEach { p->
+                    val cs = conflicts[p.data]!!
+                    if (p.confidence > cs.pref.confidence) {
+                       cs.pref = p
+                    }
+                    if (p != cs.last) {
+                        cs.last = p
+                        cs.count = 0
+                    } else {
+                        cs.count++
+                    }
+                }
+            }
+            queried.add(tx.id)
+        }
+    }
+
+    fun isPreferred(tx: Transaction): Boolean {
+        return conflicts[tx.data]!!.pref == tx
+    }
+
+    fun isStronglyPreferred(tx: Transaction): Boolean {
+        return parentSet(tx).map { isPreferred(it) }.all { it }
+    }
+
+    fun isAccepted(tx: Transaction): Boolean {
+        if (accepted.contains(tx.id)) return true
+        if (!queried.contains(tx.id)) return false
+
+        val cs = conflicts[tx.data]!!
+        val parentsAccepted = tx.parents.map { accepted.contains(it) }.all { it }
+        val isAccepted = (parentsAccepted && cs.size == 1 && tx.confidence > beta1) ||
+                (cs.pref == tx && cs.count > beta2)
+        if (isAccepted) accepted.add(tx.id)
+        return isAccepted
+    }
+
+    fun parentSet(tx: Transaction): Set<Transaction> {
+
+        if (parentSets.contains(tx.id)) return parentSets[tx.id]!!
+
+        val parents = mutableSetOf<Transaction>()
+        var ps = tx.parents.toSet()
+        while (ps.isNotEmpty()) {
+            ps.forEach {
+                if (transactions.contains(it)) parents.add(transactions[it]!!)
+            }
+            ps = ps.flatMap {
+                if (transactions.contains(it)) {
+                    transactions[it]!!.parents
+                } else {
+                    emptyList()
+                }
+            }.toSet()
+        }
+        parentSets[tx.id] = parents
+        return parents
+    }
+
+    fun parentSelection(): List<Transaction> {
+        val eps0 = transactions.values.filter { isStronglyPreferred(it) }
+        val eps1 = eps0.filter { conflicts[it.data]!!.size == 1 || it.confidence > 0 }
+        val parents = eps1.flatMap { parentSet(it) }.toSet().filterNot { eps1.contains(it) }
+        val fallback = if (transactions.size == 1) listOf(genesisTx)
+                        else transactions.values.reversed().take(10).filter { !isAccepted(it) && conflicts[it.data]!!.size == 1 }.shuffled(network.rng).take(3)
+        require(parents.isNotEmpty() || fallback.isNotEmpty()) { "Unable to select parents." }
+        return if (parents.isEmpty()) return fallback else parents
+    }
+
+    fun dumpDag(f: File) {
+        f.printWriter().use { out ->
+            out.println("digraph G {")
+            transactions.values.forEach {
+                val color = if (isAccepted(it)) "color=lightblue; style=filled;" else ""
+                val pref = if (conflicts[it.data]!!.size > 1 && isPreferred(it)) "*" else ""
+                val chit = if (queried.contains(it.id)) it.chit.toString() else "?"
+                out.println("\"${it.id}\" [$color label=\"${it.data}$pref, $chit, ${it.confidence}\"];")
+            }
+            transactions.values.forEach {
+                it.parents.forEach { p->
+                    out.println("\"${it.id}\" -> \"$p\";")
+                }
+            }
+            out.println("}")
+        }
+    }
+}

experimental/avalanche/src/main/kotlin/net/corda/avalanche/Parameters.kt

@@ -0,0 +1,38 @@
+package net.corda.avalanche
+
+import picocli.CommandLine
+
+class Parameters {
+    @CommandLine.Option(names = ["-n", "--num-transactions"], description = ["How many transactions to generate (default: 20)"])
+    var nrTransactions: Int = 20
+
+    @CommandLine.Option(names = ["-d", "--double-spend-ratio"], description = ["The double spend ratio (default: 0.02)"])
+    var doubleSpendRatio: Double = 0.02
+
+    @CommandLine.Option(names = ["-a", "--alpha"], description = ["The alpha parameter (default: 0.8)"])
+    var alpha = 0.8
+
+    @CommandLine.Option(names = ["--num-nodes"], description = ["The number of nodes (default: 50)"])
+    var nrNodes = 50
+
+    @CommandLine.Option(names = ["-k", "--sample-size"], description = ["The sample size (default `1 + nrNodes / 10`)"])
+    var k = 1 + nrNodes / 10
+
+    @CommandLine.Option(names = ["--beta1"], description = ["The beta1 parameter (default: 5)"])
+    var beta1 = 5
+
+    @CommandLine.Option(names = ["--beta2"], description = ["The beta1 parameter (default: 5)"])
+    var beta2 = 5
+
+    @CommandLine.Option(names = ["-h", "--help"], usageHelp = true, description = ["Display help and exit"])
+    var helpRequested = false
+
+    @CommandLine.Option(names = ["--seed"], description = ["The RNG seed (default: 23)"])
+    var seed = 23L
+
+    @CommandLine.Option(names = ["--dump-dags"], description = ["Dump DAGs in dot format (default: false)"])
+    var dumpDags = false
+
+    @CommandLine.Option(names = ["-v", "--verbose"], description=["Verbose mode (default: false)"])
+    var verbose = false
+}

settings.gradle

@@ -16,6 +16,7 @@ include 'client:rpc'
 include 'webserver'
 include 'webserver:webcapsule'
 include 'experimental'
+include 'experimental:avalanche'
 include 'experimental:behave'
 include 'experimental:sandbox'
 include 'experimental:quasar-hook'