add a shared memory port allocator to allow multiple processes to sha… (#5223)

* add a shared memory port allocator to allow multiple processes to share a single allocation pool

* remove dangerous reset function on port allocator

* set forkCount = 2 in node integration test

* only allow one build of a cordapp at any given time for Driver tests

* make all portallocation requests use same starting point

* globally set forks to 6

* tweak forking parameters to allow parallel builds

* tweak unit test parallelism

* 2 workers for integrationTest

* some more tweaks for parallel builds

* some more tweaks for parallel builds

* seems that 49K is not the start of ephemeral ports on all kernels

* tweak parallel settings

* try fix RPC shutdown test in parallel env

* add some logging for RPC shutdown test

* added some logging around PortAllocation tests - try figure out where they are getting stuck

* added some logging around PortAllocation tests - try figure out where they are getting stuck

* fix api-scanner tests

* minimize api changes

* revert to complying with existing API

* add the AtomicInteger for api compatibility reasons

* make sizing script executable

* address review comments pt1

* address review comments pt2

* fix compile errors after review comments

* return to using home dir as temp dir seemed to interact badly with gradle

testing/node-driver/src/test/java/net/corda/testing/node/PortAllocationRunner.java

@@ -0,0 +1,44 @@
+package net.corda.testing.node;
+
+import net.corda.testing.driver.PortAllocation;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+
+public class PortAllocationRunner {
+
+    public static void main(@NotNull String[] args) throws IOException {
+        /*
+         * each JVM will be launched with [spinnerFile, reportingIndex]
+         */
+        int reportingIndex = Integer.parseInt(args[1]);
+
+        RandomAccessFile spinnerBackingFile = new RandomAccessFile(args[0], "rw");
+        MappedByteBuffer spinnerBuffer = spinnerBackingFile.getChannel().map(FileChannel.MapMode.READ_WRITE, 0, 16);
+
+        /*
+         * notify back to launching process that we are ready to start using the reporting index we were allocated on launch
+         */
+        spinnerBuffer.putShort(reportingIndex, ((short) 10));
+
+        /*
+         * wait for parent process to notify us that all waiting processes are good to go
+         * do not Thread.sleep as we want to ensure there is as much of an overlap between the ports selected by the spawned processes
+         * and by sleeping, its frequently the case that one will complete selection before another wakes up resulting in  sequential ranges rather than overlapping
+         */
+        while (spinnerBuffer.getShort(0) != 8) {
+        }
+
+        /*
+         * allocate ports and print out for later consumption by the spawning test
+         */
+        PortAllocation pa = PortAllocation.getDefaultAllocator();
+        for (int i = 0; i < 10000; i++) {
+            System.out.println(pa.nextPort());
+        }
+    }
+
+}

testing/node-driver/src/test/kotlin/net/corda/testing/driver/PortAllocationTest.kt

@@ -0,0 +1,96 @@
+package net.corda.testing.driver
+
+import net.corda.testing.node.PortAllocationRunner
+import org.assertj.core.util.Files
+import org.hamcrest.CoreMatchers.`is`
+import org.hamcrest.core.IsNot.not
+import org.hamcrest.number.OrderingComparison
+import org.junit.Assert
+import org.junit.Test
+import java.io.RandomAccessFile
+import java.nio.channels.FileChannel
+import java.util.concurrent.TimeUnit
+
+class PortAllocationTest {
+
+    @Test
+    fun `should allocate a port whilst cycling back round if exceeding start of ephemeral range`() {
+        val startingPoint = PortAllocation.DEFAULT_START_PORT
+        val portAllocator = PortAllocation.defaultAllocator
+
+        var previous = portAllocator.nextPort()
+        (0 until 1000000).forEach { _ ->
+            val next = portAllocator.nextPort()
+            Assert.assertThat(next, `is`(not(previous)))
+            Assert.assertThat(next, `is`(OrderingComparison.lessThan(PortAllocation.FIRST_EPHEMERAL_PORT)))
+
+            if (next == startingPoint) {
+                Assert.assertThat(previous, `is`(PortAllocation.FIRST_EPHEMERAL_PORT - 1))
+            } else {
+                Assert.assertThat(next, `is`(previous + 1))
+            }
+            previous = next
+        }
+    }
+
+    @Test(timeout = 120_000)
+    fun `should support multiprocess port allocation`() {
+
+        println("Starting multiprocess port allocation test")
+        val spinnerFile = Files.newTemporaryFile().also { it.deleteOnExit() }.absolutePath
+        val process1 = buildJvmProcess(spinnerFile, 1)
+        val process2 = buildJvmProcess(spinnerFile, 2)
+
+        println("Started child processes")
+
+        val processes = listOf(process1, process2)
+
+        val spinnerBackingFile = RandomAccessFile(spinnerFile, "rw")
+        println("Mapped spinner file")
+        val spinnerBuffer = spinnerBackingFile.channel.map(FileChannel.MapMode.READ_WRITE, 0, 512)
+        println("Created spinner buffer")
+
+        var timeWaited = 0L
+
+        while (spinnerBuffer.getShort(1) != 10.toShort() && spinnerBuffer.getShort(2) != 10.toShort() && timeWaited < 60_000) {
+            println("Waiting to childProcesses to report back. waited ${timeWaited}ms")
+            Thread.sleep(1000)
+            timeWaited += 1000
+        }
+
+        //GO!
+        println("Instructing child processes to start allocating ports")
+        spinnerBuffer.putShort(0, 8)
+        println("Waiting for child processes to terminate")
+        processes.forEach { it.waitFor(1, TimeUnit.MINUTES) }
+        println("child processes terminated")
+
+        val process1Output = process1.inputStream.reader().readLines().toSet()
+        val process2Output = process2.inputStream.reader().readLines().toSet()
+
+        println("child process out captured")
+
+        Assert.assertThat(process1Output.size, `is`(10_000))
+        Assert.assertThat(process2Output.size, `is`(10_000))
+
+        //there should be no overlap between the outputs as each process should have been allocated a unique set of ports
+        Assert.assertThat(process1Output.intersect(process2Output), `is`(emptySet()))
+    }
+
+    private fun buildJvmProcess(spinnerFile: String, reportingIndex: Int): Process {
+        val separator = System.getProperty("file.separator")
+        val classpath = System.getProperty("java.class.path")
+        val path = (System.getProperty("java.home")
+                + separator + "bin" + separator + "java")
+        val processBuilder = ProcessBuilder(path, "-cp",
+                classpath,
+                PortAllocationRunner::class.java.name,
+                spinnerFile,
+                reportingIndex.toString())
+
+        return processBuilder.start()
+    }
+}
+
+
+