mirror of
https://github.com/corda/corda.git
synced 2025-06-12 20:28:18 +00:00
[CORDA-442] let Driver run without network map (#1890)
* [CORDA-442] let Driver run without network map - Nodes started by driver run without a networkMapNode. - Driver does not take a networkMapStartStrategy anymore - a new parameter in the configuration "noNetworkMapServiceMode" allows for a node not to be a networkMapNode nor to connect to one. - Driver now waits for each node to write its own NodeInfo file to disk and then copies it into each other node. - When driver starts a node N, it waits for every node to be have N nodes in their network map. Note: the code to copy around the NodeInfo files was already in DemoBench, the NodeInfoFilesCopier class was just moved from DemoBench into core (I'm very open to core not being the best place, please advise)
This commit is contained in:
Alberto Arri
GitHub
@ -0,0 +1,165 @@
|
||||
package net.corda.nodeapi
|
||||
|
||||
import net.corda.cordform.CordformNode
|
||||
import net.corda.core.internal.ThreadBox
|
||||
import net.corda.core.internal.createDirectories
|
||||
import net.corda.core.internal.isRegularFile
|
||||
import net.corda.core.internal.list
|
||||
import net.corda.core.utilities.loggerFor
|
||||
import rx.Observable
|
||||
import rx.Scheduler
|
||||
import rx.Subscription
|
||||
import rx.schedulers.Schedulers
|
||||
import java.io.IOException
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.nio.file.StandardCopyOption.COPY_ATTRIBUTES
|
||||
import java.nio.file.StandardCopyOption.REPLACE_EXISTING
|
||||
import java.nio.file.attribute.BasicFileAttributes
|
||||
import java.nio.file.attribute.FileTime
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* Utility class which copies nodeInfo files across a set of running nodes.
|
||||
*
|
||||
* This class will create paths that it needs to poll and to where it needs to copy files in case those
|
||||
* don't exist yet.
|
||||
*/
|
||||
class NodeInfoFilesCopier(scheduler: Scheduler = Schedulers.io()) : AutoCloseable {
|
||||
|
||||
companion object {
|
||||
private val log = loggerFor<NodeInfoFilesCopier>()
|
||||
const val NODE_INFO_FILE_NAME_PREFIX = "nodeInfo-"
|
||||
}
|
||||
|
||||
private val nodeDataMapBox = ThreadBox(mutableMapOf<Path, NodeData>())
|
||||
/**
|
||||
* Whether the NodeInfoFilesCopier is closed. When the NodeInfoFilesCopier is closed it will stop polling the
|
||||
* filesystem and all the public methods except [#close] will throw.
|
||||
*/
|
||||
private var closed = false
|
||||
private val subscription: Subscription
|
||||
|
||||
init {
|
||||
this.subscription = Observable.interval(5, TimeUnit.SECONDS, scheduler)
|
||||
.subscribe { poll() }
|
||||
}
|
||||
|
||||
/**
|
||||
* @param nodeDir a path to be watched for NodeInfos
|
||||
* Add a path of a node which is about to be started.
|
||||
* Its nodeInfo file will be copied to other nodes' additional-node-infos directory, and conversely,
|
||||
* other nodes' nodeInfo files will be copied to this node additional-node-infos directory.
|
||||
*/
|
||||
fun addConfig(nodeDir: Path) {
|
||||
require(!closed) { "NodeInfoFilesCopier is already closed" }
|
||||
nodeDataMapBox.locked {
|
||||
val newNodeFile = NodeData(nodeDir)
|
||||
put(nodeDir, newNodeFile)
|
||||
|
||||
for (previouslySeenFile in allPreviouslySeenFiles()) {
|
||||
atomicCopy(previouslySeenFile, newNodeFile.additionalNodeInfoDirectory.resolve(previouslySeenFile.fileName))
|
||||
}
|
||||
log.info("Now watching: $nodeDir")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param nodeConfig the configuration to be removed.
|
||||
* Remove the configuration of a node which is about to be stopped or already stopped.
|
||||
* No files written by that node will be copied to other nodes, nor files from other nodes will be copied to this
|
||||
* one.
|
||||
*/
|
||||
fun removeConfig(nodeDir: Path) {
|
||||
require(!closed) { "NodeInfoFilesCopier is already closed" }
|
||||
nodeDataMapBox.locked {
|
||||
remove(nodeDir) ?: return
|
||||
log.info("Stopped watching: $nodeDir")
|
||||
}
|
||||
}
|
||||
|
||||
fun reset() {
|
||||
require(!closed) { "NodeInfoFilesCopier is already closed" }
|
||||
nodeDataMapBox.locked {
|
||||
clear()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops polling the filesystem.
|
||||
* This function can be called as many times as one wants.
|
||||
*/
|
||||
override fun close() {
|
||||
if (!closed) {
|
||||
closed = true
|
||||
subscription.unsubscribe()
|
||||
}
|
||||
}
|
||||
|
||||
private fun allPreviouslySeenFiles() = nodeDataMapBox.alreadyLocked { values.flatMap { it.previouslySeenFiles.keys } }
|
||||
|
||||
private fun poll() {
|
||||
nodeDataMapBox.locked {
|
||||
for (nodeData in values) {
|
||||
nodeData.nodeDir.list { paths ->
|
||||
paths.filter { it.isRegularFile() }
|
||||
.filter { it.fileName.toString().startsWith(NODE_INFO_FILE_NAME_PREFIX) }
|
||||
.forEach { path -> processPath(nodeData, path) }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Takes a path under nodeData config dir and decides whether the file represented by that path needs to
|
||||
// be copied.
|
||||
private fun processPath(nodeData: NodeData, path: Path) {
|
||||
nodeDataMapBox.alreadyLocked {
|
||||
val newTimestamp = Files.readAttributes(path, BasicFileAttributes::class.java).lastModifiedTime()
|
||||
val previousTimestamp = nodeData.previouslySeenFiles.put(path, newTimestamp) ?: FileTime.fromMillis(-1)
|
||||
if (newTimestamp > previousTimestamp) {
|
||||
for (destination in this.values.filter { it.nodeDir != nodeData.nodeDir }.map { it.additionalNodeInfoDirectory }) {
|
||||
val fullDestinationPath = destination.resolve(path.fileName)
|
||||
atomicCopy(path, fullDestinationPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private fun atomicCopy(source: Path, destination: Path) {
|
||||
val tempDestination = try {
|
||||
Files.createTempFile(destination.parent, "", null)
|
||||
} catch (exception: IOException) {
|
||||
log.warn("Couldn't create a temporary file to copy $source", exception)
|
||||
throw exception
|
||||
}
|
||||
try {
|
||||
// First copy the file to a temporary file within the appropriate directory.
|
||||
Files.copy(source, tempDestination, COPY_ATTRIBUTES, REPLACE_EXISTING)
|
||||
} catch (exception: IOException) {
|
||||
log.warn("Couldn't copy $source to $tempDestination.", exception)
|
||||
Files.delete(tempDestination)
|
||||
throw exception
|
||||
}
|
||||
try {
|
||||
// Then rename it to the desired name. This way the file 'appears' on the filesystem as an atomic operation.
|
||||
Files.move(tempDestination, destination, REPLACE_EXISTING)
|
||||
} catch (exception: IOException) {
|
||||
log.warn("Couldn't move $tempDestination to $destination.", exception)
|
||||
Files.delete(tempDestination)
|
||||
throw exception
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience holder for all the paths and files relative to a single node.
|
||||
*/
|
||||
private class NodeData(val nodeDir: Path) {
|
||||
val additionalNodeInfoDirectory: Path = nodeDir.resolve(CordformNode.NODE_INFO_DIRECTORY)
|
||||
// Map from Path to its lastModifiedTime.
|
||||
val previouslySeenFiles = mutableMapOf<Path, FileTime>()
|
||||
|
||||
init {
|
||||
additionalNodeInfoDirectory.createDirectories()
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,139 @@
|
||||
package net.corda.nodeapi
|
||||
|
||||
import net.corda.cordform.CordformNode
|
||||
import net.corda.testing.eventually
|
||||
import org.junit.Before
|
||||
import org.junit.Rule
|
||||
import org.junit.Test
|
||||
import org.junit.rules.TemporaryFolder
|
||||
import rx.schedulers.TestScheduler
|
||||
import java.nio.file.Files
|
||||
import java.nio.file.Path
|
||||
import java.time.Duration
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.streams.toList
|
||||
import kotlin.test.assertEquals
|
||||
|
||||
/**
|
||||
* tests for [NodeInfoFilesCopier]
|
||||
*/
|
||||
class NodeInfoFilesCopierTest {
|
||||
|
||||
@Rule @JvmField var folder = TemporaryFolder()
|
||||
private val rootPath get() = folder.root.toPath()
|
||||
private val scheduler = TestScheduler()
|
||||
companion object {
|
||||
private const val ORGANIZATION = "Organization"
|
||||
private const val NODE_1_PATH = "node1"
|
||||
private const val NODE_2_PATH = "node2"
|
||||
|
||||
private val content = "blah".toByteArray(Charsets.UTF_8)
|
||||
private val GOOD_NODE_INFO_NAME = "${NodeInfoFilesCopier.NODE_INFO_FILE_NAME_PREFIX}test"
|
||||
private val GOOD_NODE_INFO_NAME_2 = "${NodeInfoFilesCopier.NODE_INFO_FILE_NAME_PREFIX}anotherNode"
|
||||
private val BAD_NODE_INFO_NAME = "something"
|
||||
}
|
||||
|
||||
private fun nodeDir(nodeBaseDir : String) = rootPath.resolve(nodeBaseDir).resolve(ORGANIZATION.toLowerCase())
|
||||
|
||||
private val node1RootPath by lazy { nodeDir(NODE_1_PATH) }
|
||||
private val node2RootPath by lazy { nodeDir(NODE_2_PATH) }
|
||||
private val node1AdditionalNodeInfoPath by lazy { node1RootPath.resolve(CordformNode.NODE_INFO_DIRECTORY) }
|
||||
private val node2AdditionalNodeInfoPath by lazy { node2RootPath.resolve(CordformNode.NODE_INFO_DIRECTORY) }
|
||||
|
||||
lateinit var nodeInfoFilesCopier: NodeInfoFilesCopier
|
||||
|
||||
@Before
|
||||
fun setUp() {
|
||||
nodeInfoFilesCopier = NodeInfoFilesCopier(scheduler)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `files created before a node is started are copied to that node`() {
|
||||
// Configure the first node.
|
||||
nodeInfoFilesCopier.addConfig(node1RootPath)
|
||||
// Ensure directories are created.
|
||||
advanceTime()
|
||||
|
||||
// Create 2 files, a nodeInfo and another file in node1 folder.
|
||||
Files.write(node1RootPath.resolve(GOOD_NODE_INFO_NAME), content)
|
||||
Files.write(node1RootPath.resolve(BAD_NODE_INFO_NAME), content)
|
||||
|
||||
// Configure the second node.
|
||||
nodeInfoFilesCopier.addConfig(node2RootPath)
|
||||
advanceTime()
|
||||
|
||||
eventually<AssertionError, Unit>(Duration.ofMinutes(1)) {
|
||||
// Check only one file is copied.
|
||||
checkDirectoryContainsSingleFile(node2AdditionalNodeInfoPath, GOOD_NODE_INFO_NAME)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `polling of running nodes`() {
|
||||
// Configure 2 nodes.
|
||||
nodeInfoFilesCopier.addConfig(node1RootPath)
|
||||
nodeInfoFilesCopier.addConfig(node2RootPath)
|
||||
advanceTime()
|
||||
|
||||
// Create 2 files, one of which to be copied, in a node root path.
|
||||
Files.write(node2RootPath.resolve(GOOD_NODE_INFO_NAME), content)
|
||||
Files.write(node2RootPath.resolve(BAD_NODE_INFO_NAME), content)
|
||||
advanceTime()
|
||||
|
||||
eventually<AssertionError, Unit>(Duration.ofMinutes(1)) {
|
||||
// Check only one file is copied to the other node.
|
||||
checkDirectoryContainsSingleFile(node1AdditionalNodeInfoPath, GOOD_NODE_INFO_NAME)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `remove nodes`() {
|
||||
// Configure 2 nodes.
|
||||
nodeInfoFilesCopier.addConfig(node1RootPath)
|
||||
nodeInfoFilesCopier.addConfig(node2RootPath)
|
||||
advanceTime()
|
||||
|
||||
// Create a file, in node 2 root path.
|
||||
Files.write(node2RootPath.resolve(GOOD_NODE_INFO_NAME), content)
|
||||
advanceTime()
|
||||
|
||||
// Remove node 2
|
||||
nodeInfoFilesCopier.removeConfig(node2RootPath)
|
||||
|
||||
// Create another file in node 2 directory.
|
||||
Files.write(node2RootPath.resolve(GOOD_NODE_INFO_NAME_2), content)
|
||||
advanceTime()
|
||||
|
||||
eventually<AssertionError, Unit>(Duration.ofMinutes(1)) {
|
||||
// Check only one file is copied to the other node.
|
||||
checkDirectoryContainsSingleFile(node1AdditionalNodeInfoPath, GOOD_NODE_INFO_NAME)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `clear`() {
|
||||
// Configure 2 nodes.
|
||||
nodeInfoFilesCopier.addConfig(node1RootPath)
|
||||
nodeInfoFilesCopier.addConfig(node2RootPath)
|
||||
advanceTime()
|
||||
|
||||
nodeInfoFilesCopier.reset()
|
||||
|
||||
advanceTime()
|
||||
Files.write(node2RootPath.resolve(GOOD_NODE_INFO_NAME_2), content)
|
||||
|
||||
// Give some time to the filesystem to report the change.
|
||||
Thread.sleep(100)
|
||||
assertEquals(0, Files.list(node1AdditionalNodeInfoPath).toList().size)
|
||||
}
|
||||
|
||||
private fun advanceTime() {
|
||||
scheduler.advanceTimeBy(1, TimeUnit.HOURS)
|
||||
}
|
||||
|
||||
private fun checkDirectoryContainsSingleFile(path: Path, filename: String) {
|
||||
assertEquals(1, Files.list(path).toList().size)
|
||||
val onlyFileName = Files.list(path).toList().first().fileName.toString()
|
||||
assertEquals(filename, onlyFileName)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user