From 805378ef1177f13c925a8a2dce2b626b2c8c16f6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 26 Nov 2020 20:53:57 -0500 Subject: [PATCH] Do more path stuff with FilePath --- src/allmydata/scripts/common.py | 2 +- src/allmydata/scripts/create_node.py | 5 +- src/allmydata/test/check_memory.py | 25 +++++---- src/allmydata/test/common.py | 2 +- src/allmydata/test/test_client.py | 74 ++++++++++++++------------- src/allmydata/test/test_introducer.py | 35 +++++++------ src/allmydata/test/test_system.py | 15 +++--- 7 files changed, 86 insertions(+), 72 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 6d633b502..f38a241dd 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -124,7 +124,7 @@ def write_introducer(basedir, petname, furl): Overwrite the node's ``introducers.yaml`` with a file containing the given introducer information. """ - FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent( + basedir.child(b"private").child(b"introducers.yaml").setContent( safe_dump({ "introducers": { petname: { diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 30024dfd9..a4b2213ed 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -5,6 +5,9 @@ import json from twisted.internet import reactor, defer from twisted.python.usage import UsageError +from twisted.python.filepath import ( + FilePath, +) from allmydata.scripts.common import ( BasedirOptions, @@ -308,7 +311,7 @@ def write_client_config(c, config): introducer = config.get("introducer", None) if introducer is not None: write_introducer( - config["basedir"], + FilePath(config["basedir"]), "default", introducer, ) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 3ab88b9c6..6ec90eeae 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -8,6 +8,9 @@ if PY2: from future.builtins import str # noqa: F401 from six.moves import cStringIO as StringIO +from twisted.python.filepath import ( + FilePath, +) from twisted.internet import defer, reactor, protocol, error from twisted.application import service, internet from twisted.web import client as tw_client @@ -184,15 +187,17 @@ class SystemFramework(pollmixin.PollMixin): self.introducer_furl = self.introducer.introducer_url def make_nodes(self): + root = FilePath(self.testdir) self.nodes = [] for i in range(self.numnodes): - nodedir = os.path.join(self.testdir, "node%d" % i) - os.makedirs(nodedir + b"/private") + nodedir = root.child("node%d" % (i,)) + private = nodedir.child("private") + private.makedirs() write_introducer(nodedir, "default", self.introducer_url) - f = open(os.path.join(nodedir, "tahoe.cfg"), "w") - f.write("[client]\n" - "shares.happy = 1\n" - "[storage]\n" + config = ( + "[client]\n" + "shares.happy = 1\n" + "[storage]\n" ) # the only tests for which we want the internal nodes to actually # retain shares are the ones where somebody's going to download @@ -204,13 +209,13 @@ class SystemFramework(pollmixin.PollMixin): # for these tests, we tell the storage servers to pretend to # accept shares, but really just throw them out, since we're # only testing upload and not download. - f.write("debug_discard = true\n") + config += "debug_discard = true\n" if self.mode in ("receive",): # for this mode, the client-under-test gets all the shares, # so our internal nodes can refuse requests - f.write("readonly = true\n") - f.close() - c = client.Client(basedir=nodedir) + config += "readonly = true\n" + nodedir.child("tahoe.cfg").setContent(config) + c = client.Client(basedir=nodedir.path) c.setServiceParent(self) self.nodes.append(c) # the peers will start running, eventually they will connect to each diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index cbe4d768c..9691877fd 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -251,7 +251,7 @@ class UseNode(object): ) write_introducer( - self.basedir.asBytesMode().path, + self.basedir, "default", self.introducer_furl, ) diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index e082a33d1..080716507 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -667,11 +667,11 @@ class AnonymousStorage(SyncTestCase): """ If anonymous storage access is enabled then the client announces it. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() write_introducer(basedir, "someintroducer", SOME_FURL) config = client.config_from_string( - basedir, + basedir.path, "tub.port", BASECONFIG + ( "[storage]\n" @@ -687,7 +687,7 @@ class AnonymousStorage(SyncTestCase): get_published_announcements(node), MatchesListwise([ matches_storage_announcement( - basedir, + basedir.path, anonymous=True, ), ]), @@ -699,11 +699,11 @@ class AnonymousStorage(SyncTestCase): If anonymous storage access is disabled then the client does not announce it nor does it write a fURL for it to beneath the node directory. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() write_introducer(basedir, "someintroducer", SOME_FURL) config = client.config_from_string( - basedir, + basedir.path, "tub.port", BASECONFIG + ( "[storage]\n" @@ -719,7 +719,7 @@ class AnonymousStorage(SyncTestCase): get_published_announcements(node), MatchesListwise([ matches_storage_announcement( - basedir, + basedir.path, anonymous=False, ), ]), @@ -737,10 +737,10 @@ class AnonymousStorage(SyncTestCase): possible to reach the anonymous storage server via the originally published fURL. """ - basedir = self.id() - os.makedirs(basedir + b"/private") + basedir = FilePath(self.id()) + basedir.child("private").makedirs() enabled_config = client.config_from_string( - basedir, + basedir.path, "tub.port", BASECONFIG + ( "[storage]\n" @@ -764,7 +764,7 @@ class AnonymousStorage(SyncTestCase): ) disabled_config = client.config_from_string( - basedir, + basedir.path, "tub.port", BASECONFIG + ( "[storage]\n" @@ -956,22 +956,24 @@ class Run(unittest.TestCase, testutil.StallMixin): A configuration consisting only of an introducer can be turned into a client node. """ - basedir = "test_client.Run.test_loadable" - os.makedirs(basedir + b"/private") + basedir = FilePath("test_client.Run.test_loadable") + private = basedir.child("private") + private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) - fileutil.write(os.path.join(basedir, client._Client.EXIT_TRIGGER_FILE), "") - yield client.create_client(basedir) + basedir.child("tahoe.cfg").setContent(BASECONFIG) + basedir.child(client._Client.EXIT_TRIGGER_FILE).touch() + yield client.create_client(basedir.path) @defer.inlineCallbacks def test_reloadable(self): - basedir = "test_client.Run.test_reloadable" - os.makedirs(basedir + b"/private") + basedir = FilePath("test_client.Run.test_reloadable") + private = basedir.child("private") + private.makedirs() dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus" write_introducer(basedir, "someintroducer", dummy) - fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG) - c1 = yield client.create_client(basedir) + basedir.child("tahoe.cfg").setContent(BASECONFIG) + c1 = yield client.create_client(basedir.path) c1.setServiceParent(self.sparent) # delay to let the service start up completely. I'm not entirely sure @@ -993,7 +995,7 @@ class Run(unittest.TestCase, testutil.StallMixin): # also change _check_exit_trigger to use it instead of a raw # reactor.stop, also instrument the shutdown event in an # attribute that we can check.) - c2 = yield client.create_client(basedir) + c2 = yield client.create_client(basedir.path) c2.setServiceParent(self.sparent) yield c2.disownServiceParent() @@ -1132,8 +1134,8 @@ class StorageAnnouncementTests(SyncTestCase): """ def setUp(self): super(StorageAnnouncementTests, self).setUp() - self.basedir = self.useFixture(TempDir()).path - create_node_dir(self.basedir, u"") + self.basedir = FilePath(self.useFixture(TempDir()).path) + create_node_dir(self.basedir.path, u"") # Write an introducer configuration or we can't observer # announcements. write_introducer(self.basedir, "someintroducer", SOME_FURL) @@ -1164,7 +1166,7 @@ enabled = {storage_enabled} No storage announcement is published if storage is not enabled. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config(storage_enabled=False), ) @@ -1186,7 +1188,7 @@ enabled = {storage_enabled} storage is enabled. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config(storage_enabled=True), ) @@ -1203,7 +1205,7 @@ enabled = {storage_enabled} # Match the following list (of one element) ... MatchesListwise([ # The only element in the list ... - matches_storage_announcement(self.basedir), + matches_storage_announcement(self.basedir.path), ]), )), ) @@ -1218,7 +1220,7 @@ enabled = {storage_enabled} value = u"thing" config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1238,7 +1240,7 @@ enabled = {storage_enabled} get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1259,7 +1261,7 @@ enabled = {storage_enabled} self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1281,7 +1283,7 @@ enabled = {storage_enabled} get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1307,7 +1309,7 @@ enabled = {storage_enabled} self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1343,7 +1345,7 @@ enabled = {storage_enabled} self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1359,7 +1361,7 @@ enabled = {storage_enabled} get_published_announcements, MatchesListwise([ matches_storage_announcement( - self.basedir, + self.basedir.path, options=[ matches_dummy_announcement( u"tahoe-lafs-dummy-v1", @@ -1381,7 +1383,7 @@ enabled = {storage_enabled} self.useFixture(UseTestPlugins()) config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, @@ -1408,7 +1410,7 @@ enabled = {storage_enabled} available on the system. """ config = client.config_from_string( - self.basedir, + self.basedir.path, "tub.port", self.get_config( storage_enabled=True, diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index f6497eeb4..1fb456e72 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -795,21 +795,24 @@ class Announcements(AsyncTestCase): Announcements received by an introducer client are written to that introducer client's cache file. """ - basedir = "introducer/ClientSeqnums/test_client_cache_1" - fileutil.make_dirs(basedir + b"/private") + basedir = FilePath("introducer/ClientSeqnums/test_client_cache_1") + private = basedir.child("private") + private.makedirs() write_introducer(basedir, "default", "nope") - cache_filepath = FilePath(os.path.join(basedir, "private", - "introducer_default_cache.yaml")) + cache_filepath = basedir.descendant([ + "private", + "introducer_default_cache.yaml", + ]) # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. - with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: + with basedir.child("tahoe.cfg").open("w") as f: f.write("[storage]\n") f.write("enabled = false\n") - c = yield create_client(basedir) + c = yield create_client(basedir.path) ic = c.introducer_clients[0] private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), "pub-") @@ -875,7 +878,7 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"], furl3) - c2 = yield create_client(basedir) + c2 = yield create_client(basedir.path) c2.introducer_clients[0]._load_announcements() yield flushEventualQueue() self.assertEqual(c2.storage_broker.get_all_serverids(), @@ -885,26 +888,24 @@ class ClientSeqnums(AsyncBrokenTestCase): @defer.inlineCallbacks def test_client(self): - basedir = "introducer/ClientSeqnums/test_client" - fileutil.make_dirs(basedir + b"/private") + basedir = FilePath("introducer/ClientSeqnums/test_client") + private = basedir.child("private") + private.makedirs() write_introducer(basedir, "default", "nope") # if storage is enabled, the Client will publish its storage server # during startup (although the announcement will wait in a queue # until the introducer connection is established). To avoid getting # confused by this, disable storage. - f = open(os.path.join(basedir, "tahoe.cfg"), "w") - f.write("[storage]\n") - f.write("enabled = false\n") - f.close() + with basedir.child("tahoe.cfg").open("w") as f: + f.write("[storage]\n") + f.write("enabled = false\n") - c = yield create_client(basedir) + c = yield create_client(basedir.path) ic = c.introducer_clients[0] outbound = ic._outbound_announcements published = ic._published_announcements def read_seqnum(): - f = open(os.path.join(basedir, "announcement-seqnum")) - seqnum = f.read().strip() - f.close() + seqnum = basedir.child("announcement-seqnum").getContent() return int(seqnum) ic.publish("sA", {"key": "value1"}, c._node_private_key) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 5a34dcc4d..7a7fe117b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -33,6 +33,9 @@ from allmydata.mutable.publish import MutableData from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure +from twisted.python.filepath import ( + FilePath, +) from .common import ( TEST_RSA_KEY_SIZE, @@ -903,21 +906,21 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): # usually this node is *not* parented to our self.sparent, so we can # shut it down separately from the rest, to exercise the # connection-lost code - basedir = self.getdir("client%d" % client_num) - if not os.path.isdir(basedir): - fileutil.make_dirs(basedir) + basedir = FilePath(self.getdir("client%d" % client_num)) + basedir.makedirs() config = "[client]\n" if helper_furl: config += "helper.furl = %s\n" % helper_furl - fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) - os.makedirs(basedir + b"/private") + basedir.child("tahoe.cfg").setContent(config) + private = basedir.child("private") + private.makedirs() write_introducer( basedir, "default", self.introducer_furl, ) - c = yield client.create_client(basedir) + c = yield client.create_client(basedir.path) self.clients.append(c) c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) self.numclients += 1