From fbcef2d1ae7f0893e4e4dc55066baa0b01feff4e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:32:40 -0400 Subject: [PATCH 01/45] Safely customize the Tor introducer's configuration Previously we clobbered the whole generated configuration and potentially wiped out additional important fields. Now we modify the configuration by just changing the fields we need to change. --- integration/conftest.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 33e7998c1..54632be26 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -43,7 +43,7 @@ from .util import ( generate_ssh_key, block_with_timeout, ) - +from allmydata.node import read_config # pytest customization hooks @@ -275,13 +275,6 @@ def introducer_furl(introducer, temp_dir): include_result=False, ) def tor_introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer_tor -web.port = 4561 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer_tor') print("making introducer", intro_dir) @@ -301,9 +294,11 @@ log_gatherer.furl = {log_furl} ) pytest_twisted.blockon(done_proto.done) - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) + # adjust a few settings + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer-tor") + config.set_config("node", "web.port", "4561") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 1c11f9e7d4957fcb1312418feb65c6a56847586e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:35:14 -0400 Subject: [PATCH 02/45] Add a little more debug info to the integration test suite output --- integration/conftest.py | 3 +++ integration/test_tor.py | 2 +- integration/util.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 54632be26..280d98f72 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -321,7 +321,9 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): pass request.addfinalizer(cleanup) + print("Waiting for introducer to be ready...") pytest_twisted.blockon(protocol.magic_seen) + print("Introducer ready.") return transport @@ -332,6 +334,7 @@ def tor_introducer_furl(tor_introducer, temp_dir): print("Don't see {} yet".format(furl_fname)) sleep(.1) furl = open(furl_fname, 'r').read() + print(f"Found Tor introducer furl: {furl}") return furl diff --git a/integration/test_tor.py b/integration/test_tor.py index 901858347..8398cf9a4 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -93,7 +93,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ web_port = "tcp:{}:interface=localhost".format(control_port + 2000) if True: - print("creating", node_dir.path) + print(f"creating {node_dir.path} with introducer {introducer_furl}") node_dir.makedirs() proto = util._DumpOutputProtocol(None) reactor.spawnProcess( diff --git a/integration/util.py b/integration/util.py index 05fef8fed..08c07a059 100644 --- a/integration/util.py +++ b/integration/util.py @@ -607,7 +607,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve continue if len(js['servers']) < minimum_number_of_servers: - print("waiting because insufficient servers") + print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})") time.sleep(1) continue server_times = [ From 1c99817e1b0d098a422a2d0ccdc4cb6a5cb3d489 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 13:41:51 -0400 Subject: [PATCH 03/45] Safely customize the client node's configuration This is similar to the fix to the `tor_introducer` fixture. --- integration/test_tor.py | 39 +++++++++++---------------------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index 8398cf9a4..b116fe319 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -25,6 +25,7 @@ from twisted.python.filepath import ( from allmydata.test.common import ( write_introducer, ) +from allmydata.client import read_config # see "conftest.py" for the fixtures (e.g. "tor_network") @@ -103,10 +104,14 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ sys.executable, '-b', '-m', 'allmydata.scripts.runner', 'create-node', '--nickname', name, + '--webport', web_port, '--introducer', introducer_furl, '--hide-ip', '--tor-control-port', 'tcp:localhost:{}'.format(control_port), '--listen', 'tor', + '--shares-needed', '1', + '--shares-happy', '1', + '--shares-total', '2', node_dir.path, ) ) @@ -115,35 +120,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ # Which services should this client connect to? write_introducer(node_dir, "default", introducer_furl) - with node_dir.child('tahoe.cfg').open('w') as f: - node_config = ''' -[node] -nickname = %(name)s -web.port = %(web_port)s -web.static = public_html -log_gatherer.furl = %(log_furl)s -[tor] -control.port = tcp:localhost:%(control_port)d -onion.external_port = 3457 -onion.local_port = %(local_port)d -onion = true -onion.private_key_file = private/tor_onion.privkey - -[client] -shares.needed = 1 -shares.happy = 1 -shares.total = 2 - -''' % { - 'name': name, - 'web_port': web_port, - 'log_furl': flog_gatherer, - 'control_port': control_port, - 'local_port': control_port + 1000, -} - node_config = node_config.encode("utf-8") - f.write(node_config) + config = read_config(node_dir.path, "tub.port") + config.set_config("node", "log_gatherer.furl", flog_gatherer) + config.set_config("tor", "onion", "true") + config.set_config("tor", "onion.external_port", "3457") + config.set_config("tor", "onion.local_port", str(control_port + 1000)) + config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") print("running") result = yield util._run_node(reactor, node_dir.path, request, None) From 50c4ad81136a21096a6ce540938c70afc299fadd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 27 Mar 2023 14:07:53 -0400 Subject: [PATCH 04/45] news fragment --- newsfragments/3999.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3999.minor diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor new file mode 100644 index 000000000..e69de29bb From 1de8e811b5d963695af4c02886e85ee1ede36619 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Apr 2023 10:58:22 -0400 Subject: [PATCH 05/45] Tweaks. --- integration/test_tor.py | 6 ++++-- integration/util.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index b116fe319..d418f786b 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -43,8 +43,8 @@ if PY2: def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield util.await_client_ready(carol, minimum_number_of_servers=2) - yield util.await_client_ready(dave, minimum_number_of_servers=2) + yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60) + yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. @@ -125,6 +125,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ config.set_config("node", "log_gatherer.furl", flog_gatherer) config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") + config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") + #config.set_config("tor", "launch", "True") config.set_config("tor", "onion.local_port", str(control_port + 1000)) config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") diff --git a/integration/util.py b/integration/util.py index 08c07a059..a11c02225 100644 --- a/integration/util.py +++ b/integration/util.py @@ -90,6 +90,7 @@ class _CollectOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): + print("OUT: {!r}".format(data)) self.output.write(data) def errReceived(self, data): From efa51d41dcdbf430510927b4ad6d41a9835b267a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Apr 2023 10:58:28 -0400 Subject: [PATCH 06/45] Newer chutney. --- integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 280d98f72..36e7eef0b 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -465,7 +465,7 @@ def chutney(reactor, temp_dir): 'git', ( 'git', 'clone', - 'https://git.torproject.org/chutney.git', + 'https://gitlab.torproject.org/tpo/core/chutney.git', chutney_dir, ), env=environ, @@ -481,7 +481,7 @@ def chutney(reactor, temp_dir): ( 'git', '-C', chutney_dir, 'reset', '--hard', - 'c825cba0bcd813c644c6ac069deeb7347d3200ee' + 'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2' ), env=environ, ) From 812458699dc22a62f49419a2fd62bcf7510b08b2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Apr 2023 11:38:28 -0400 Subject: [PATCH 07/45] The tcp listening port needs to match the onion local port, or you get connection refused when you try to connect to the hidden service. --- integration/test_tor.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/test_tor.py b/integration/test_tor.py index d418f786b..6f6f54c25 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -126,8 +126,6 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_ config.set_config("tor", "onion", "true") config.set_config("tor", "onion.external_port", "3457") config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1") - #config.set_config("tor", "launch", "True") - config.set_config("tor", "onion.local_port", str(control_port + 1000)) config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey") print("running") From 13e9f88309c15f5f4bfe71c0abe8bff5a2c2326b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Apr 2023 15:23:20 -0400 Subject: [PATCH 08/45] Add necessary config option to ensure it listens on Tor, and also give correct Tor control port. --- integration/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 621c0224c..f3cf9a9d8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -294,7 +294,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', - '--tor-control-port', 'tcp:localhost:8010', + '--tor-control-port', 'tcp:localhost:8007', + '--hide-ip', '--listen=tor', intro_dir, ), From 7b9432482724297c6d637aee20c2a6f5d94339ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Apr 2023 15:23:51 -0400 Subject: [PATCH 09/45] More debugging. --- integration/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/util.py b/integration/util.py index a11c02225..ac3fe2833 100644 --- a/integration/util.py +++ b/integration/util.py @@ -140,6 +140,7 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): + print("OUT", data) data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) @@ -148,6 +149,7 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): + print("ERR", data) data = str(data, sys.stderr.encoding) sys.stdout.write(data) From 507d1f8394cedc672715fee7bec1b5ef75cb6037 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 12 Apr 2023 22:34:45 -0600 Subject: [PATCH 10/45] Fix some Chutney things (and a couple cleanups): wait for bootstrap, increase timeout --- integration/conftest.py | 44 ++++++++++++++++++++++------------------- integration/test_tor.py | 2 +- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index f3cf9a9d8..7a4234de7 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -206,13 +206,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): include_result=False, ) def introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer0 -web.port = 4560 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer') print("making introducer", intro_dir) @@ -232,6 +225,10 @@ log_gatherer.furl = {log_furl} ) pytest_twisted.blockon(done_proto.done) + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer-tor") + config.set_config("node", "web.port", "4561") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # over-write the config file with our stuff with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: f.write(config) @@ -283,7 +280,8 @@ def introducer_furl(introducer, temp_dir): ) def tor_introducer(reactor, temp_dir, flog_gatherer, request): intro_dir = join(temp_dir, 'introducer_tor') - print("making introducer", intro_dir) + print("making Tor introducer in {}".format(intro_dir)) + print("(this can take tens of seconds to allocate Onion address)") if not exists(intro_dir): mkdir(intro_dir) @@ -342,7 +340,7 @@ def tor_introducer_furl(tor_introducer, temp_dir): print("Don't see {} yet".format(furl_fname)) sleep(.1) furl = open(furl_fname, 'r').read() - print(f"Found Tor introducer furl: {furl}") + print(f"Found Tor introducer furl: {furl} in {furl_fname}") return furl @@ -510,7 +508,13 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: ) pytest_twisted.blockon(proto.done) - return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) + return ( + chutney_dir, + { + "PYTHONPATH": join(chutney_dir, "lib"), + "CHUTNEY_START_TIME": "200", # default is 60 + } + ) @pytest.fixture(scope='session') @@ -544,17 +548,9 @@ def tor_network(reactor, temp_dir, chutney, request): return proto.done # now, as per Chutney's README, we have to create the network - # ./chutney configure networks/basic - # ./chutney start networks/basic pytest_twisted.blockon(chutney(("configure", basic_network))) - pytest_twisted.blockon(chutney(("start", basic_network))) - - # print some useful stuff - try: - pytest_twisted.blockon(chutney(("status", basic_network))) - except ProcessTerminated: - print("Chutney.TorNet status failed (continuing)") + # ensure we will tear down the network right before we start it def cleanup(): print("Tearing down Chutney Tor network") try: @@ -563,5 +559,13 @@ def tor_network(reactor, temp_dir, chutney, request): # If this doesn't exit cleanly, that's fine, that shouldn't fail # the test suite. pass - request.addfinalizer(cleanup) + + pytest_twisted.blockon(chutney(("start", basic_network))) + pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network))) + + # print some useful stuff + try: + pytest_twisted.blockon(chutney(("status", basic_network))) + except ProcessTerminated: + print("Chutney.TorNet status failed (continuing)") diff --git a/integration/test_tor.py b/integration/test_tor.py index fb9d8c086..c3041f6d3 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -61,7 +61,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne ) yield proto.done cap = proto.output.getvalue().strip().split()[-1] - print("TEH CAP!", cap) + print("capability: {}".format(cap)) proto = util._CollectOutputProtocol(capture_stderr=False) reactor.spawnProcess( From 9472841c39120b408bfb7efab3b90b3fcb048a53 Mon Sep 17 00:00:00 2001 From: meejah Date: Wed, 12 Apr 2023 23:01:28 -0600 Subject: [PATCH 11/45] enable tor, i2p services --- src/allmydata/introducer/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 98136157d..e0ff138cc 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -83,6 +83,8 @@ def create_introducer(basedir=u"."): i2p_provider, tor_provider, ) + i2p_provider.setServiceParent(node) + tor_provider.setServiceParent(node) return defer.succeed(node) except Exception: return Failure() From 175473df407157db53276e0721fec324242e4bd2 Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 13 Apr 2023 00:37:32 -0600 Subject: [PATCH 12/45] longer timeouts, forget less --- integration/conftest.py | 2 +- integration/test_tor.py | 4 ++-- src/allmydata/introducer/server.py | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 7a4234de7..e7e021016 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -512,7 +512,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: chutney_dir, { "PYTHONPATH": join(chutney_dir, "lib"), - "CHUTNEY_START_TIME": "200", # default is 60 + "CHUTNEY_START_TIME": "600", # default is 60 } ) diff --git a/integration/test_tor.py b/integration/test_tor.py index c3041f6d3..10e326e46 100644 --- a/integration/test_tor.py +++ b/integration/test_tor.py @@ -33,8 +33,8 @@ if sys.platform.startswith('win'): def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl): carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl) - yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60) - yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60) + yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600) + yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index e0ff138cc..5dad89ae8 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -68,10 +68,6 @@ def create_introducer(basedir=u"."): default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider) tub_options = create_tub_options(config) - # we don't remember these because the Introducer doesn't make - # outbound connections. - i2p_provider = None - tor_provider = None main_tub = create_main_tub( config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, From 250efe7d24cd43a676a6a7116da35f6ab226401a Mon Sep 17 00:00:00 2001 From: meejah Date: Thu, 13 Apr 2023 16:42:02 -0600 Subject: [PATCH 13/45] leftover --- integration/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index e7e021016..b54e18e26 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -229,9 +229,6 @@ def introducer(reactor, temp_dir, flog_gatherer, request): config.set_config("node", "nickname", "introducer-tor") config.set_config("node", "web.port", "4561") config.set_config("node", "log_gatherer.furl", flog_gatherer) - # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 76ce54ea53e4e802da612f1f2cbc53c88e9764da Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 13:23:28 -0600 Subject: [PATCH 14/45] remove debugging --- integration/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index ac3fe2833..887602906 100644 --- a/integration/util.py +++ b/integration/util.py @@ -90,11 +90,9 @@ class _CollectOutputProtocol(ProcessProtocol): self.done.errback(reason) def outReceived(self, data): - print("OUT: {!r}".format(data)) self.output.write(data) def errReceived(self, data): - print("ERR: {!r}".format(data)) if self.capture_stderr: self.output.write(data) From abfca04af5a8c7e43fa18351b1131ebcf741efd8 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 13:24:22 -0600 Subject: [PATCH 15/45] turn off i2p tests for now --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 96619a93a..597623d9c 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -133,7 +133,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): +def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading From d3c39f8604fd924bbac424ce201533b4656416b3 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:27:19 -0600 Subject: [PATCH 16/45] fix i2p introducer, different ports --- integration/conftest.py | 2 +- integration/test_i2p.py | 13 ++++--------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index b54e18e26..f65c84141 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -227,7 +227,7 @@ def introducer(reactor, temp_dir, flog_gatherer, request): config = read_config(intro_dir, "tub.port") config.set_config("node", "nickname", "introducer-tor") - config.set_config("node", "web.port", "4561") + config.set_config("node", "web.port", "4562") config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 597623d9c..df619c6eb 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -68,13 +68,6 @@ def i2p_network(reactor, temp_dir, request): include_result=False, ) def i2p_introducer(reactor, temp_dir, flog_gatherer, request): - config = ''' -[node] -nickname = introducer_i2p -web.port = 4561 -log_gatherer.furl = {log_furl} -'''.format(log_furl=flog_gatherer) - intro_dir = join(temp_dir, 'introducer_i2p') print("making introducer", intro_dir) @@ -94,8 +87,10 @@ log_gatherer.furl = {log_furl} pytest_twisted.blockon(done_proto.done) # over-write the config file with our stuff - with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: - f.write(config) + config = read_config(intro_dir, "tub.port") + config.set_config("node", "nickname", "introducer_i2p") + config.set_config("node", "web.port", "4563") + config.set_config("node", "log_gatherer.furl", flog_gatherer) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 34cee7ff73c05d978354894f734d710d3a5c1a2c Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:44:52 -0600 Subject: [PATCH 17/45] missing import --- integration/test_i2p.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index df619c6eb..10abb7e30 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -23,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready from allmydata.test.common import ( write_introducer, ) +from allmydata.node import read_config + if which("docker") is None: pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) From 3ccb7c4d1c1f26991a3a467fe4a559738eac0638 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 15:45:17 -0600 Subject: [PATCH 18/45] re-enable i2p tests --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 10abb7e30..2aa1a536f 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading From 8b81bd7ebef79617d75ab1d5d5745d50a3e212b9 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 16:33:52 -0600 Subject: [PATCH 19/45] remove more debug --- integration/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/integration/util.py b/integration/util.py index 887602906..177983e2e 100644 --- a/integration/util.py +++ b/integration/util.py @@ -138,7 +138,6 @@ class _MagicTextProtocol(ProcessProtocol): self.exited.callback(None) def outReceived(self, data): - print("OUT", data) data = str(data, sys.stdout.encoding) sys.stdout.write(data) self._output.write(data) @@ -147,7 +146,6 @@ class _MagicTextProtocol(ProcessProtocol): self.magic_seen.callback(self) def errReceived(self, data): - print("ERR", data) data = str(data, sys.stderr.encoding) sys.stdout.write(data) From 8652bb71ad0ece21f9f75a1c290ccdb4abed6e92 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 17:05:57 -0600 Subject: [PATCH 20/45] skip i2p tests again? --- integration/test_i2p.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 2aa1a536f..42b848130 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks +@pytest.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) From b5f6fa8933c03d5de2069de47d67230ae3d641f2 Mon Sep 17 00:00:00 2001 From: meejah Date: Fri, 14 Apr 2023 19:07:27 -0600 Subject: [PATCH 21/45] skip properly --- integration/test_i2p.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 42b848130..a94648593 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks -@pytest.skip("I2P tests are not functioning at all, for unknown reasons") +@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons") def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) From ce93a7b869ceb29b49d985125e80a80fa19dad98 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 11:29:43 -0400 Subject: [PATCH 22/45] News fragment. --- newsfragments/4015.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4015.minor diff --git a/newsfragments/4015.minor b/newsfragments/4015.minor new file mode 100644 index 000000000..e69de29bb From 5da5a82a8cc36399c7dbc228109afee2a17ff21e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 12:02:04 -0400 Subject: [PATCH 23/45] Get rid of default mutable arguments. --- .ruff.toml | 2 ++ src/allmydata/client.py | 6 +++--- src/allmydata/dirnode.py | 4 +++- src/allmydata/hashtree.py | 7 +++++-- src/allmydata/immutable/upload.py | 4 +++- src/allmydata/interfaces.py | 6 +++--- src/allmydata/node.py | 12 ++++++++---- src/allmydata/nodemaker.py | 5 +++-- src/allmydata/test/cli/wormholetesting.py | 3 ++- src/allmydata/test/no_network.py | 4 +++- src/allmydata/test/test_system.py | 21 +++++++++------------ src/allmydata/test/web/test_web.py | 12 +++++++++--- src/allmydata/util/dbutil.py | 4 +++- 13 files changed, 56 insertions(+), 34 deletions(-) diff --git a/.ruff.toml b/.ruff.toml index 75ff62c2d..516255d2a 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,4 +9,6 @@ select = [ # Make sure we bind closure variables in a loop (equivalent to pylint # cell-var-from-loop): "B023", + # Don't use mutable default arguments: + "B006", ] \ No newline at end of file diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 8a10fe9e7..1d959cb98 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,7 +7,7 @@ import os import stat import time import weakref -from typing import Optional +from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial # On Python 2 this will be the backported package: @@ -189,7 +189,7 @@ class Terminator(service.Service): return service.Service.stopService(self) -def read_config(basedir, portnumfile, generated_files=[]): +def read_config(basedir, portnumfile, generated_files: Iterable=()): """ Read and validate configuration for a client-style Node. See :method:`allmydata.node.read_config` for parameter meanings (the @@ -1103,7 +1103,7 @@ class _Client(node.Node, pollmixin.PollMixin): # may get an opaque node if there were any problems. return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) - def create_dirnode(self, initial_children={}, version=None): + def create_dirnode(self, initial_children=None, version=None): d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) return d diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index fdf373b45..ccd045b05 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -678,8 +678,10 @@ class DirectoryNode(object): return d # XXX: Too many arguments? Worthwhile to break into mutable/immutable? - def create_subdirectory(self, namex, initial_children={}, overwrite=True, + def create_subdirectory(self, namex, initial_children=None, overwrite=True, mutable=True, mutable_version=None, metadata=None): + if initial_children is None: + initial_children = {} name = normalize(namex) if self.is_readonly(): return defer.fail(NotWriteableError()) diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 17467459b..57bdbd9a1 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): name += " (leaf [%d] of %d)" % (leafnum, numleaves) return name - def set_hashes(self, hashes={}, leaves={}): + def set_hashes(self, hashes=None, leaves=None): """Add a bunch of hashes to the tree. I will validate these to the best of my ability. If I already have a @@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list): corrupted or one of the received hashes was corrupted. If it raises NotEnoughHashesError, then the otherhashes dictionary was incomplete. """ - + if hashes is None: + hashes = {} + if leaves is None: + leaves = {} assert isinstance(hashes, dict) for h in hashes.values(): assert isinstance(h, bytes) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 0421de4e0..a331cc5db 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1391,7 +1391,9 @@ class CHKUploader(object): def get_upload_status(self): return self._upload_status -def read_this_many_bytes(uploadable, size, prepend_data=[]): +def read_this_many_bytes(uploadable, size, prepend_data=None): + if prepend_data is None: + prepend_data = [] if size == 0: return defer.succeed([]) d = uploadable.read(size) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 467d0d450..201ab082e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1447,7 +1447,7 @@ class IDirectoryNode(IFilesystemNode): is a file, or if must_be_file is True and the child is a directory, I raise ChildOfWrongTypeError.""" - def create_subdirectory(name, initial_children={}, overwrite=True, + def create_subdirectory(name, initial_children=None, overwrite=True, mutable=True, mutable_version=None, metadata=None): """I create and attach a directory at the given name. The new directory can be empty, or it can be populated with children @@ -2586,7 +2586,7 @@ class IClient(Interface): @return: a Deferred that fires with an IMutableFileNode instance. """ - def create_dirnode(initial_children={}): + def create_dirnode(initial_children=None): """Create a new unattached dirnode, possibly with initial children. @param initial_children: dict with keys that are unicode child names, @@ -2641,7 +2641,7 @@ class INodeMaker(Interface): for use by unit tests, to create mutable files that are smaller than usual.""" - def create_new_mutable_directory(initial_children={}): + def create_new_mutable_directory(initial_children=None): """I create a new mutable directory, and return a Deferred that will fire with the IDirectoryNode instance when it is ready. If initial_children= is provided (a dict mapping unicode child name to diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 58ee33ef5..6c3082b50 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -17,7 +17,7 @@ import errno from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn -from typing import Union +from typing import Union, Iterable import attr @@ -172,7 +172,7 @@ def create_node_dir(basedir, readme_text): f.write(readme_text) -def read_config(basedir, portnumfile, generated_files=[], _valid_config=None): +def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None): """ Read and validate configuration. @@ -741,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider): def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers, - handler_overrides={}, force_foolscap=False, **kwargs): + handler_overrides=None, force_foolscap=False, **kwargs): """ Create a Tub with the right options and handlers. It will be ephemeral unless the caller provides certFile= in kwargs @@ -755,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS storage protocol. """ + if handler_overrides is None: + handler_overrides = {} # We listen simultaneously for both Foolscap and HTTPS on the same port, # so we have to create a special Foolscap Tub for that to work: if force_foolscap: @@ -922,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location): def create_main_tub(config, tub_options, default_connection_handlers, foolscap_connection_handlers, i2p_provider, tor_provider, - handler_overrides={}, cert_filename="node.pem"): + handler_overrides=None, cert_filename="node.pem"): """ Creates a 'main' Foolscap Tub, typically for use as the top-level access point for a running Node. @@ -943,6 +945,8 @@ def create_main_tub(config, tub_options, :param tor_provider: None, or a _Provider instance if txtorcon + Tor are installed. """ + if handler_overrides is None: + handler_overrides = {} portlocation = _tub_portlocation( config, iputil.get_local_addresses_sync, diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 1b7ea5f45..39663bda9 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -135,8 +135,9 @@ class NodeMaker(object): d.addCallback(lambda res: n) return d - def create_new_mutable_directory(self, initial_children={}, version=None): - # initial_children must have metadata (i.e. {} instead of None) + def create_new_mutable_directory(self, initial_children=None, version=None): + if initial_children is None: + initial_children = {} for (name, (node, metadata)) in initial_children.items(): precondition(isinstance(metadata, dict), "create_new_mutable_directory requires metadata to be a dict, not None", metadata) diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py index d1a3bfd07..647798bc8 100644 --- a/src/allmydata/test/cli/wormholetesting.py +++ b/src/allmydata/test/cli/wormholetesting.py @@ -70,7 +70,8 @@ class MemoryWormholeServer(object): appid: str, relay_url: str, reactor: Any, - versions: Any={}, + # Unfortunately we need a mutable default to match the real API + versions: Any={}, # noqa: B006 delegate: Optional[Any]=None, journal: Optional[Any]=None, tor: Optional[Any]=None, diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ee1f48b17..e3b57fb95 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -476,7 +476,7 @@ class GridTestMixin(object): ]) def set_up_grid(self, num_clients=1, num_servers=10, - client_config_hooks={}, oneshare=False): + client_config_hooks=None, oneshare=False): """ Create a Tahoe-LAFS storage grid. @@ -489,6 +489,8 @@ class GridTestMixin(object): :return: ``None`` """ + if client_config_hooks is None: + client_config_hooks = {} # self.basedir must be set port_assigner = SameProcessStreamEndpointAssigner() port_assigner.setUp() diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d11a6e866..b3287bf3b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1,20 +1,13 @@ """ Ported to Python 3. """ -from __future__ import print_function -from __future__ import absolute_import -from __future__ import division -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - # Don't import bytes since it causes issues on (so far unported) modules on Python 2. - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 +from __future__ import annotations from past.builtins import chr as byteschr, long from six import ensure_text import os, re, sys, time, json +from typing import Optional from bs4 import BeautifulSoup @@ -56,10 +49,12 @@ from .common_util import run_cli_unicode class RunBinTahoeMixin(object): - def run_bintahoe(self, args, stdin=None, python_options=[], env=None): + def run_bintahoe(self, args, stdin=None, python_options:Optional[list[str]]=None, env=None): # test_runner.run_bintahoe has better unicode support but doesn't # support env yet and is also synchronous. If we could get rid of # this in favor of that, though, it would probably be an improvement. + if python_options is None: + python_options = [] command = sys.executable argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args @@ -1088,7 +1083,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii") return self.POST2(urlpath, body, headers, use_helper) - def POST2(self, urlpath, body=b"", headers={}, use_helper=False): + def POST2(self, urlpath, body=b"", headers=None, use_helper=False): + if headers is None: + headers = {} if use_helper: url = self.helper_webish_url + urlpath else: @@ -1409,7 +1406,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs) defer.returnValue((out,err)) - def _check_ls(out_and_err, expected_children, unexpected_children=[]): + def _check_ls(out_and_err, expected_children, unexpected_children=()): (out, err) = out_and_err self.failUnlessEqual(err, "") for s in expected_children: diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 4c828817a..08dce0ac0 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -565,7 +565,9 @@ class WebMixin(TimezoneMixin): returnValue(data) @inlineCallbacks - def HEAD(self, urlpath, return_response=False, headers={}): + def HEAD(self, urlpath, return_response=False, headers=None): + if headers is None: + headers = {} url = self.webish_url + urlpath response = yield treq.request("head", url, persistent=False, headers=headers) @@ -573,7 +575,9 @@ class WebMixin(TimezoneMixin): raise Error(response.code, response="") returnValue( ("", response.code, response.headers) ) - def PUT(self, urlpath, data, headers={}): + def PUT(self, urlpath, data, headers=None): + if headers is None: + headers = {} url = self.webish_url + urlpath return do_http("put", url, data=data, headers=headers) @@ -618,7 +622,9 @@ class WebMixin(TimezoneMixin): body, headers = self.build_form(**fields) return self.POST2(urlpath, body, headers) - def POST2(self, urlpath, body="", headers={}, followRedirect=False): + def POST2(self, urlpath, body="", headers=None, followRedirect=False): + if headers is None: + headers = {} url = self.webish_url + urlpath if isinstance(body, str): body = body.encode("utf-8") diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 916382972..45e59cf00 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -25,7 +25,7 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, - create_version=(None, None), updaters={}, just_create=False, dbname="db", + create_version=(None, None), updaters=None, just_create=False, dbname="db", ): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. @@ -33,6 +33,8 @@ def get_db(dbfile, stderr=sys.stderr, to get from ver=1 to ver=2. Returns a (sqlite3,db) tuple, or raises DBError. """ + if updaters is None: + updaters = {} must_create = not os.path.exists(dbfile) try: db = sqlite3.connect(dbfile) From 2a4dcb7a27cd0fd0592a9002e13c644a07ea8420 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 17 Apr 2023 13:08:26 -0400 Subject: [PATCH 24/45] More checks that are probably useful (doesn't trigger anything at the moment). --- .ruff.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.ruff.toml b/.ruff.toml index 516255d2a..2dd6b59b5 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,6 +9,10 @@ select = [ # Make sure we bind closure variables in a loop (equivalent to pylint # cell-var-from-loop): "B023", + # Don't silence exceptions in finally by accident: + "B012", # Don't use mutable default arguments: "B006", + # Errors from PyLint: + "PLE", ] \ No newline at end of file From 3d2e4d0798b874c493dc6b86487b527e36aa3324 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:26:58 -0400 Subject: [PATCH 25/45] note about port selection --- integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index f65c84141..f670c6486 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -289,6 +289,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request): request, ( 'create-introducer', + # The control port should agree with the configuration of the + # Tor network we bootstrap with chutney. '--tor-control-port', 'tcp:localhost:8007', '--hide-ip', '--listen=tor', From c595eea33e78eac579b3f3b63163a0354348a0d6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:27:51 -0400 Subject: [PATCH 26/45] always set the "start time" timeout in both the "we installed it ourselves" and the "we found an existing installation" cases. --- integration/conftest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index f670c6486..eaf740190 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -511,7 +511,6 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: chutney_dir, { "PYTHONPATH": join(chutney_dir, "lib"), - "CHUTNEY_START_TIME": "600", # default is 60 } ) @@ -534,6 +533,10 @@ def tor_network(reactor, temp_dir, chutney, request): env = environ.copy() env.update(chutney_env) + env.update({ + # default is 60, probably too short for reliable automated use. + "CHUTNEY_START_TIME": "600", + }) chutney_argv = (sys.executable, '-m', 'chutney.TorNet') def chutney(argv): proto = _DumpOutputProtocol(None) From ba387453cf95ff04a322d8ba4531d81a5f31d7b2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:30:53 -0400 Subject: [PATCH 27/45] it's a bug fix! it's user-facing! --- newsfragments/3999.bugfix | 1 + newsfragments/3999.minor | 0 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3999.bugfix delete mode 100644 newsfragments/3999.minor diff --git a/newsfragments/3999.bugfix b/newsfragments/3999.bugfix new file mode 100644 index 000000000..a8a8396f4 --- /dev/null +++ b/newsfragments/3999.bugfix @@ -0,0 +1 @@ +A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed. \ No newline at end of file diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor deleted file mode 100644 index e69de29bb..000000000 From 825bcf3f3b8ca5a924d8c3075db653f9e5bf3c99 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:31:04 -0400 Subject: [PATCH 28/45] revert reformatting --- integration/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index eaf740190..b0d8da90f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -507,12 +507,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]: ) pytest_twisted.blockon(proto.done) - return ( - chutney_dir, - { - "PYTHONPATH": join(chutney_dir, "lib"), - } - ) + return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")}) @pytest.fixture(scope='session') From fbb5f4c359800e606cc3d29d39d80896292e4c40 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 25 Apr 2023 09:31:10 -0400 Subject: [PATCH 29/45] slightly clarified comment --- integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index b0d8da90f..cb590ef6f 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -547,7 +547,7 @@ def tor_network(reactor, temp_dir, chutney, request): # now, as per Chutney's README, we have to create the network pytest_twisted.blockon(chutney(("configure", basic_network))) - # ensure we will tear down the network right before we start it + # before we start the network, ensure we will tear down at the end def cleanup(): print("Tearing down Chutney Tor network") try: From c0e49064ce64eb2860dba6c2957a86485c1a1e41 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 27 Apr 2023 09:50:02 -0400 Subject: [PATCH 30/45] Attempt to get more information about client unready state --- integration/util.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/integration/util.py b/integration/util.py index 177983e2e..39e5dfa6d 100644 --- a/integration/util.py +++ b/integration/util.py @@ -604,19 +604,27 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve print("waiting because '{}'".format(e)) time.sleep(1) continue + servers = js['servers'] - if len(js['servers']) < minimum_number_of_servers: - print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})") + if len(servers) < minimum_number_of_servers: + print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})") time.sleep(1) continue + + print( + f"Now: {time.ctime()}\n" + f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}" + ) + server_times = [ server['last_received_data'] - for server in js['servers'] + for server in servers ] # if any times are null/None that server has never been # contacted (so it's down still, probably) - if any(t is None for t in server_times): - print("waiting because at least one server not contacted") + never_received_data = server_times.count(None) + if never_received_data > 0: + print(f"waiting because {never_received_data} server(s) not contacted") time.sleep(1) continue From 8f1d1cc1a0db48a1aa219b9a8814ef4201206098 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:23:06 -0400 Subject: [PATCH 31/45] Include node name in the logging output from subprocesses. --- integration/conftest.py | 6 +++--- integration/test_i2p.py | 4 ++-- integration/util.py | 11 ++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index d76b2a9c7..69d1934b0 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -161,7 +161,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): ) pytest_twisted.blockon(out_protocol.done) - twistd_protocol = _MagicTextProtocol("Gatherer waiting at") + twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer") twistd_process = reactor.spawnProcess( twistd_protocol, which('twistd')[0], @@ -244,7 +244,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = _MagicTextProtocol('introducer running') + protocol = _MagicTextProtocol('introducer running', "introducer") transport = _tahoe_runner_optional_coverage( protocol, reactor, @@ -320,7 +320,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = _MagicTextProtocol('introducer running') + protocol = _MagicTextProtocol('introducer running', "tor_introducer") transport = _tahoe_runner_optional_coverage( protocol, reactor, diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 96619a93a..4d4dbe620 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -35,7 +35,7 @@ if sys.platform.startswith('win'): @pytest.fixture def i2p_network(reactor, temp_dir, request): """Fixture to start up local i2pd.""" - proto = util._MagicTextProtocol("ephemeral keys") + proto = util._MagicTextProtocol("ephemeral keys", "i2pd") reactor.spawnProcess( proto, which("docker"), @@ -99,7 +99,7 @@ log_gatherer.furl = {log_furl} # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. - protocol = util._MagicTextProtocol('introducer running') + protocol = util._MagicTextProtocol('introducer running', "introducer") transport = util._tahoe_runner_optional_coverage( protocol, reactor, diff --git a/integration/util.py b/integration/util.py index 05fef8fed..b1692a7a3 100644 --- a/integration/util.py +++ b/integration/util.py @@ -12,7 +12,7 @@ import sys import time import json from os import mkdir, environ -from os.path import exists, join +from os.path import exists, join, basename from io import StringIO, BytesIO from subprocess import check_output @@ -129,8 +129,9 @@ class _MagicTextProtocol(ProcessProtocol): and then .callback()s on self.done and .errback's if the process exits """ - def __init__(self, magic_text): + def __init__(self, magic_text: str, name: str) -> None: self.magic_seen = Deferred() + self.name = f"{name}: " self.exited = Deferred() self._magic_text = magic_text self._output = StringIO() @@ -140,7 +141,7 @@ class _MagicTextProtocol(ProcessProtocol): def outReceived(self, data): data = str(data, sys.stdout.encoding) - sys.stdout.write(data) + sys.stdout.write(self.name + data) self._output.write(data) if not self.magic_seen.called and self._magic_text in self._output.getvalue(): print("Saw '{}' in the logs".format(self._magic_text)) @@ -148,7 +149,7 @@ class _MagicTextProtocol(ProcessProtocol): def errReceived(self, data): data = str(data, sys.stderr.encoding) - sys.stdout.write(data) + sys.stdout.write(self.name + data) def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None: @@ -282,7 +283,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True): """ if magic_text is None: magic_text = "client running" - protocol = _MagicTextProtocol(magic_text) + protocol = _MagicTextProtocol(magic_text, basename(node_dir)) # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old # "start" command. From 86a513282f67556e1791faa7ab09ac4fc3f5b6b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:36:39 -0400 Subject: [PATCH 32/45] Include Foolscap logging in node output in integration tests. --- integration/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index 69d1934b0..cdc65f9e8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -54,6 +54,10 @@ from .util import ( # integration tests. See allmydata/scripts/common_http.py for usage. os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120" +# Make Foolscap logging go into Twisted logging, so that integration test logs +# include extra information +# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst): +os.environ["FLOGTOTWISTED"] = "1" # pytest customization hooks From 9faf742b411e5dddd11e098eee7421b7154e3b25 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 10:36:59 -0400 Subject: [PATCH 33/45] News file. --- newsfragments/4018.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4018.minor diff --git a/newsfragments/4018.minor b/newsfragments/4018.minor new file mode 100644 index 000000000..e69de29bb From 0f200e422e3278091843f6384fa91cfe5bf1101c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 27 Apr 2023 15:48:49 -0400 Subject: [PATCH 34/45] Give it more time. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index b1692a7a3..c58fc2e93 100644 --- a/integration/util.py +++ b/integration/util.py @@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs): @run_in_thread -def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): +def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be From 44cd746ce480c3a1641ebfe55350d15698625323 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Apr 2023 11:43:26 -0400 Subject: [PATCH 35/45] Limit klein version for now. --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 2418c6dbe..751c7ebc3 100644 --- a/setup.py +++ b/setup.py @@ -141,8 +141,10 @@ install_requires = [ # HTTP server and client "klein", - # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 - "werkzeug != 2.2.0", + # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 and 2.3 is + # incompatible with klein 21.8 and earlier; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4020 for the latter. + "werkzeug != 2.2.0,<2.3", "treq", "cbor2", From c15dd6c9f0fb9d43c9a17db523d6f726f67ee593 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 28 Apr 2023 11:43:48 -0400 Subject: [PATCH 36/45] This wasn't the issue. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index c58fc2e93..b1692a7a3 100644 --- a/integration/util.py +++ b/integration/util.py @@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs): @run_in_thread -def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1): +def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1): """ Uses the status API to wait for a client-type node (in `tahoe`, a `TahoeProcess` instance usually from a fixture e.g. `alice`) to be From 5c2f18dfec743c104410ef514120a8b97ccc1364 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:03:14 -0400 Subject: [PATCH 37/45] Set a higher timeout. --- src/allmydata/test/test_system.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index d11a6e866..72e91f9b4 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1749,6 +1749,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d + # In CI this test can be very slow, so give it a longer timeout: + test_filesystem.timeout = 360 + + def test_filesystem_with_cli_in_subprocess(self): # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe. From 8fa89bd98585bd64b2510a6ec50ab44a5090bd4b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:05:40 -0400 Subject: [PATCH 38/45] Run a little faster. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 72e91f9b4..a6bed7f87 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem(self): self.data = LARGE_DATA - d = self.set_up_nodes() + d = self.set_up_nodes(4) def _new_happy_semantics(ign): for c in self.clients: c.encoding_params['happy'] = 1 From d4f2038fd1ae943944c9a90e3e154371bc76d57d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:23 -0400 Subject: [PATCH 39/45] Rearrange nodes so it's possible to create even fewer. --- src/allmydata/test/common_system.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 3491d413d..fa8d943e5 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -819,8 +819,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): helper_furl = f.read() self.helper_furl = helper_furl - if self.numclients >= 4: - with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f: + if self.numclients >= 2: + with open(os.path.join(basedirs[1], 'tahoe.cfg'), 'a+') as f: f.write( "[client]\n" "helper.furl = {}\n".format(helper_furl) @@ -836,9 +836,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): log.msg("CONNECTED") # now find out where the web port was self.webish_url = self.clients[0].getServiceNamed("webish").getURL() - if self.numclients >=4: + if self.numclients >=2: # and the helper-using webport - self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() + self.helper_webish_url = self.clients[1].getServiceNamed("webish").getURL() def _generate_config(self, which, basedir, force_foolscap=False): config = {} @@ -854,10 +854,10 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): ("node", "tub.location"): allclients, # client 0 runs a webserver and a helper - # client 3 runs a webserver but no helper - ("node", "web.port"): {0, 3}, + # client 1 runs a webserver but no helper + ("node", "web.port"): {0, 1}, ("node", "timeout.keepalive"): {0}, - ("node", "timeout.disconnect"): {3}, + ("node", "timeout.disconnect"): {1}, ("helper", "enabled"): {0}, } From 9f78fd5c7f2d3340a3c64f79483373292570b732 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:31 -0400 Subject: [PATCH 40/45] Use even fewer nodes. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index a6bed7f87..c2fe1339f 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def test_filesystem(self): self.data = LARGE_DATA - d = self.set_up_nodes(4) + d = self.set_up_nodes(2) def _new_happy_semantics(ign): for c in self.clients: c.encoding_params['happy'] = 1 From 1ca30e1d2fc00c61b56ad6603984dd4d705bd549 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:11:44 -0400 Subject: [PATCH 41/45] News entry. --- newsfragments/4022.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4022.minor diff --git a/newsfragments/4022.minor b/newsfragments/4022.minor new file mode 100644 index 000000000..e69de29bb From 22715abc854b68eaa38fd664d1ab207c88582894 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:17:55 -0400 Subject: [PATCH 42/45] This is fine. --- src/allmydata/test/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index c2fe1339f..58384e4d8 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -1750,7 +1750,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): return d # In CI this test can be very slow, so give it a longer timeout: - test_filesystem.timeout = 360 + test_filesystem.timeout = 360 # type: ignore[attr-defined] def test_filesystem_with_cli_in_subprocess(self): From 63b082759dabe44dfbed45effd10e2cae26d037b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 2 May 2023 12:59:00 -0400 Subject: [PATCH 43/45] Use a modern coverage.py. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d01efdf83..9837b3bab 100644 --- a/setup.py +++ b/setup.py @@ -418,7 +418,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "subunitreporter==22.2.0", "python-subunit==1.4.2", "junitxml==0.7", - "coverage ~= 5.0", + "coverage==7.2.5", ], # Here are the library dependencies of the test suite. From 5a5031f02046c77a66761b5988177081631a6afc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 16:46:57 -0400 Subject: [PATCH 44/45] Try with newer Python, 3.11 might make it faster. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bb7c9efb..fe911e34d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,12 +169,12 @@ jobs: python-version: "3.9" force-foolscap: false - os: windows-latest - python-version: "3.9" + python-version: "3.11" force-foolscap: false # 22.04 has some issue with Tor at the moment: # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943 - os: ubuntu-20.04 - python-version: "3.11" + python-version: "3.10" force-foolscap: false steps: From dca19525b9d9f2cd6c6f63fdaba6945bb5f4759a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 3 May 2023 16:58:47 -0400 Subject: [PATCH 45/45] =?UTF-8?q?=F0=9F=AA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- integration/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 7360b891b..bf04e4424 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -50,9 +50,9 @@ from .util import ( ) from allmydata.node import read_config -# No reason for HTTP requests to take longer than two minutes in the +# No reason for HTTP requests to take longer than four minutes in the # integration tests. See allmydata/scripts/common_http.py for usage. -os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120" +os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "240" # Make Foolscap logging go into Twisted logging, so that integration test logs # include extra information