Merge remote-tracking branch 'origin/master' into 4016-http-storage-content-type

.circleci/create-virtualenv.sh

@@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 # above, it may still not be able to get us a compatible version unless we
 # explicitly ask for one.
 "${PIP}" install --upgrade setuptools==44.0.0 wheel
+
+# Just about every user of this image wants to use tox from the bootstrap
+# virtualenv so go ahead and install it now.
+"${PIP}" install "tox~=3.0"

.circleci/populate-wheelhouse.sh

@@ -3,18 +3,6 @@
 # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
-# Basic Python packages that you just need to have around to do anything,
-# practically speaking.
-BASIC_DEPS="pip wheel"
-
-# Python packages we need to support the test infrastructure.  *Not* packages
-# Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0"
-
-# Python packages we need to generate test reports for CI infrastructure.
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
-
 # The filesystem location of the wheelhouse which we'll populate with wheels
 # for all of our dependencies.
 WHEELHOUSE_PATH="$1"
@@ -41,15 +29,5 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
-    "${PROJECT_ROOT}"[test] \
+    "${PROJECT_ROOT}"[testenv] \
-    ${BASIC_DEPS} \
+    "${PROJECT_ROOT}"[test]
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
-
-# Not strictly wheelhouse population but ... Note we omit basic deps here.
-# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
-# have to ask.
-"${PIP}" \
-    install \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}

.circleci/run-tests.sh

@@ -79,9 +79,10 @@ else
     alternative="false"
 fi
 
+WORKDIR=/tmp/tahoe-lafs.tox
 ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \
     -c ${PROJECT_ROOT}/tox.ini \
-    --workdir /tmp/tahoe-lafs.tox \
+    --workdir "${WORKDIR}" \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
     ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
 
@@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then
 
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+
+    "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

.circleci/setup-virtualenv.sh

@@ -26,12 +26,7 @@ shift || :
 
 # Tell pip where it can find any existing wheels.
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
-
+export PIP_NO_INDEX="1"
-# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
-# between the time dependencies change and the images are re-built and (b) the
-# upcoming-deprecations job wants to install some dependencies from github and
-# it's awkward to get that done any earlier than the tox run.  So, we don't
-# set it.
 
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \

.ruff.toml

@@ -9,4 +9,10 @@ select = [
     # Make sure we bind closure variables in a loop (equivalent to pylint
     # cell-var-from-loop):
     "B023",
+    # Don't silence exceptions in finally by accident:
+    "B012",
+    # Don't use mutable default arguments:
+    "B006",
+    # Errors from PyLint:
+    "PLE",
 ]

integration/conftest.py

@@ -48,12 +48,16 @@ from .util import (
     generate_ssh_key,
     block_with_timeout,
 )
-
+from allmydata.node import read_config
 
 # No reason for HTTP requests to take longer than two minutes in the
 # integration tests. See allmydata/scripts/common_http.py for usage.
 os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
 
+# Make Foolscap logging go into Twisted logging, so that integration test logs
+# include extra information
+# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst):
+os.environ["FLOGTOTWISTED"] = "1"
 
 # pytest customization hooks
 
@@ -161,7 +165,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     )
     pytest_twisted.blockon(out_protocol.done)
 
-    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
+    twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer")
     twistd_process = reactor.spawnProcess(
         twistd_protocol,
         which('twistd')[0],
@@ -212,13 +216,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     include_result=False,
 )
 def introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer0
-web.port = 4560
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer')
     print("making introducer", intro_dir)
 
@@ -238,13 +235,14 @@ log_gatherer.furl = {log_furl}
         )
         pytest_twisted.blockon(done_proto.done)
 
-    # over-write the config file with our stuff
+    config = read_config(intro_dir, "tub.port")
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
+    config.set_config("node", "nickname", "introducer-tor")
-        f.write(config)
+    config.set_config("node", "web.port", "4562")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
@@ -288,15 +286,9 @@ def introducer_furl(introducer, temp_dir):
     include_result=False,
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_tor
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_tor')
-    print("making introducer", intro_dir)
+    print("making Tor introducer in {}".format(intro_dir))
+    print("(this can take tens of seconds to allocate Onion address)")
 
     if not exists(intro_dir):
         mkdir(intro_dir)
@@ -307,20 +299,25 @@ log_gatherer.furl = {log_furl}
             request,
             (
                 'create-introducer',
-                '--tor-control-port', 'tcp:localhost:8010',
+                # The control port should agree with the configuration of the
+                # Tor network we bootstrap with chutney.
+                '--tor-control-port', 'tcp:localhost:8007',
+                '--hide-ip',
                 '--listen=tor',
                 intro_dir,
             ),
         )
         pytest_twisted.blockon(done_proto.done)
 
-    # over-write the config file with our stuff
+    # adjust a few settings
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
+    config = read_config(intro_dir, "tub.port")
-        f.write(config)
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "tor_introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
@@ -339,7 +336,9 @@ log_gatherer.furl = {log_furl}
             pass
     request.addfinalizer(cleanup)
 
+    print("Waiting for introducer to be ready...")
     pytest_twisted.blockon(protocol.magic_seen)
+    print("Introducer ready.")
     return transport
 
 
@@ -350,6 +349,7 @@ def tor_introducer_furl(tor_introducer, temp_dir):
         print("Don't see {} yet".format(furl_fname))
         sleep(.1)
     furl = open(furl_fname, 'r').read()
+    print(f"Found Tor introducer furl: {furl} in {furl_fname}")
     return furl
 
 
@@ -495,7 +495,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         'git',
         (
             'git', 'clone',
-            'https://git.torproject.org/chutney.git',
+            'https://gitlab.torproject.org/tpo/core/chutney.git',
             chutney_dir,
         ),
         env=environ,
@@ -511,7 +511,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         (
             'git', '-C', chutney_dir,
             'reset', '--hard',
-            'c825cba0bcd813c644c6ac069deeb7347d3200ee'
+            'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2'
         ),
         env=environ,
     )
@@ -538,6 +538,10 @@ def tor_network(reactor, temp_dir, chutney, request):
 
     env = environ.copy()
     env.update(chutney_env)
+    env.update({
+        # default is 60, probably too short for reliable automated use.
+        "CHUTNEY_START_TIME": "600",
+    })
     chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
     def chutney(argv):
         proto = _DumpOutputProtocol(None)
@@ -551,17 +555,9 @@ def tor_network(reactor, temp_dir, chutney, request):
         return proto.done
 
     # now, as per Chutney's README, we have to create the network
-    # ./chutney configure networks/basic
-    # ./chutney start networks/basic
     pytest_twisted.blockon(chutney(("configure", basic_network)))
-    pytest_twisted.blockon(chutney(("start", basic_network)))
-
-    # print some useful stuff
-    try:
-        pytest_twisted.blockon(chutney(("status", basic_network)))
-    except ProcessTerminated:
-        print("Chutney.TorNet status failed (continuing)")
 
+    # before we start the network, ensure we will tear down at the end
     def cleanup():
         print("Tearing down Chutney Tor network")
         try:
@@ -570,5 +566,13 @@ def tor_network(reactor, temp_dir, chutney, request):
             # If this doesn't exit cleanly, that's fine, that shouldn't fail
             # the test suite.
             pass
-
     request.addfinalizer(cleanup)
+
+    pytest_twisted.blockon(chutney(("start", basic_network)))
+    pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network)))
+
+    # print some useful stuff
+    try:
+        pytest_twisted.blockon(chutney(("status", basic_network)))
+    except ProcessTerminated:
+        print("Chutney.TorNet status failed (continuing)")

integration/test_i2p.py

@@ -23,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.node import read_config
+
 
 if which("docker") is None:
     pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)
@@ -35,7 +37,7 @@ if sys.platform.startswith('win'):
 @pytest.fixture
 def i2p_network(reactor, temp_dir, request):
     """Fixture to start up local i2pd."""
-    proto = util._MagicTextProtocol("ephemeral keys")
+    proto = util._MagicTextProtocol("ephemeral keys", "i2pd")
     reactor.spawnProcess(
         proto,
         which("docker"),
@@ -68,13 +70,6 @@ def i2p_network(reactor, temp_dir, request):
     include_result=False,
 )
 def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_i2p
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_i2p')
     print("making introducer", intro_dir)
 
@@ -94,12 +89,14 @@ log_gatherer.furl = {log_furl}
         pytest_twisted.blockon(done_proto.done)
 
     # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
+    config = read_config(intro_dir, "tub.port")
-        f.write(config)
+    config.set_config("node", "nickname", "introducer_i2p")
+    config.set_config("node", "web.port", "4563")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = util._MagicTextProtocol('introducer running')
+    protocol = util._MagicTextProtocol('introducer running', "introducer")
     transport = util._tahoe_runner_optional_coverage(
         protocol,
         reactor,
@@ -133,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
+@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)

integration/test_tor.py

@@ -18,6 +18,7 @@ from twisted.python.filepath import (
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.client import read_config
 
 # see "conftest.py" for the fixtures (e.g. "tor_network")
 
@@ -32,8 +33,8 @@ if sys.platform.startswith('win'):
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
-    yield util.await_client_ready(dave, minimum_number_of_servers=2)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
@@ -60,7 +61,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     )
     yield proto.done
     cap = proto.output.getvalue().strip().split()[-1]
-    print("TEH CAP!", cap)
+    print("capability: {}".format(cap))
 
     proto = util._CollectOutputProtocol(capture_stderr=False)
     reactor.spawnProcess(
@@ -85,7 +86,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
     if True:
-        print("creating", node_dir.path)
+        print(f"creating {node_dir.path} with introducer {introducer_furl}")
         node_dir.makedirs()
         proto = util._DumpOutputProtocol(None)
         reactor.spawnProcess(
@@ -95,10 +96,14 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                 'create-node',
                 '--nickname', name,
+                '--webport', web_port,
                 '--introducer', introducer_furl,
                 '--hide-ip',
                 '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                 '--listen', 'tor',
+                '--shares-needed', '1',
+                '--shares-happy', '1',
+                '--shares-total', '2',
                 node_dir.path,
             ),
             env=environ,
@@ -108,35 +113,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
-    with node_dir.child('tahoe.cfg').open('w') as f:
-        node_config = '''
-[node]
-nickname = %(name)s
-web.port = %(web_port)s
-web.static = public_html
-log_gatherer.furl = %(log_furl)s
 
-[tor]
+    config = read_config(node_dir.path, "tub.port")
-control.port = tcp:localhost:%(control_port)d
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
-onion.external_port = 3457
+    config.set_config("tor", "onion", "true")
-onion.local_port = %(local_port)d
+    config.set_config("tor", "onion.external_port", "3457")
-onion = true
+    config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
-onion.private_key_file = private/tor_onion.privkey
+    config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
-
-[client]
-shares.needed = 1
-shares.happy = 1
-shares.total = 2
-
-''' % {
-    'name': name,
-    'web_port': web_port,
-    'log_furl': flog_gatherer,
-    'control_port': control_port,
-    'local_port': control_port + 1000,
-}
-        node_config = node_config.encode("utf-8")
-        f.write(node_config)
 
     print("running")
     result = yield util._run_node(reactor, node_dir.path, request, None)

integration/util.py

@@ -12,7 +12,7 @@ import sys
 import time
 import json
 from os import mkdir, environ
-from os.path import exists, join
+from os.path import exists, join, basename
 from io import StringIO, BytesIO
 from subprocess import check_output
 
@@ -93,7 +93,6 @@ class _CollectOutputProtocol(ProcessProtocol):
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {!r}".format(data))
         if self.capture_stderr:
             self.output.write(data)
 
@@ -129,8 +128,9 @@ class _MagicTextProtocol(ProcessProtocol):
     and then .callback()s on self.done and .errback's if the process exits
     """
 
-    def __init__(self, magic_text):
+    def __init__(self, magic_text: str, name: str) -> None:
         self.magic_seen = Deferred()
+        self.name = f"{name}: "
         self.exited = Deferred()
         self._magic_text = magic_text
         self._output = StringIO()
@@ -140,7 +140,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def outReceived(self, data):
         data = str(data, sys.stdout.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
@@ -148,7 +148,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         data = str(data, sys.stderr.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
 
 
 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@@ -282,7 +282,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
     """
     if magic_text is None:
         magic_text = "client running"
-    protocol = _MagicTextProtocol(magic_text)
+    protocol = _MagicTextProtocol(magic_text, basename(node_dir))
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
@@ -605,19 +605,27 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             print("waiting because '{}'".format(e))
             time.sleep(1)
             continue
+        servers = js['servers']
 
-        if len(js['servers']) < minimum_number_of_servers:
+        if len(servers) < minimum_number_of_servers:
-            print("waiting because insufficient servers")
+            print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})")
             time.sleep(1)
             continue
+
+        print(
+            f"Now: {time.ctime()}\n"
+            f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}"
+        )
+
         server_times = [
             server['last_received_data']
-            for server in js['servers']
+            for server in servers
         ]
         # if any times are null/None that server has never been
         # contacted (so it's down still, probably)
-        if any(t is None for t in server_times):
+        never_received_data = server_times.count(None)
-            print("waiting because at least one server not contacted")
+        if never_received_data > 0:
+            print(f"waiting because {never_received_data} server(s) not contacted")
             time.sleep(1)
             continue
 

newsfragments/3999.bugfix

@@ -0,0 +1 @@
+A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.

newsfragments/4020.minor

@@ -0,0 +1 @@
+

setup.py

@@ -142,7 +142,8 @@ install_requires = [
     # HTTP server and client
     "klein",
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    "werkzeug != 2.2.0",
+    # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
+    "werkzeug != 2.2.0, < 2.3",
     "treq",
     "cbor2",
 
@@ -398,10 +399,31 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "dulwich",
               "gpg",
           ],
-          "test": [
+
+          # Here are the dependencies required to set up a reproducible test
+          # environment.  This could be for CI or local development.  These
+          # are *not* library dependencies of the test suite itself.  They are
+          # the tools we use to run the test suite at all.
+          "testenv": [
+              # Pin all of these versions for the same reason you ever want to
+              # pin anything: to prevent new releases with regressions from
+              # introducing spurious failures into CI runs for whatever
+              # development work is happening at the time.  The versions
+              # selected here are just the current versions at the time.
+              # Bumping them to keep up with future releases is fine as long
+              # as those releases are known to actually work.
+              "pip==22.0.3",
+              "wheel==0.37.1",
+              "setuptools==60.9.1",
+              "subunitreporter==22.2.0",
+              "python-subunit==1.4.2",
+              "junitxml==0.7",
               "coverage ~= 5.0",
+          ],
+
+          # Here are the library dependencies of the test suite.
+          "test": [
               "mock",
-              "tox ~= 3.0",
               "pytest",
               "pytest-twisted",
               "hypothesis >= 3.6.1",
@@ -410,7 +432,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "fixtures",
               "beautifulsoup4",
               "html5lib",
-              "junitxml",
               # Pin old version until
               # https://github.com/paramiko/paramiko/issues/1961 is fixed.
               "paramiko < 2.9",

src/allmydata/client.py

@@ -7,7 +7,7 @@ import os
 import stat
 import time
 import weakref
-from typing import Optional
+from typing import Optional, Iterable
 from base64 import urlsafe_b64encode
 from functools import partial
 # On Python 2 this will be the backported package:
@@ -189,7 +189,7 @@ class Terminator(service.Service):
         return service.Service.stopService(self)
 
 
-def read_config(basedir, portnumfile, generated_files=[]):
+def read_config(basedir, portnumfile, generated_files: Iterable=()):
     """
     Read and validate configuration for a client-style Node. See
     :method:`allmydata.node.read_config` for parameter meanings (the
@@ -1103,7 +1103,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         # may get an opaque node if there were any problems.
         return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
 
-    def create_dirnode(self, initial_children={}, version=None):
+    def create_dirnode(self, initial_children=None, version=None):
         d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
         return d
 

src/allmydata/dirnode.py

@@ -678,8 +678,10 @@ class DirectoryNode(object):
         return d
 
     # XXX: Too many arguments? Worthwhile to break into mutable/immutable?
-    def create_subdirectory(self, namex, initial_children={}, overwrite=True,
+    def create_subdirectory(self, namex, initial_children=None, overwrite=True,
                             mutable=True, mutable_version=None, metadata=None):
+        if initial_children is None:
+            initial_children = {}
         name = normalize(namex)
         if self.is_readonly():
             return defer.fail(NotWriteableError())

src/allmydata/hashtree.py

@@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
             name += " (leaf [%d] of %d)" % (leafnum, numleaves)
         return name
 
-    def set_hashes(self, hashes={}, leaves={}):
+    def set_hashes(self, hashes=None, leaves=None):
         """Add a bunch of hashes to the tree.
 
         I will validate these to the best of my ability. If I already have a
@@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
         corrupted or one of the received hashes was corrupted. If it raises
         NotEnoughHashesError, then the otherhashes dictionary was incomplete.
         """
-
+        if hashes is None:
+            hashes = {}
+        if leaves is None:
+            leaves = {}
         assert isinstance(hashes, dict)
         for h in hashes.values():
             assert isinstance(h, bytes)

src/allmydata/immutable/upload.py

@@ -1391,7 +1391,9 @@ class CHKUploader(object):
     def get_upload_status(self):
         return self._upload_status
 
-def read_this_many_bytes(uploadable, size, prepend_data=[]):
+def read_this_many_bytes(uploadable, size, prepend_data=None):
+    if prepend_data is None:
+        prepend_data = []
     if size == 0:
         return defer.succeed([])
     d = uploadable.read(size)

src/allmydata/interfaces.py

@@ -1447,7 +1447,7 @@ class IDirectoryNode(IFilesystemNode):
         is a file, or if must_be_file is True and the child is a directory,
         I raise ChildOfWrongTypeError."""
 
-    def create_subdirectory(name, initial_children={}, overwrite=True,
+    def create_subdirectory(name, initial_children=None, overwrite=True,
                             mutable=True, mutable_version=None, metadata=None):
         """I create and attach a directory at the given name. The new
         directory can be empty, or it can be populated with children
@@ -2586,7 +2586,7 @@ class IClient(Interface):
         @return: a Deferred that fires with an IMutableFileNode instance.
         """
 
-    def create_dirnode(initial_children={}):
+    def create_dirnode(initial_children=None):
         """Create a new unattached dirnode, possibly with initial children.
 
         @param initial_children: dict with keys that are unicode child names,
@@ -2641,7 +2641,7 @@ class INodeMaker(Interface):
         for use by unit tests, to create mutable files that are smaller than
         usual."""
 
-    def create_new_mutable_directory(initial_children={}):
+    def create_new_mutable_directory(initial_children=None):
         """I create a new mutable directory, and return a Deferred that will
         fire with the IDirectoryNode instance when it is ready. If
         initial_children= is provided (a dict mapping unicode child name to

src/allmydata/introducer/server.py

@@ -68,10 +68,6 @@ def create_introducer(basedir=u"."):
         default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider)
         tub_options = create_tub_options(config)
 
-        # we don't remember these because the Introducer doesn't make
-        # outbound connections.
-        i2p_provider = None
-        tor_provider = None
         main_tub = create_main_tub(
             config, tub_options, default_connection_handlers,
             foolscap_connection_handlers, i2p_provider, tor_provider,
@@ -83,6 +79,8 @@ def create_introducer(basedir=u"."):
             i2p_provider,
             tor_provider,
         )
+        i2p_provider.setServiceParent(node)
+        tor_provider.setServiceParent(node)
         return defer.succeed(node)
     except Exception:
         return Failure()

src/allmydata/node.py

@@ -17,7 +17,7 @@ import errno
 from base64 import b32decode, b32encode
 from errno import ENOENT, EPERM
 from warnings import warn
-from typing import Union
+from typing import Union, Iterable
 
 import attr
 
@@ -172,7 +172,7 @@ def create_node_dir(basedir, readme_text):
             f.write(readme_text)
 
 
-def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
+def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None):
     """
     Read and validate configuration.
 
@@ -741,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider):
 
 
 def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
-               handler_overrides={}, force_foolscap=False, **kwargs):
+               handler_overrides=None, force_foolscap=False, **kwargs):
     """
     Create a Tub with the right options and handlers. It will be
     ephemeral unless the caller provides certFile= in kwargs
@@ -755,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
     :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS
         storage protocol.
     """
+    if handler_overrides is None:
+        handler_overrides = {}
     # We listen simultaneously for both Foolscap and HTTPS on the same port,
     # so we have to create a special Foolscap Tub for that to work:
     if force_foolscap:
@@ -922,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location):
 def create_main_tub(config, tub_options,
                     default_connection_handlers, foolscap_connection_handlers,
                     i2p_provider, tor_provider,
-                    handler_overrides={}, cert_filename="node.pem"):
+                    handler_overrides=None, cert_filename="node.pem"):
     """
     Creates a 'main' Foolscap Tub, typically for use as the top-level
     access point for a running Node.
@@ -943,6 +945,8 @@ def create_main_tub(config, tub_options,
     :param tor_provider: None, or a _Provider instance if txtorcon +
         Tor are installed.
     """
+    if handler_overrides is None:
+        handler_overrides = {}
     portlocation = _tub_portlocation(
         config,
         iputil.get_local_addresses_sync,

src/allmydata/nodemaker.py

@@ -135,8 +135,9 @@ class NodeMaker(object):
         d.addCallback(lambda res: n)
         return d
 
-    def create_new_mutable_directory(self, initial_children={}, version=None):
+    def create_new_mutable_directory(self, initial_children=None, version=None):
-        # initial_children must have metadata (i.e. {} instead of None)
+        if initial_children is None:
+            initial_children = {}
         for (name, (node, metadata)) in initial_children.items():
             precondition(isinstance(metadata, dict),
                          "create_new_mutable_directory requires metadata to be a dict, not None", metadata)

src/allmydata/test/cli/wormholetesting.py

@@ -70,7 +70,8 @@ class MemoryWormholeServer(object):
         appid: str,
         relay_url: str,
         reactor: Any,
-        versions: Any={},
+        # Unfortunately we need a mutable default to match the real API
+        versions: Any={},  # noqa: B006
         delegate: Optional[Any]=None,
         journal: Optional[Any]=None,
         tor: Optional[Any]=None,

src/allmydata/test/no_network.py

@@ -476,7 +476,7 @@ class GridTestMixin(object):
         ])
 
     def set_up_grid(self, num_clients=1, num_servers=10,
-                    client_config_hooks={}, oneshare=False):
+                    client_config_hooks=None, oneshare=False):
         """
         Create a Tahoe-LAFS storage grid.
 
@@ -489,6 +489,8 @@ class GridTestMixin(object):
 
         :return: ``None``
         """
+        if client_config_hooks is None:
+            client_config_hooks = {}
         # self.basedir must be set
         port_assigner = SameProcessStreamEndpointAssigner()
         port_assigner.setUp()

src/allmydata/test/test_system.py

@@ -1,20 +1,13 @@
 """
 Ported to Python 3.
 """
-from __future__ import print_function
+from __future__ import annotations
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    # Don't import bytes since it causes issues on (so far unported) modules on Python 2.
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str  # noqa: F401
 
 from past.builtins import chr as byteschr, long
 from six import ensure_text
 
 import os, re, sys, time, json
+from typing import Optional
 
 from bs4 import BeautifulSoup
 
@@ -56,10 +49,12 @@ from .common_util import run_cli_unicode
 
 
 class RunBinTahoeMixin(object):
-    def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
+    def run_bintahoe(self, args, stdin=None, python_options:Optional[list[str]]=None, env=None):
         # test_runner.run_bintahoe has better unicode support but doesn't
         # support env yet and is also synchronous.  If we could get rid of
         # this in favor of that, though, it would probably be an improvement.
+        if python_options is None:
+            python_options = []
         command = sys.executable
         argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args
 
@@ -1088,7 +1083,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii")
         return self.POST2(urlpath, body, headers, use_helper)
 
-    def POST2(self, urlpath, body=b"", headers={}, use_helper=False):
+    def POST2(self, urlpath, body=b"", headers=None, use_helper=False):
+        if headers is None:
+            headers = {}
         if use_helper:
             url = self.helper_webish_url + urlpath
         else:
@@ -1409,7 +1406,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
             defer.returnValue((out,err))
 
-        def _check_ls(out_and_err, expected_children, unexpected_children=[]):
+        def _check_ls(out_and_err, expected_children, unexpected_children=()):
             (out, err) = out_and_err
             self.failUnlessEqual(err, "")
             for s in expected_children:

src/allmydata/test/web/test_web.py

@@ -565,7 +565,9 @@ class WebMixin(TimezoneMixin):
         returnValue(data)
 
     @inlineCallbacks
-    def HEAD(self, urlpath, return_response=False, headers={}):
+    def HEAD(self, urlpath, return_response=False, headers=None):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         response = yield treq.request("head", url, persistent=False,
                                       headers=headers)
@@ -573,7 +575,9 @@ class WebMixin(TimezoneMixin):
             raise Error(response.code, response="")
         returnValue( ("", response.code, response.headers) )
 
-    def PUT(self, urlpath, data, headers={}):
+    def PUT(self, urlpath, data, headers=None):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         return do_http("put", url, data=data, headers=headers)
 
@@ -618,7 +622,9 @@ class WebMixin(TimezoneMixin):
         body, headers = self.build_form(**fields)
         return self.POST2(urlpath, body, headers)
 
-    def POST2(self, urlpath, body="", headers={}, followRedirect=False):
+    def POST2(self, urlpath, body="", headers=None, followRedirect=False):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         if isinstance(body, str):
             body = body.encode("utf-8")

src/allmydata/util/dbutil.py

@@ -25,7 +25,7 @@ class DBError(Exception):
 
 
 def get_db(dbfile, stderr=sys.stderr,
-           create_version=(None, None), updaters={}, just_create=False, dbname="db",
+           create_version=(None, None), updaters=None, just_create=False, dbname="db",
            ):
     """Open or create the given db file. The parent directory must exist.
     create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table.
@@ -33,6 +33,8 @@ def get_db(dbfile, stderr=sys.stderr,
     to get from ver=1 to ver=2. Returns a (sqlite3,db) tuple, or raises
     DBError.
     """
+    if updaters is None:
+        updaters = {}
     must_create = not os.path.exists(dbfile)
     try:
         db = sqlite3.connect(dbfile)

tox.ini

@@ -23,38 +23,34 @@ minversion = 2.4
 
 [testenv]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
-# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
-# a package to be installed (with setuptools) then it'll fail on certain
-# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
-# requirements (TLS >= 1.2) are incompatible with the old TLS clients
-# available to those systems.  Installing it ahead of time (with pip) avoids
-# this problem.
 deps =
-     # Pin all of these versions for the same reason you ever want to pin
+    # We pull in certify *here* to avoid bug #2913. Basically if a
-     # anything: to prevent new releases with regressions from introducing
+    # `setup_requires=...` causes a package to be installed (with setuptools)
-     # spurious failures into CI runs for whatever development work is
+    # then it'll fail on certain platforms (travis's OX-X 10.12, Slackware
-     # happening at the time.  The versions selected here are just the current
+    # 14.2) because PyPI's TLS requirements (TLS >= 1.2) are incompatible with
-     # versions at the time.  Bumping them to keep up with future releases is
+    # the old TLS clients available to those systems.  Installing it ahead of
-     # fine as long as those releases are known to actually work.
+    # time (with pip) avoids this problem.
-     pip==22.0.3
+    #
-     setuptools==60.9.1
+    # We don't pin an exact version of it because it contains CA certificates
-     wheel==0.37.1
+    # which necessarily change over time.  Pinning this is guaranteed to cause
-     subunitreporter==22.2.0
+    # things to break eventually as old certificates expire and as new ones
-     # As an exception, we don't pin certifi because it contains CA
+    # are used in the wild that aren't present in whatever version we pin.
-     # certificates which necessarily change over time.  Pinning this is
+    # Hopefully there won't be functionality regressions in new releases of
-     # guaranteed to cause things to break eventually as old certificates
+    # this package that cause us the kind of suffering we're trying to avoid
-     # expire and as new ones are used in the wild that aren't present in
+    # with the above pins.
-     # whatever version we pin.  Hopefully there won't be functionality
-     # regressions in new releases of this package that cause us the kind of
-     # suffering we're trying to avoid with the above pins.
     certifi
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.
 usedevelop = False
-# We use extras=test to get things like "mock" that are required for our unit
+
-# tests.
+extras =
-extras = test
+       # Get general testing environment dependencies so we can run the tests
+       # how we like.
+       testenv
+
+       # And get all of the test suite's actual direct Python dependencies.
+       test
 
 setenv =
        # Define TEST_SUITE in the environment as an aid to constructing the