From 2d34b6a998f3f5eb454e915415e94e1387a1ee06 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Apr 2022 16:29:00 -0400
Subject: [PATCH 01/94] Log the request and response in the server.

---
 src/allmydata/storage/http_server.py | 36 ++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 7c4860d57..935390d10 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -8,6 +8,7 @@ from functools import wraps
 from base64 import b64decode
 import binascii
 
+from eliot import start_action
 from zope.interface import implementer
 from klein import Klein
 from twisted.web import http
@@ -83,8 +84,9 @@ def _extract_secrets(
 
 def _authorization_decorator(required_secrets):
     """
-    Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization``
-    headers and pass them in.
+    1. Check the ``Authorization`` header matches server swissnum.
+    2. Extract ``X-Tahoe-Authorization`` headers and pass them in.
+    3. Log the request and response.
     """
 
     def decorator(f):
@@ -106,7 +108,22 @@ def _authorization_decorator(required_secrets):
             except ClientSecretsException:
                 request.setResponseCode(http.BAD_REQUEST)
                 return b"Missing required secrets"
-            return f(self, request, secrets, *args, **kwargs)
+            with start_action(
+                action_type="allmydata:storage:http-server:request",
+                method=request.method,
+                path=request.path,
+            ) as ctx:
+                try:
+                    result = f(self, request, secrets, *args, **kwargs)
+                except _HTTPError as e:
+                    # This isn't an error necessarily for logging purposes,
+                    # it's an implementation detail, an easier way to set
+                    # response codes.
+                    ctx.add_success_fields(response_code=e.code)
+                    ctx.finish()
+                    raise
+                ctx.add_success_fields(response_code=request.code)
+                return result
 
         return route
 
@@ -239,19 +256,24 @@ class _HTTPError(Exception):
 # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258
 # indicates a set.
 _SCHEMAS = {
-    "allocate_buckets": Schema("""
+    "allocate_buckets": Schema(
+        """
     message = {
       share-numbers: #6.258([* uint])
       allocated-size: uint
     }
-    """),
-    "advise_corrupt_share": Schema("""
+    """
+    ),
+    "advise_corrupt_share": Schema(
+        """
     message = {
       reason: tstr
     }
-    """)
+    """
+    ),
 }
 
+
 class HTTPServer(object):
     """
     A HTTP interface to the storage server.

From 2722e81d2b4c8bffeebbab3be9654f1cb40736ad Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Apr 2022 16:33:48 -0400
Subject: [PATCH 02/94] News file.

---
 newsfragments/3880.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3880.minor

diff --git a/newsfragments/3880.minor b/newsfragments/3880.minor
new file mode 100644
index 000000000..e69de29bb

From bd631665f4c5d66d16574dbab036c76529a16bca Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 4 May 2022 10:21:16 -0400
Subject: [PATCH 03/94] Add logging of HTTP requests from client.

---
 src/allmydata/storage/http_client.py | 88 +++++++++++++++-------------
 1 file changed, 48 insertions(+), 40 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index da350e0c6..b4bde6a95 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -4,6 +4,7 @@ HTTP client that talks to the HTTP storage server.
 
 from __future__ import annotations
 
+from eliot import start_action, register_exception_extractor
 from typing import Union, Optional, Sequence, Mapping
 from base64 import b64encode
 
@@ -55,6 +56,8 @@ class ClientException(Exception):
         Exception.__init__(self, code, *additional_args)
         self.code = code
 
+register_exception_extractor(ClientException, lambda e: {"response_code": e.code})
+
 
 # Schemas for server responses.
 #
@@ -280,6 +283,7 @@ class StorageClient(object):
         )
         return headers
 
+    @inlineCallbacks
     def request(
         self,
         method,
@@ -299,37 +303,40 @@ class StorageClient(object):
         If ``message_to_serialize`` is set, it will be serialized (by default
         with CBOR) and set as the request body.
         """
-        headers = self._get_headers(headers)
+        with start_action(action_type="allmydata:storage:http-client:request", method=method, url=str(url)) as ctx:
+            headers = self._get_headers(headers)
 
-        # Add secrets:
-        for secret, value in [
-            (Secrets.LEASE_RENEW, lease_renew_secret),
-            (Secrets.LEASE_CANCEL, lease_cancel_secret),
-            (Secrets.UPLOAD, upload_secret),
-            (Secrets.WRITE_ENABLER, write_enabler_secret),
-        ]:
-            if value is None:
-                continue
-            headers.addRawHeader(
-                "X-Tahoe-Authorization",
-                b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
-            )
-
-        # Note we can accept CBOR:
-        headers.addRawHeader("Accept", CBOR_MIME_TYPE)
-
-        # If there's a request message, serialize it and set the Content-Type
-        # header:
-        if message_to_serialize is not None:
-            if "data" in kwargs:
-                raise TypeError(
-                    "Can't use both `message_to_serialize` and `data` "
-                    "as keyword arguments at the same time"
+            # Add secrets:
+            for secret, value in [
+                (Secrets.LEASE_RENEW, lease_renew_secret),
+                (Secrets.LEASE_CANCEL, lease_cancel_secret),
+                (Secrets.UPLOAD, upload_secret),
+                (Secrets.WRITE_ENABLER, write_enabler_secret),
+            ]:
+                if value is None:
+                    continue
+                headers.addRawHeader(
+                    "X-Tahoe-Authorization",
+                    b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
                 )
-            kwargs["data"] = dumps(message_to_serialize)
-            headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
-        return self._treq.request(method, url, headers=headers, **kwargs)
+            # Note we can accept CBOR:
+            headers.addRawHeader("Accept", CBOR_MIME_TYPE)
+
+            # If there's a request message, serialize it and set the Content-Type
+            # header:
+            if message_to_serialize is not None:
+                if "data" in kwargs:
+                    raise TypeError(
+                        "Can't use both `message_to_serialize` and `data` "
+                        "as keyword arguments at the same time"
+                    )
+                kwargs["data"] = dumps(message_to_serialize)
+                headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
+
+            response = yield self._treq.request(method, url, headers=headers, **kwargs)
+            ctx.add_success_fields(response_code=response.code)
+            return response
 
 
 class StorageClientGeneral(object):
@@ -538,18 +545,19 @@ class StorageClientImmutables(object):
         """
         Return the set of shares for a given storage index.
         """
-        url = self._client.relative_url(
-            "/v1/immutable/{}/shares".format(_encode_si(storage_index))
-        )
-        response = yield self._client.request(
-            "GET",
-            url,
-        )
-        if response.code == http.OK:
-            body = yield _decode_cbor(response, _SCHEMAS["list_shares"])
-            returnValue(set(body))
-        else:
-            raise ClientException(response.code)
+        with start_action(action_type="allmydata:storage:http-client:immutable:list-shares", storage_index=storage_index) as ctx:
+            url = self._client.relative_url(
+                "/v1/immutable/{}/shares".format(_encode_si(storage_index))
+            )
+            response = yield self._client.request(
+                "GET",
+                url,
+            )
+            if response.code == http.OK:
+                body = yield _decode_cbor(response, _SCHEMAS["list_shares"])
+                return set(body)
+            else:
+                raise ClientException(response.code)
 
     @inlineCallbacks
     def add_or_renew_lease(

From 4211fd8525bfe5e45c323a9389df1ec5f54aab8d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Mar 2023 13:41:30 -0400
Subject: [PATCH 04/94] Revert to old code.

---
 src/allmydata/storage/http_client.py | 46 ++++++++++++++++------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index a61f94708..44ba64363 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -355,7 +355,6 @@ class StorageClient(object):
         )
         return headers
 
-    @inlineCallbacks
     def request(
         self,
         method,
@@ -378,26 +377,35 @@ class StorageClient(object):
 
         Default timeout is 60 seconds.
         """
-        with start_action(
-            action_type="allmydata:storage:http-client:request",
-            method=method,
-            url=str(url),
-        ) as ctx:
-            headers = self._get_headers(headers)
+        headers = self._get_headers(headers)
 
-            # Add secrets:
-            for secret, value in [
-                (Secrets.LEASE_RENEW, lease_renew_secret),
-                (Secrets.LEASE_CANCEL, lease_cancel_secret),
-                (Secrets.UPLOAD, upload_secret),
-                (Secrets.WRITE_ENABLER, write_enabler_secret),
-            ]:
-                if value is None:
-                    continue
-                headers.addRawHeader(
-                    "X-Tahoe-Authorization",
-                    b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
+        # Add secrets:
+        for secret, value in [
+            (Secrets.LEASE_RENEW, lease_renew_secret),
+            (Secrets.LEASE_CANCEL, lease_cancel_secret),
+            (Secrets.UPLOAD, upload_secret),
+            (Secrets.WRITE_ENABLER, write_enabler_secret),
+        ]:
+            if value is None:
+                continue
+            headers.addRawHeader(
+                "X-Tahoe-Authorization",
+                b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
+            )
+
+        # Note we can accept CBOR:
+        headers.addRawHeader("Accept", CBOR_MIME_TYPE)
+
+        # If there's a request message, serialize it and set the Content-Type
+        # header:
+        if message_to_serialize is not None:
+            if "data" in kwargs:
+                raise TypeError(
+                    "Can't use both `message_to_serialize` and `data` "
+                    "as keyword arguments at the same time"
                 )
+            kwargs["data"] = dumps(message_to_serialize)
+            headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
         return self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs

From fbcef2d1ae7f0893e4e4dc55066baa0b01feff4e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:32:40 -0400
Subject: [PATCH 05/94] Safely customize the Tor introducer's configuration

Previously we clobbered the whole generated configuration and potentially
wiped out additional important fields.

Now we modify the configuration by just changing the fields we need to change.
---
 integration/conftest.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 33e7998c1..54632be26 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -43,7 +43,7 @@ from .util import (
     generate_ssh_key,
     block_with_timeout,
 )
-
+from allmydata.node import read_config
 
 # pytest customization hooks
 
@@ -275,13 +275,6 @@ def introducer_furl(introducer, temp_dir):
     include_result=False,
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_tor
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_tor')
     print("making introducer", intro_dir)
 
@@ -301,9 +294,11 @@ log_gatherer.furl = {log_furl}
         )
         pytest_twisted.blockon(done_proto.done)
 
-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    # adjust a few settings
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 1c11f9e7d4957fcb1312418feb65c6a56847586e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:35:14 -0400
Subject: [PATCH 06/94] Add a little more debug info to the integration test
 suite output

---
 integration/conftest.py | 3 +++
 integration/test_tor.py | 2 +-
 integration/util.py     | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 54632be26..280d98f72 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -321,7 +321,9 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             pass
     request.addfinalizer(cleanup)
 
+    print("Waiting for introducer to be ready...")
     pytest_twisted.blockon(protocol.magic_seen)
+    print("Introducer ready.")
     return transport
 
 
@@ -332,6 +334,7 @@ def tor_introducer_furl(tor_introducer, temp_dir):
         print("Don't see {} yet".format(furl_fname))
         sleep(.1)
     furl = open(furl_fname, 'r').read()
+    print(f"Found Tor introducer furl: {furl}")
     return furl
 
 
diff --git a/integration/test_tor.py b/integration/test_tor.py
index 901858347..8398cf9a4 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -93,7 +93,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
     if True:
-        print("creating", node_dir.path)
+        print(f"creating {node_dir.path} with introducer {introducer_furl}")
         node_dir.makedirs()
         proto = util._DumpOutputProtocol(None)
         reactor.spawnProcess(
diff --git a/integration/util.py b/integration/util.py
index 05fef8fed..08c07a059 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -607,7 +607,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             continue
 
         if len(js['servers']) < minimum_number_of_servers:
-            print("waiting because insufficient servers")
+            print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})")
             time.sleep(1)
             continue
         server_times = [

From 1c99817e1b0d098a422a2d0ccdc4cb6a5cb3d489 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:41:51 -0400
Subject: [PATCH 07/94] Safely customize the client node's configuration

This is similar to the fix to the `tor_introducer` fixture.
---
 integration/test_tor.py | 39 +++++++++++----------------------------
 1 file changed, 11 insertions(+), 28 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 8398cf9a4..b116fe319 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -25,6 +25,7 @@ from twisted.python.filepath import (
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.client import read_config
 
 # see "conftest.py" for the fixtures (e.g. "tor_network")
 
@@ -103,10 +104,14 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                 'create-node',
                 '--nickname', name,
+                '--webport', web_port,
                 '--introducer', introducer_furl,
                 '--hide-ip',
                 '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                 '--listen', 'tor',
+                '--shares-needed', '1',
+                '--shares-happy', '1',
+                '--shares-total', '2',
                 node_dir.path,
             )
         )
@@ -115,35 +120,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
-    with node_dir.child('tahoe.cfg').open('w') as f:
-        node_config = '''
-[node]
-nickname = %(name)s
-web.port = %(web_port)s
-web.static = public_html
-log_gatherer.furl = %(log_furl)s
 
-[tor]
-control.port = tcp:localhost:%(control_port)d
-onion.external_port = 3457
-onion.local_port = %(local_port)d
-onion = true
-onion.private_key_file = private/tor_onion.privkey
-
-[client]
-shares.needed = 1
-shares.happy = 1
-shares.total = 2
-
-''' % {
-    'name': name,
-    'web_port': web_port,
-    'log_furl': flog_gatherer,
-    'control_port': control_port,
-    'local_port': control_port + 1000,
-}
-        node_config = node_config.encode("utf-8")
-        f.write(node_config)
+    config = read_config(node_dir.path, "tub.port")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
+    config.set_config("tor", "onion", "true")
+    config.set_config("tor", "onion.external_port", "3457")
+    config.set_config("tor", "onion.local_port", str(control_port + 1000))
+    config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
     print("running")
     result = yield util._run_node(reactor, node_dir.path, request, None)

From 50c4ad81136a21096a6ce540938c70afc299fadd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:07:53 -0400
Subject: [PATCH 08/94] news fragment

---
 newsfragments/3999.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3999.minor

diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor
new file mode 100644
index 000000000..e69de29bb

From e8c72e6753db8287ef1dcfa824080e1191746c2a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 28 Mar 2023 12:55:41 -0400
Subject: [PATCH 09/94] Not sure if per method logging is worth it, will start
 from assumption that HTTP logging is enough.

---
 src/allmydata/storage/http_client.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 44ba64363..fcfc5bff3 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -746,22 +746,18 @@ class StorageClientImmutables(object):
         """
         Return the set of shares for a given storage index.
         """
-        with start_action(
-            action_type="allmydata:storage:http-client:immutable:list-shares",
-            storage_index=storage_index,
-        ) as ctx:
-            url = self._client.relative_url(
-                "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
-            )
-            response = yield self._client.request(
-                "GET",
-                url,
-            )
-            if response.code == http.OK:
-                body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
-                returnValue(set(body))
-            else:
-                raise ClientException(response.code)
+        url = self._client.relative_url(
+            "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
+        )
+        response = yield self._client.request(
+            "GET",
+            url,
+        )
+        if response.code == http.OK:
+            body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            returnValue(set(body))
+        else:
+            raise ClientException(response.code)
 
     def advise_corrupt_share(
         self,

From d36adf33a41be87814da8ccdf9a10c21813d53db Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 28 Mar 2023 13:06:43 -0400
Subject: [PATCH 10/94] Refactor; failing tests for some reason.

---
 src/allmydata/storage/http_server.py | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 4f970b5a7..517771c02 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -106,28 +106,31 @@ def _authorization_decorator(required_secrets):
     def decorator(f):
         @wraps(f)
         def route(self, request, *args, **kwargs):
-            if not timing_safe_compare(
-                request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
-                    "utf-8"
-                ),
-                swissnum_auth_header(self._swissnum),
-            ):
-                request.setResponseCode(http.UNAUTHORIZED)
-                return b""
-            authorization = request.requestHeaders.getRawHeaders(
-                "X-Tahoe-Authorization", []
-            )
-            try:
-                secrets = _extract_secrets(authorization, required_secrets)
-            except ClientSecretsException:
-                request.setResponseCode(http.BAD_REQUEST)
-                return b"Missing required secrets"
             with start_action(
-                action_type="allmydata:storage:http-server:request",
+                action_type="allmydata:storage:http-server:handle_request",
                 method=request.method,
                 path=request.path,
             ) as ctx:
                 try:
+                    # Check Authorization header:
+                    if not timing_safe_compare(
+                        request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
+                            "utf-8"
+                        ),
+                        swissnum_auth_header(self._swissnum),
+                    ):
+                        raise _HTTPError(http.UNAUTHORIZED)
+
+                    # Check secrets:
+                    authorization = request.requestHeaders.getRawHeaders(
+                        "X-Tahoe-Authorization", []
+                    )
+                    try:
+                        secrets = _extract_secrets(authorization, required_secrets)
+                    except ClientSecretsException:
+                        raise _HTTPError(http.BAD_REQUEST)
+
+                    # Run the business logic:
                     result = f(self, request, secrets, *args, **kwargs)
                 except _HTTPError as e:
                     # This isn't an error necessarily for logging purposes,
@@ -136,8 +139,9 @@ def _authorization_decorator(required_secrets):
                     ctx.add_success_fields(response_code=e.code)
                     ctx.finish()
                     raise
-                ctx.add_success_fields(response_code=request.code)
-                return result
+                else:
+                    ctx.add_success_fields(response_code=request.code)
+                    return result
 
         return route
 

From b81fad2970ff3eeea87fa869318dcee1f6db6ce9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 10:37:49 -0400
Subject: [PATCH 11/94] Make sure tests have the same error testing
 infrastructure as the real thing.

---
 src/allmydata/storage/http_server.py    | 28 ++++++++++++++-----------
 src/allmydata/test/test_storage_http.py |  2 ++
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 517771c02..5ccb43c60 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -489,6 +489,21 @@ def read_range(
     return d
 
 
+def _add_error_handling(app: Klein):
+    """Add exception handlers to a Klein app."""
+    @app.handle_errors(_HTTPError)
+    def _http_error(_, request, failure):
+        """Handle ``_HTTPError`` exceptions."""
+        request.setResponseCode(failure.value.code)
+        return b""
+
+    @app.handle_errors(CDDLValidationError)
+    def _cddl_validation_error(_, request, failure):
+        """Handle CDDL validation errors."""
+        request.setResponseCode(http.BAD_REQUEST)
+        return str(failure.value).encode("utf-8")
+
+
 class HTTPServer(object):
     """
     A HTTP interface to the storage server.
@@ -496,18 +511,7 @@ class HTTPServer(object):
 
     _app = Klein()
     _app.url_map.converters["storage_index"] = StorageIndexConverter
-
-    @_app.handle_errors(_HTTPError)
-    def _http_error(self, request, failure):
-        """Handle ``_HTTPError`` exceptions."""
-        request.setResponseCode(failure.value.code)
-        return b""
-
-    @_app.handle_errors(CDDLValidationError)
-    def _cddl_validation_error(self, request, failure):
-        """Handle CDDL validation errors."""
-        request.setResponseCode(http.BAD_REQUEST)
-        return str(failure.value).encode("utf-8")
+    _add_error_handling(_app)
 
     def __init__(
         self,
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index eb5bcd4db..19529cd0e 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -54,6 +54,7 @@ from ..storage.http_server import (
     ClientSecretsException,
     _authorized_route,
     StorageIndexConverter,
+    _add_error_handling
 )
 from ..storage.http_client import (
     StorageClient,
@@ -253,6 +254,7 @@ class TestApp(object):
 
     clock: IReactorTime
     _app = Klein()
+    _add_error_handling(_app)
     _swissnum = SWISSNUM_FOR_TEST  # Match what the test client is using
 
     @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"])

From 41939e2b286fedca55af7ad202739751c40d7f3d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:11:24 -0400
Subject: [PATCH 12/94] Add some type annotations.

---
 src/allmydata/storage/http_client.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index fcfc5bff3..6450050a9 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -341,7 +341,7 @@ class StorageClient(object):
         https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
         return cls(https_url, swissnum, treq_client, reactor)
 
-    def relative_url(self, path):
+    def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
         return self._base_url.click(path)
 
@@ -357,14 +357,14 @@ class StorageClient(object):
 
     def request(
         self,
-        method,
-        url,
-        lease_renew_secret=None,
-        lease_cancel_secret=None,
-        upload_secret=None,
-        write_enabler_secret=None,
-        headers=None,
-        message_to_serialize=None,
+        method: str,
+        url: DecodedURL,
+        lease_renew_secret: Optional[bytes]=None,
+        lease_cancel_secret: Optional[bytes]=None,
+        upload_secret: Optional[bytes]=None,
+        write_enabler_secret: Optional[bytes]=None,
+        headers: Optional[Headers]=None,
+        message_to_serialize: object=None,
         timeout: float = 60,
         **kwargs,
     ):

From 3b3ea5409c7c31b5158c2ec5093d7021a927d2d3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:26:08 -0400
Subject: [PATCH 13/94] Type says we should only pass in DecodedURL.

---
 src/allmydata/test/test_storage_http.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 19529cd0e..ea93ad360 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -54,7 +54,7 @@ from ..storage.http_server import (
     ClientSecretsException,
     _authorized_route,
     StorageIndexConverter,
-    _add_error_handling
+    _add_error_handling,
 )
 from ..storage.http_client import (
     StorageClient,
@@ -348,7 +348,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/upload_secret",
+                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
             )
         )
         self.assertEqual(response.code, 400)
@@ -356,7 +356,9 @@ class CustomHTTPServerTests(SyncTestCase):
         # With secret, we're good.
         response = result_of(
             self.client.request(
-                "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC"
+                "GET",
+                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
+                upload_secret=b"MAGIC",
             )
         )
         self.assertEqual(response.code, 200)
@@ -380,7 +382,7 @@ class CustomHTTPServerTests(SyncTestCase):
             response = result_of(
                 self.client.request(
                     "GET",
-                    f"http://127.0.0.1/bytes/{length}",
+                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                 )
             )
 
@@ -401,7 +403,7 @@ class CustomHTTPServerTests(SyncTestCase):
             response = result_of(
                 self.client.request(
                     "GET",
-                    f"http://127.0.0.1/bytes/{length}",
+                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                 )
             )
 
@@ -416,7 +418,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/slowly_never_finish_result",
+                DecodedURL.from_text("http://127.0.0.1/slowly_never_finish_result"),
             )
         )
 
@@ -444,7 +446,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/die",
+                DecodedURL.from_text("http://127.0.0.1/die"),
             )
         )
 
@@ -461,6 +463,7 @@ class Reactor(Clock):
 
     Advancing the clock also runs any callbacks scheduled via callFromThread.
     """
+
     def __init__(self):
         Clock.__init__(self)
         self._queue = Queue()
@@ -501,7 +504,9 @@ class HttpTestFixture(Fixture):
         self.storage_server = StorageServer(
             self.tempdir.path, b"\x00" * 20, clock=self.clock
         )
-        self.http_server = HTTPServer(self.clock, self.storage_server, SWISSNUM_FOR_TEST)
+        self.http_server = HTTPServer(
+            self.clock, self.storage_server, SWISSNUM_FOR_TEST
+        )
         self.treq = StubTreq(self.http_server.get_resource())
         self.client = StorageClient(
             DecodedURL.from_text("http://127.0.0.1"),

From 57ec669e1e70dc0fd6be4b6ddfbe3fe0f32221de Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:29:57 -0400
Subject: [PATCH 14/94] Add logging for request().

---
 src/allmydata/storage/http_client.py | 58 ++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 6450050a9..cbd4634b4 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -19,7 +19,7 @@ from collections_extended import RangeMap
 from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
-from twisted.web.iweb import IPolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS, IResponse
 from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
@@ -355,19 +355,20 @@ class StorageClient(object):
         )
         return headers
 
-    def request(
+    @async_to_deferred
+    async def request(
         self,
         method: str,
-        url: DecodedURL,
-        lease_renew_secret: Optional[bytes]=None,
-        lease_cancel_secret: Optional[bytes]=None,
-        upload_secret: Optional[bytes]=None,
-        write_enabler_secret: Optional[bytes]=None,
-        headers: Optional[Headers]=None,
-        message_to_serialize: object=None,
+        url: str,
+        lease_renew_secret: Optional[bytes] = None,
+        lease_cancel_secret: Optional[bytes] = None,
+        upload_secret: Optional[bytes] = None,
+        write_enabler_secret: Optional[bytes] = None,
+        headers: Optional[Headers] = None,
+        message_to_serialize: object = None,
         timeout: float = 60,
         **kwargs,
-    ):
+    ) -> Deferred[IResponse]:
         """
         Like ``treq.request()``, but with optional secrets that get translated
         into corresponding HTTP headers.
@@ -377,6 +378,41 @@ class StorageClient(object):
 
         Default timeout is 60 seconds.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:request",
+            method=method,
+            url=url.to_text(),
+            timeout=timeout,
+        ) as ctx:
+            response = await self._request(
+                method,
+                url,
+                lease_renew_secret,
+                lease_cancel_secret,
+                upload_secret,
+                write_enabler_secret,
+                headers,
+                message_to_serialize,
+                timeout,
+                **kwargs,
+            )
+            ctx.add_success_fields(response_code=response.code)
+            return response
+
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        lease_renew_secret: Optional[bytes] = None,
+        lease_cancel_secret: Optional[bytes] = None,
+        upload_secret: Optional[bytes] = None,
+        write_enabler_secret: Optional[bytes] = None,
+        headers: Optional[Headers] = None,
+        message_to_serialize: object = None,
+        timeout: float = 60,
+        **kwargs,
+    ) -> IResponse:
+        """The implementation of request()."""
         headers = self._get_headers(headers)
 
         # Add secrets:
@@ -407,7 +443,7 @@ class StorageClient(object):
             kwargs["data"] = dumps(message_to_serialize)
             headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
-        return self._treq.request(
+        return await self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 

From 5e3fa04a3a6e0ae2210fcdc49347e18c3e384ec2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:30:22 -0400
Subject: [PATCH 15/94] Reformat with black.

---
 src/allmydata/storage/http_client.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index cbd4634b4..cead33732 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -488,12 +488,14 @@ class StorageClientGeneral(object):
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
-        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update({
-            b'tolerates-immutable-read-overrun': True,
-            b'delete-mutable-shares-with-zero-length-writev': True,
-            b'fills-holes-with-zero-bytes': True,
-            b'prevents-read-past-end-of-share-data': True,
-        })
+        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update(
+            {
+                b"tolerates-immutable-read-overrun": True,
+                b"delete-mutable-shares-with-zero-length-writev": True,
+                b"fills-holes-with-zero-bytes": True,
+                b"prevents-read-past-end-of-share-data": True,
+            }
+        )
         returnValue(decoded_response)
 
     @inlineCallbacks

From e19aeb5aea2bd57bfb2de3bc6f7f87a9097723c6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:40:48 -0400
Subject: [PATCH 16/94] Correct the annotation.

---
 src/allmydata/storage/http_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index cead33732..bd9e3fc39 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -359,7 +359,7 @@ class StorageClient(object):
     async def request(
         self,
         method: str,
-        url: str,
+        url: DecodedURL,
         lease_renew_secret: Optional[bytes] = None,
         lease_cancel_secret: Optional[bytes] = None,
         upload_secret: Optional[bytes] = None,
@@ -402,7 +402,7 @@ class StorageClient(object):
     async def _request(
         self,
         method: str,
-        url: str,
+        url: DecodedURL,
         lease_renew_secret: Optional[bytes] = None,
         lease_cancel_secret: Optional[bytes] = None,
         upload_secret: Optional[bytes] = None,

From 1de8e811b5d963695af4c02886e85ee1ede36619 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 4 Apr 2023 10:58:22 -0400
Subject: [PATCH 17/94] Tweaks.

---
 integration/test_tor.py | 6 ++++--
 integration/util.py     | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index b116fe319..d418f786b 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -43,8 +43,8 @@ if PY2:
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2)
-    yield util.await_client_ready(dave, minimum_number_of_servers=2)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
@@ -125,6 +125,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
     config.set_config("tor", "onion", "true")
     config.set_config("tor", "onion.external_port", "3457")
+    config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
+    #config.set_config("tor", "launch", "True")
     config.set_config("tor", "onion.local_port", str(control_port + 1000))
     config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
diff --git a/integration/util.py b/integration/util.py
index 08c07a059..a11c02225 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -90,6 +90,7 @@ class _CollectOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
+        print("OUT: {!r}".format(data))
         self.output.write(data)
 
     def errReceived(self, data):

From efa51d41dcdbf430510927b4ad6d41a9835b267a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 4 Apr 2023 10:58:28 -0400
Subject: [PATCH 18/94] Newer chutney.

---
 integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 280d98f72..36e7eef0b 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -465,7 +465,7 @@ def chutney(reactor, temp_dir):
         'git',
         (
             'git', 'clone',
-            'https://git.torproject.org/chutney.git',
+            'https://gitlab.torproject.org/tpo/core/chutney.git',
             chutney_dir,
         ),
         env=environ,
@@ -481,7 +481,7 @@ def chutney(reactor, temp_dir):
         (
             'git', '-C', chutney_dir,
             'reset', '--hard',
-            'c825cba0bcd813c644c6ac069deeb7347d3200ee'
+            'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2'
         ),
         env=environ,
     )

From 2be9e949f0c22c1daba57c2951bb5ff8eac9654d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:02:34 -0400
Subject: [PATCH 19/94] add Ubuntu 22.04 unit test job to CircleCI

---
 .circleci/config.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d46e255af..638b4fc3e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -39,6 +39,8 @@ dockerhub-auth-template: &DOCKERHUB_AUTH
         <<: *DOCKERHUB_CONTEXT
     - "build-image-ubuntu-20-04":
         <<: *DOCKERHUB_CONTEXT
+    - "build-image-ubuntu-22-04":
+        <<: *DOCKERHUB_CONTEXT
     - "build-image-fedora-35":
         <<: *DOCKERHUB_CONTEXT
     - "build-image-oraclelinux-8":
@@ -78,6 +80,9 @@ workflows:
       - "ubuntu-20-04":
           {}
 
+      - "ubuntu-22-04":
+          {}
+
       # Equivalent to RHEL 8; CentOS 8 is dead.
       - "oraclelinux-8":
           {}
@@ -333,6 +338,16 @@ jobs:
       <<: *UTF_8_ENVIRONMENT
       TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
 
+  ubuntu-22-04:
+    <<: *DEBIAN
+    docker:
+      - <<: *DOCKERHUB_AUTH
+        image: "tahoelafsci/ubuntu:22.04-py3.10"
+        user: "nobody"
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py310"
+
   oraclelinux-8: &RHEL_DERIV
     docker:
       - <<: *DOCKERHUB_AUTH
@@ -479,6 +494,15 @@ jobs:
       PYTHON_VERSION: "3.9"
 
 
+  build-image-ubuntu-22-04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "22.04"
+      PYTHON_VERSION: "3.10"
+
+
   build-image-oraclelinux-8:
     <<: *BUILD_IMAGE
 

From 8557c66b39ed6ec806160eec857ba017ee2506de Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:03:20 -0400
Subject: [PATCH 20/94] Remove the "ubuntu-latest" unit test job from GitHub
 Actions

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e006d90ac..adcf6cc5d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,7 +46,6 @@ jobs:
       matrix:
         os:
           - windows-latest
-          - ubuntu-latest
         python-version:
           - "3.8"
           - "3.9"

From 7ae7db678eae92fa41450240f91b56335aaff9dd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:03:51 -0400
Subject: [PATCH 21/94] add CPython 3.8 and CPython 3.9 unit test jobs to
 CircleCI

---
 .circleci/config.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 638b4fc3e..77c29734d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -93,6 +93,8 @@ workflows:
           matrix:
             parameters:
               pythonVersion:
+                - "python38"
+                - "python39"
                 - "python310"
 
       - "nixos":

From 4c542dfa9b5aa06b3514e95204f74c0860e07329 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:37:16 -0400
Subject: [PATCH 22/94] news fragment

---
 newsfragments/4006.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4006.minor

diff --git a/newsfragments/4006.minor b/newsfragments/4006.minor
new file mode 100644
index 000000000..e69de29bb

From 812458699dc22a62f49419a2fd62bcf7510b08b2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 5 Apr 2023 11:38:28 -0400
Subject: [PATCH 23/94] The tcp listening port needs to match the onion local
 port, or you get connection refused when you try to connect to the hidden
 service.

---
 integration/test_tor.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index d418f786b..6f6f54c25 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -126,8 +126,6 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     config.set_config("tor", "onion", "true")
     config.set_config("tor", "onion.external_port", "3457")
     config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
-    #config.set_config("tor", "launch", "True")
-    config.set_config("tor", "onion.local_port", str(control_port + 1000))
     config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
     print("running")

From 13e9f88309c15f5f4bfe71c0abe8bff5a2c2326b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 7 Apr 2023 15:23:20 -0400
Subject: [PATCH 24/94] Add necessary config option to ensure it listens on
 Tor, and also give correct Tor control port.

---
 integration/conftest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 621c0224c..f3cf9a9d8 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -294,7 +294,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             request,
             (
                 'create-introducer',
-                '--tor-control-port', 'tcp:localhost:8010',
+                '--tor-control-port', 'tcp:localhost:8007',
+                '--hide-ip',
                 '--listen=tor',
                 intro_dir,
             ),

From 7b9432482724297c6d637aee20c2a6f5d94339ff Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 7 Apr 2023 15:23:51 -0400
Subject: [PATCH 25/94] More debugging.

---
 integration/util.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/util.py b/integration/util.py
index a11c02225..ac3fe2833 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -140,6 +140,7 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
+        print("OUT", data)
         data = str(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
@@ -148,6 +149,7 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
+        print("ERR", data)
         data = str(data, sys.stderr.encoding)
         sys.stdout.write(data)
 

From 4d4649f5c24ff89f1b538a09740eaffefea80dd2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 Apr 2023 11:28:26 -0400
Subject: [PATCH 26/94] Apply suggestions from code review

Co-authored-by: Jean-Paul Calderone <exarkun@twistedmatrix.com>
---
 src/allmydata/storage/http_client.py | 2 +-
 src/allmydata/storage/http_server.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index bd9e3fc39..ea142ed85 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -368,7 +368,7 @@ class StorageClient(object):
         message_to_serialize: object = None,
         timeout: float = 60,
         **kwargs,
-    ) -> Deferred[IResponse]:
+    ) -> IResponse:
         """
         Like ``treq.request()``, but with optional secrets that get translated
         into corresponding HTTP headers.
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 5ccb43c60..8647274f8 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -107,7 +107,7 @@ def _authorization_decorator(required_secrets):
         @wraps(f)
         def route(self, request, *args, **kwargs):
             with start_action(
-                action_type="allmydata:storage:http-server:handle_request",
+                action_type="allmydata:storage:http-server:handle-request",
                 method=request.method,
                 path=request.path,
             ) as ctx:

From cebf62176ee7ec064936c111aa28cb65f69d649b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 10 Apr 2023 11:40:59 -0400
Subject: [PATCH 27/94] WIP add logging to decode_cbor.

---
 src/allmydata/storage/http_client.py | 46 +++++++++++++++-------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ea142ed85..131f23846 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -447,24 +447,28 @@ class StorageClient(object):
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 
-    def decode_cbor(self, response, schema: Schema):
+    async def decode_cbor(self, response, schema: Schema) -> object:
         """Given HTTP response, return decoded CBOR body."""
-
-        def got_content(f: BinaryIO):
-            data = f.read()
-            schema.validate_cbor(data)
-            return loads(data)
-
-        if response.code > 199 and response.code < 300:
-            content_type = get_content_type(response.headers)
-            if content_type == CBOR_MIME_TYPE:
-                return limited_content(response, self._clock).addCallback(got_content)
+        with start_action(action_type="allmydata:storage:http-client:decode-cbor"):
+            if response.code > 199 and response.code < 300:
+                content_type = get_content_type(response.headers)
+                if content_type == CBOR_MIME_TYPE:
+                    f = await limited_content(response, self._clock)
+                    data = f.read()
+                    schema.validate_cbor(data)
+                    return loads(data)
+                else:
+                    raise ClientException(
+                        -1,
+                        "Server didn't send CBOR, content type is {}".format(
+                            content_type
+                        ),
+                    )
             else:
-                raise ClientException(-1, "Server didn't send CBOR")
-        else:
-            return treq.content(response).addCallback(
-                lambda data: fail(ClientException(response.code, response.phrase, data))
-            )
+                data = (
+                    await limited_content(response, self._clock, max_length=10_000)
+                ).read()
+                raise ClientException(response.code, response.phrase, data)
 
 
 @define(hash=True)
@@ -475,14 +479,14 @@ class StorageClientGeneral(object):
 
     _client: StorageClient
 
-    @inlineCallbacks
-    def get_version(self):
+    @async_to_deferred
+    async def get_version(self):
         """
         Return the version metadata for the server.
         """
         url = self._client.relative_url("/storage/v1/version")
-        response = yield self._client.request("GET", url)
-        decoded_response = yield self._client.decode_cbor(
+        response = await self._client.request("GET", url)
+        decoded_response = await self._client.decode_cbor(
             response, _SCHEMAS["get_version"]
         )
         # Add some features we know are true because the HTTP API
@@ -496,7 +500,7 @@ class StorageClientGeneral(object):
                 b"prevents-read-past-end-of-share-data": True,
             }
         )
-        returnValue(decoded_response)
+        return decoded_response
 
     @inlineCallbacks
     def add_or_renew_lease(

From 2a7616e0bebe29865767141255e36e9058db77e6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 16:43:46 -0400
Subject: [PATCH 28/94] Get tests passing again.

---
 src/allmydata/storage/http_client.py    | 36 ++++++++++++-------------
 src/allmydata/test/test_storage_http.py |  3 ++-
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 131f23846..7fc68c902 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -651,8 +651,8 @@ class StorageClientImmutables(object):
 
     _client: StorageClient
 
-    @inlineCallbacks
-    def create(
+    @async_to_deferred
+    async def create(
         self,
         storage_index,
         share_numbers,
@@ -679,7 +679,7 @@ class StorageClientImmutables(object):
         )
         message = {"share-numbers": share_numbers, "allocated-size": allocated_size}
 
-        response = yield self._client.request(
+        response = await self._client.request(
             "POST",
             url,
             lease_renew_secret=lease_renew_secret,
@@ -687,14 +687,12 @@ class StorageClientImmutables(object):
             upload_secret=upload_secret,
             message_to_serialize=message,
         )
-        decoded_response = yield self._client.decode_cbor(
+        decoded_response = await self._client.decode_cbor(
             response, _SCHEMAS["allocate_buckets"]
         )
-        returnValue(
-            ImmutableCreateResult(
-                already_have=decoded_response["already-have"],
-                allocated=decoded_response["allocated"],
-            )
+        return ImmutableCreateResult(
+            already_have=decoded_response["already-have"],
+            allocated=decoded_response["allocated"],
         )
 
     @inlineCallbacks
@@ -720,8 +718,8 @@ class StorageClientImmutables(object):
                 response.code,
             )
 
-    @inlineCallbacks
-    def write_share_chunk(
+    @async_to_deferred
+    async def write_share_chunk(
         self, storage_index, share_number, upload_secret, offset, data
     ):  # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress]
         """
@@ -741,7 +739,7 @@ class StorageClientImmutables(object):
                 _encode_si(storage_index), share_number
             )
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "PATCH",
             url,
             upload_secret=upload_secret,
@@ -765,13 +763,13 @@ class StorageClientImmutables(object):
             raise ClientException(
                 response.code,
             )
-        body = yield self._client.decode_cbor(
+        body = await self._client.decode_cbor(
             response, _SCHEMAS["immutable_write_share_chunk"]
         )
         remaining = RangeMap()
         for chunk in body["required"]:
             remaining.set(True, chunk["begin"], chunk["end"])
-        returnValue(UploadProgress(finished=finished, required=remaining))
+        return UploadProgress(finished=finished, required=remaining)
 
     def read_share_chunk(
         self, storage_index, share_number, offset, length
@@ -783,21 +781,21 @@ class StorageClientImmutables(object):
             self._client, "immutable", storage_index, share_number, offset, length
         )
 
-    @inlineCallbacks
-    def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
+    @async_to_deferred
+    async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
         """
         Return the set of shares for a given storage index.
         """
         url = self._client.relative_url(
             "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "GET",
             url,
         )
         if response.code == http.OK:
-            body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
-            returnValue(set(body))
+            body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            return set(body)
         else:
             raise ClientException(response.code)
 
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index ea93ad360..eca2be1c1 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -34,7 +34,7 @@ from hyperlink import DecodedURL
 from collections_extended import RangeMap
 from twisted.internet.task import Clock, Cooperator
 from twisted.internet.interfaces import IReactorTime, IReactorFromThreads
-from twisted.internet.defer import CancelledError, Deferred
+from twisted.internet.defer import CancelledError, Deferred, ensureDeferred
 from twisted.web import http
 from twisted.web.http_headers import Headers
 from werkzeug import routing
@@ -520,6 +520,7 @@ class HttpTestFixture(Fixture):
         Like ``result_of``, but supports fake reactor and ``treq`` testing
         infrastructure necessary to support asynchronous HTTP server endpoints.
         """
+        d = ensureDeferred(d)
         result = []
         error = []
         d.addCallbacks(result.append, error.append)

From 3997eaaf9048af3daca9cc291875f17b2a403218 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:00:31 -0400
Subject: [PATCH 29/94] Fix type annotations.

---
 src/allmydata/storage/http_client.py | 49 ++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 7fc68c902..b1877cd5f 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server.
 from __future__ import annotations
 
 from eliot import start_action, register_exception_extractor
-from typing import Union, Optional, Sequence, Mapping, BinaryIO
+from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
@@ -486,13 +486,17 @@ class StorageClientGeneral(object):
         """
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
-        decoded_response = await self._client.decode_cbor(
-            response, _SCHEMAS["get_version"]
+        decoded_response = cast(
+            dict[bytes, object],
+            await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
         )
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
-        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update(
+        cast(
+            dict[bytes, object],
+            decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
+        ).update(
             {
                 b"tolerates-immutable-read-overrun": True,
                 b"delete-mutable-shares-with-zero-length-writev": True,
@@ -687,8 +691,9 @@ class StorageClientImmutables(object):
             upload_secret=upload_secret,
             message_to_serialize=message,
         )
-        decoded_response = await self._client.decode_cbor(
-            response, _SCHEMAS["allocate_buckets"]
+        decoded_response = cast(
+            dict[str, set[int]],
+            await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]),
         )
         return ImmutableCreateResult(
             already_have=decoded_response["already-have"],
@@ -763,8 +768,11 @@ class StorageClientImmutables(object):
             raise ClientException(
                 response.code,
             )
-        body = await self._client.decode_cbor(
-            response, _SCHEMAS["immutable_write_share_chunk"]
+        body = cast(
+            dict[str, list[dict[str, int]]],
+            await self._client.decode_cbor(
+                response, _SCHEMAS["immutable_write_share_chunk"]
+            ),
         )
         remaining = RangeMap()
         for chunk in body["required"]:
@@ -794,7 +802,10 @@ class StorageClientImmutables(object):
             url,
         )
         if response.code == http.OK:
-            body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            body = cast(
+                set[int],
+                await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
+            )
             return set(body)
         else:
             raise ClientException(response.code)
@@ -865,6 +876,12 @@ class ReadTestWriteResult:
     reads: Mapping[int, Sequence[bytes]]
 
 
+# Result type for mutable read/test/write HTTP response.
+MUTABLE_RTW = TypedDict(
+    "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]}
+)
+
+
 @frozen
 class StorageClientMutables:
     """
@@ -911,8 +928,11 @@ class StorageClientMutables:
             message_to_serialize=message,
         )
         if response.code == http.OK:
-            result = await self._client.decode_cbor(
-                response, _SCHEMAS["mutable_read_test_write"]
+            result = cast(
+                MUTABLE_RTW,
+                await self._client.decode_cbor(
+                    response, _SCHEMAS["mutable_read_test_write"]
+                ),
             )
             return ReadTestWriteResult(success=result["success"], reads=result["data"])
         else:
@@ -942,8 +962,11 @@ class StorageClientMutables:
         )
         response = await self._client.request("GET", url)
         if response.code == http.OK:
-            return await self._client.decode_cbor(
-                response, _SCHEMAS["mutable_list_shares"]
+            return cast(
+                set[int],
+                await self._client.decode_cbor(
+                    response, _SCHEMAS["mutable_list_shares"]
+                ),
             )
         else:
             raise ClientException(response.code)

From 8bda370b30ef187d321840ecea97c56a439e1280 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:00:47 -0400
Subject: [PATCH 30/94] News fragment.

---
 newsfragments/4005.misc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4005.misc

diff --git a/newsfragments/4005.misc b/newsfragments/4005.misc
new file mode 100644
index 000000000..e69de29bb

From 840ed0bf47561c7c9720c91deab3d5a5c56a7b35 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:04:00 -0400
Subject: [PATCH 31/94] Unused imports.

---
 src/allmydata/storage/http_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index b1877cd5f..ef4005414 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -20,7 +20,7 @@ from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
 from twisted.web.iweb import IPolicyForHTTPS, IResponse
-from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed
+from twisted.internet.defer import inlineCallbacks, Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
     IReactorTime,

From 33ab0ce0422ff6211bd614bb8429fbe2084b9e02 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:10:33 -0400
Subject: [PATCH 32/94] Fix name.

---
 newsfragments/{4005.misc => 4005.minor} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename newsfragments/{4005.misc => 4005.minor} (100%)

diff --git a/newsfragments/4005.misc b/newsfragments/4005.minor
similarity index 100%
rename from newsfragments/4005.misc
rename to newsfragments/4005.minor

From 507d1f8394cedc672715fee7bec1b5ef75cb6037 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Wed, 12 Apr 2023 22:34:45 -0600
Subject: [PATCH 33/94] Fix some Chutney things (and a couple cleanups): wait
 for bootstrap, increase timeout

---
 integration/conftest.py | 44 ++++++++++++++++++++++-------------------
 integration/test_tor.py |  2 +-
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index f3cf9a9d8..7a4234de7 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -206,13 +206,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     include_result=False,
 )
 def introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer0
-web.port = 4560
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer')
     print("making introducer", intro_dir)
 
@@ -232,6 +225,10 @@ log_gatherer.furl = {log_furl}
         )
         pytest_twisted.blockon(done_proto.done)
 
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
     # over-write the config file with our stuff
     with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
         f.write(config)
@@ -283,7 +280,8 @@ def introducer_furl(introducer, temp_dir):
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request):
     intro_dir = join(temp_dir, 'introducer_tor')
-    print("making introducer", intro_dir)
+    print("making Tor introducer in {}".format(intro_dir))
+    print("(this can take tens of seconds to allocate Onion address)")
 
     if not exists(intro_dir):
         mkdir(intro_dir)
@@ -342,7 +340,7 @@ def tor_introducer_furl(tor_introducer, temp_dir):
         print("Don't see {} yet".format(furl_fname))
         sleep(.1)
     furl = open(furl_fname, 'r').read()
-    print(f"Found Tor introducer furl: {furl}")
+    print(f"Found Tor introducer furl: {furl} in {furl_fname}")
     return furl
 
 
@@ -510,7 +508,13 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
     )
     pytest_twisted.blockon(proto.done)
 
-    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
+    return (
+        chutney_dir,
+        {
+            "PYTHONPATH": join(chutney_dir, "lib"),
+            "CHUTNEY_START_TIME": "200",  # default is 60
+        }
+    )
 
 
 @pytest.fixture(scope='session')
@@ -544,17 +548,9 @@ def tor_network(reactor, temp_dir, chutney, request):
         return proto.done
 
     # now, as per Chutney's README, we have to create the network
-    # ./chutney configure networks/basic
-    # ./chutney start networks/basic
     pytest_twisted.blockon(chutney(("configure", basic_network)))
-    pytest_twisted.blockon(chutney(("start", basic_network)))
-
-    # print some useful stuff
-    try:
-        pytest_twisted.blockon(chutney(("status", basic_network)))
-    except ProcessTerminated:
-        print("Chutney.TorNet status failed (continuing)")
 
+    # ensure we will tear down the network right before we start it
     def cleanup():
         print("Tearing down Chutney Tor network")
         try:
@@ -563,5 +559,13 @@ def tor_network(reactor, temp_dir, chutney, request):
             # If this doesn't exit cleanly, that's fine, that shouldn't fail
             # the test suite.
             pass
-
     request.addfinalizer(cleanup)
+
+    pytest_twisted.blockon(chutney(("start", basic_network)))
+    pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network)))
+
+    # print some useful stuff
+    try:
+        pytest_twisted.blockon(chutney(("status", basic_network)))
+    except ProcessTerminated:
+        print("Chutney.TorNet status failed (continuing)")
diff --git a/integration/test_tor.py b/integration/test_tor.py
index fb9d8c086..c3041f6d3 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -61,7 +61,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     )
     yield proto.done
     cap = proto.output.getvalue().strip().split()[-1]
-    print("TEH CAP!", cap)
+    print("capability: {}".format(cap))
 
     proto = util._CollectOutputProtocol(capture_stderr=False)
     reactor.spawnProcess(

From 9472841c39120b408bfb7efab3b90b3fcb048a53 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Wed, 12 Apr 2023 23:01:28 -0600
Subject: [PATCH 34/94] enable tor, i2p services

---
 src/allmydata/introducer/server.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index 98136157d..e0ff138cc 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -83,6 +83,8 @@ def create_introducer(basedir=u"."):
             i2p_provider,
             tor_provider,
         )
+        i2p_provider.setServiceParent(node)
+        tor_provider.setServiceParent(node)
         return defer.succeed(node)
     except Exception:
         return Failure()

From 175473df407157db53276e0721fec324242e4bd2 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 13 Apr 2023 00:37:32 -0600
Subject: [PATCH 35/94] longer timeouts, forget less

---
 integration/conftest.py            | 2 +-
 integration/test_tor.py            | 4 ++--
 src/allmydata/introducer/server.py | 4 ----
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 7a4234de7..e7e021016 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -512,7 +512,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         chutney_dir,
         {
             "PYTHONPATH": join(chutney_dir, "lib"),
-            "CHUTNEY_START_TIME": "200",  # default is 60
+            "CHUTNEY_START_TIME": "600",  # default is 60
         }
     )
 
diff --git a/integration/test_tor.py b/integration/test_tor.py
index c3041f6d3..10e326e46 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -33,8 +33,8 @@ if sys.platform.startswith('win'):
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60)
-    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index e0ff138cc..5dad89ae8 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -68,10 +68,6 @@ def create_introducer(basedir=u"."):
         default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider)
         tub_options = create_tub_options(config)
 
-        # we don't remember these because the Introducer doesn't make
-        # outbound connections.
-        i2p_provider = None
-        tor_provider = None
         main_tub = create_main_tub(
             config, tub_options, default_connection_handlers,
             foolscap_connection_handlers, i2p_provider, tor_provider,

From af845a40c6dd1fe626771c63dff0c8dd7a6d86c8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:38:33 -0400
Subject: [PATCH 36/94] Fix type annotations, removing Deferred in particular.

---
 src/allmydata/storage/http_client.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ef4005414..e21cfc5cc 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -658,13 +658,13 @@ class StorageClientImmutables(object):
     @async_to_deferred
     async def create(
         self,
-        storage_index,
-        share_numbers,
-        allocated_size,
-        upload_secret,
-        lease_renew_secret,
-        lease_cancel_secret,
-    ):  # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult]
+        storage_index: bytes,
+        share_numbers: set[int],
+        allocated_size: int,
+        upload_secret: bytes,
+        lease_renew_secret: bytes,
+        lease_cancel_secret: bytes,
+    ) -> ImmutableCreateResult:
         """
         Create a new storage index for an immutable.
 
@@ -725,8 +725,13 @@ class StorageClientImmutables(object):
 
     @async_to_deferred
     async def write_share_chunk(
-        self, storage_index, share_number, upload_secret, offset, data
-    ):  # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress]
+        self,
+        storage_index: bytes,
+        share_number: int,
+        upload_secret: bytes,
+        offset: int,
+        data: bytes,
+    ) -> UploadProgress:
         """
         Upload a chunk of data for a specific share.
 
@@ -790,7 +795,7 @@ class StorageClientImmutables(object):
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
+    async def list_shares(self, storage_index: bytes) -> set[int]:
         """
         Return the set of shares for a given storage index.
         """

From e9a9ac7110a88e8410a80d0481040fb44f614be2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:44:52 -0400
Subject: [PATCH 37/94] Rip out codecov for now.

---
 .circleci/config.yml             | 2 +-
 .circleci/populate-wheelhouse.sh | 2 +-
 .github/workflows/ci.yml         | 2 +-
 newsfragments/4010.minor         | 0
 4 files changed, 3 insertions(+), 3 deletions(-)
 create mode 100644 newsfragments/4010.minor

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 77c29734d..54b2706cd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -260,7 +260,7 @@ jobs:
           name: "Submit coverage results"
           command: |
             if [ -n "${UPLOAD_COVERAGE}" ]; then
-              /tmp/venv/bin/codecov
+              echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011"
             fi
 
     docker:
diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 857171979..374ca0adb 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel"
 
 # Python packages we need to support the test infrastructure.  *Not* packages
 # Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0 codecov"
+TEST_DEPS="tox~=3.0"
 
 # Python packages we need to generate test reports for CI infrastructure.
 # *Not* packages Tahoe-LAFS itself (implement or test suite) need.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index adcf6cc5d..1bb7c9efb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -79,7 +79,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade codecov "tox<4" tox-gh-actions setuptools
+          pip install --upgrade "tox<4" tox-gh-actions setuptools
           pip list
 
       - name: Display tool versions
diff --git a/newsfragments/4010.minor b/newsfragments/4010.minor
new file mode 100644
index 000000000..e69de29bb

From 464b47619028e9d194e660ad461467acaf8986b4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 13:11:17 -0400
Subject: [PATCH 38/94] Work on 3.8.

---
 src/allmydata/storage/http_client.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index e21cfc5cc..fc165bedd 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server.
 from __future__ import annotations
 
 from eliot import start_action, register_exception_extractor
-from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict
+from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict, Set
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
@@ -487,14 +487,14 @@ class StorageClientGeneral(object):
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
         decoded_response = cast(
-            dict[bytes, object],
+            Mapping[bytes, object],
             await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
         )
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
         cast(
-            dict[bytes, object],
+            Mapping[bytes, object],
             decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
         ).update(
             {
@@ -692,7 +692,7 @@ class StorageClientImmutables(object):
             message_to_serialize=message,
         )
         decoded_response = cast(
-            dict[str, set[int]],
+            Mapping[str, Set[int]],
             await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]),
         )
         return ImmutableCreateResult(
@@ -774,7 +774,7 @@ class StorageClientImmutables(object):
                 response.code,
             )
         body = cast(
-            dict[str, list[dict[str, int]]],
+            Mapping[str, Sequence[Mapping[str, int]]],
             await self._client.decode_cbor(
                 response, _SCHEMAS["immutable_write_share_chunk"]
             ),
@@ -795,7 +795,7 @@ class StorageClientImmutables(object):
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> set[int]:
+    async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         Return the set of shares for a given storage index.
         """
@@ -808,7 +808,7 @@ class StorageClientImmutables(object):
         )
         if response.code == http.OK:
             body = cast(
-                set[int],
+                Set[int],
                 await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
             )
             return set(body)
@@ -881,9 +881,10 @@ class ReadTestWriteResult:
     reads: Mapping[int, Sequence[bytes]]
 
 
-# Result type for mutable read/test/write HTTP response.
+# Result type for mutable read/test/write HTTP response. Can't just use
+# dict[int,list[bytes]] because on Python 3.8 that will error out.
 MUTABLE_RTW = TypedDict(
-    "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]}
+    "MUTABLE_RTW", {"success": bool, "data": Mapping[int, Sequence[bytes]]}
 )
 
 
@@ -958,7 +959,7 @@ class StorageClientMutables:
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> set[int]:
+    async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         List the share numbers for a given storage index.
         """
@@ -968,7 +969,7 @@ class StorageClientMutables:
         response = await self._client.request("GET", url)
         if response.code == http.OK:
             return cast(
-                set[int],
+                Set[int],
                 await self._client.decode_cbor(
                     response, _SCHEMAS["mutable_list_shares"]
                 ),

From 250efe7d24cd43a676a6a7116da35f6ab226401a Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 13 Apr 2023 16:42:02 -0600
Subject: [PATCH 39/94] leftover

---
 integration/conftest.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index e7e021016..b54e18e26 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -229,9 +229,6 @@ def introducer(reactor, temp_dir, flog_gatherer, request):
     config.set_config("node", "nickname", "introducer-tor")
     config.set_config("node", "web.port", "4561")
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 5dcbc00989c94e314a018a8a9d79027796e95ffe Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:18:55 -0400
Subject: [PATCH 40/94] News fragment.

---
 newsfragments/4012.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/4012.bugfix

diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
new file mode 100644
index 000000000..97dfe6aad
--- /dev/null
+++ b/newsfragments/4012.bugfix
@@ -0,0 +1 @@
+The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file

From d7ee1637dfabc3654ea6be735fbcd2094d39c1f3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:22:06 -0400
Subject: [PATCH 41/94] Set a timeout.

---
 src/allmydata/scripts/common_http.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 95099a2eb..7542a045f 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -62,9 +62,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port)
+        c = http_client.HTTPConnection(host, port, timeout=60)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port)
+        c = http_client.HTTPSConnection(host, port, timeout=60)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From 67702572a9d4ce03c6614207234ae909672d6df5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:22:14 -0400
Subject: [PATCH 42/94] Do a little modernization.

---
 src/allmydata/scripts/common_http.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 7542a045f..4da1345c9 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -1,18 +1,11 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 from io import BytesIO
-from six.moves import urllib, http_client
+from http import client as http_client
+import urllib
 import six
 import allmydata # for __full_version__
 

From 1823dd4c03b3d715ef896453542d0ac10e7f4aad Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:24:00 -0400
Subject: [PATCH 43/94] Switch to a slightly larger block size.

---
 src/allmydata/scripts/common_http.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 4da1345c9..4c0319d3c 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -55,9 +55,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=60)
+        c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=60)
+        c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)
@@ -78,7 +78,7 @@ def do_http(method, url, body=b""):
         return BadResponse(url, err)
 
     while True:
-        data = body.read(8192)
+        data = body.read(65536)
         if not data:
             break
         c.send(data)

From 2916984114bdb9ef85df5a34aa439f45c5a14cab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:29:25 -0400
Subject: [PATCH 44/94] More modernization.

---
 src/allmydata/scripts/common_http.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 4c0319d3c..7d627a7ad 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -1,12 +1,11 @@
 """
-Ported to Python 3.
+Blocking HTTP client APIs.
 """
 
 import os
 from io import BytesIO
 from http import client as http_client
 import urllib
-import six
 import allmydata # for __full_version__
 
 from allmydata.util.encodingutil import quote_output
@@ -44,7 +43,7 @@ class BadResponse(object):
 def do_http(method, url, body=b""):
     if isinstance(body, bytes):
         body = BytesIO(body)
-    elif isinstance(body, six.text_type):
+    elif isinstance(body, str):
         raise TypeError("do_http body must be a bytestring, not unicode")
     else:
         # We must give a Content-Length header to twisted.web, otherwise it
@@ -87,16 +86,14 @@ def do_http(method, url, body=b""):
 
 
 def format_http_success(resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s %s" % (resp.status, six.ensure_text(resp.reason)),
+        "%s %s" % (resp.status, resp.reason),
         quotemarks=False)
 
 def format_http_error(msg, resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason),
-                           six.ensure_text(resp.read())),
+        "%s: %s %s\n%s" % (msg, resp.status, resp.reason,
+                           resp.read()),
         quotemarks=False)
 
 def check_http_error(resp, stderr):

From 2d81ddc297b336607bc8eac691913e7e3ac2f178 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 11:15:47 -0400
Subject: [PATCH 45/94] Don't call str() on bytes.

---
 src/allmydata/scripts/common_http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 7d627a7ad..a2cae5a85 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -92,7 +92,7 @@ def format_http_success(resp):
 
 def format_http_error(msg, resp):
     return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, resp.reason,
+        "%s: %s %s\n%r" % (msg, resp.status, resp.reason,
                            resp.read()),
         quotemarks=False)
 

From 76ce54ea53e4e802da612f1f2cbc53c88e9764da Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 13:23:28 -0600
Subject: [PATCH 46/94] remove debugging

---
 integration/util.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index ac3fe2833..887602906 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -90,11 +90,9 @@ class _CollectOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
-        print("OUT: {!r}".format(data))
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {!r}".format(data))
         if self.capture_stderr:
             self.output.write(data)
 

From abfca04af5a8c7e43fa18351b1131ebcf741efd8 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 13:24:22 -0600
Subject: [PATCH 47/94] turn off i2p tests for now

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 96619a93a..597623d9c 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -133,7 +133,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
+def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     # ensure both nodes are connected to "a grid" by uploading

From d3c39f8604fd924bbac424ce201533b4656416b3 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:27:19 -0600
Subject: [PATCH 48/94] fix i2p introducer, different ports

---
 integration/conftest.py |  2 +-
 integration/test_i2p.py | 13 ++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index b54e18e26..f65c84141 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -227,7 +227,7 @@ def introducer(reactor, temp_dir, flog_gatherer, request):
 
     config = read_config(intro_dir, "tub.port")
     config.set_config("node", "nickname", "introducer-tor")
-    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "web.port", "4562")
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 597623d9c..df619c6eb 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -68,13 +68,6 @@ def i2p_network(reactor, temp_dir, request):
     include_result=False,
 )
 def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_i2p
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_i2p')
     print("making introducer", intro_dir)
 
@@ -94,8 +87,10 @@ log_gatherer.furl = {log_furl}
         pytest_twisted.blockon(done_proto.done)
 
     # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer_i2p")
+    config.set_config("node", "web.port", "4563")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 34cee7ff73c05d978354894f734d710d3a5c1a2c Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:44:52 -0600
Subject: [PATCH 49/94] missing import

---
 integration/test_i2p.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index df619c6eb..10abb7e30 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -23,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.node import read_config
+
 
 if which("docker") is None:
     pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)

From 3ccb7c4d1c1f26991a3a467fe4a559738eac0638 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:45:17 -0600
Subject: [PATCH 50/94] re-enable i2p tests

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 10abb7e30..2aa1a536f 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
+def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     # ensure both nodes are connected to "a grid" by uploading

From 8b81bd7ebef79617d75ab1d5d5745d50a3e212b9 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 16:33:52 -0600
Subject: [PATCH 51/94] remove more debug

---
 integration/util.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 887602906..177983e2e 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -138,7 +138,6 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
-        print("OUT", data)
         data = str(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
@@ -147,7 +146,6 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
-        print("ERR", data)
         data = str(data, sys.stderr.encoding)
         sys.stdout.write(data)
 

From 8652bb71ad0ece21f9f75a1c290ccdb4abed6e92 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 17:05:57 -0600
Subject: [PATCH 52/94] skip i2p tests again?

---
 integration/test_i2p.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 2aa1a536f..42b848130 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
+@pytest.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)

From b5f6fa8933c03d5de2069de47d67230ae3d641f2 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 19:07:27 -0600
Subject: [PATCH 53/94] skip properly

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 42b848130..a94648593 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-@pytest.skip("I2P tests are not functioning at all, for unknown reasons")
+@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)

From cda97e4fa63deac934e3085e8bc0edb0a30b85da Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:06:50 -0400
Subject: [PATCH 54/94] Remove pylint, replacing with faster alternative.

---
 tox.ini | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tox.ini b/tox.ini
index 447745784..982157bf1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -100,9 +100,7 @@ commands =
 [testenv:codechecks]
 basepython = python3
 deps =
-     # Make sure we get a version of PyLint that respects config, and isn't too
-     # old.
-     pylint < 2.18, >2.14
+     ruff
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME
 setenv =
@@ -114,9 +112,10 @@ commands =
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
-         # PyLint has other useful checks, might want to enable them:
-         # http://pylint.pycqa.org/en/latest/technical_reference/features.html
-         pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}}
+         # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
+         #       cell-var-from-loop.
+         # ruff could probably replace flake8 and perhaps above tools as well...
+         ruff check --select=B023 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From aafbb00333312e6fc77dcb9558a8b5c9cad75b9e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:10:09 -0400
Subject: [PATCH 55/94] Use ruff for trailing whitespace.

---
 misc/coding_tools/find-trailing-spaces.py | 44 -----------------------
 newsfragments/4014.minor                  |  0
 tox.ini                                   |  4 +--
 3 files changed, 2 insertions(+), 46 deletions(-)
 delete mode 100644 misc/coding_tools/find-trailing-spaces.py
 create mode 100644 newsfragments/4014.minor

diff --git a/misc/coding_tools/find-trailing-spaces.py b/misc/coding_tools/find-trailing-spaces.py
deleted file mode 100644
index 19e7e3c28..000000000
--- a/misc/coding_tools/find-trailing-spaces.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os, sys
-
-from twisted.python import usage
-
-class Options(usage.Options):
-    optFlags = [
-        ("recursive", "r", "Search for .py files recursively"),
-        ]
-    def parseArgs(self, *starting_points):
-        self.starting_points = starting_points
-
-found = [False]
-
-def check(fn):
-    f = open(fn, "r")
-    for i,line in enumerate(f.readlines()):
-        if line == "\n":
-            continue
-        if line[-1] == "\n":
-            line = line[:-1]
-        if line.rstrip() != line:
-            # the %s:%d:%d: lets emacs' compile-mode jump to those locations
-            print("%s:%d:%d: trailing whitespace" % (fn, i+1, len(line)+1))
-            found[0] = True
-    f.close()
-
-o = Options()
-o.parseOptions()
-if o['recursive']:
-    for starting_point in o.starting_points:
-        for root, dirs, files in os.walk(starting_point):
-            for fn in [f for f in files if f.endswith(".py")]:
-                fn = os.path.join(root, fn)
-                check(fn)
-else:
-    for fn in o.starting_points:
-        check(fn)
-if found[0]:
-    sys.exit(1)
-sys.exit(0)
diff --git a/newsfragments/4014.minor b/newsfragments/4014.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/tox.ini b/tox.ini
index 982157bf1..a191d6078 100644
--- a/tox.ini
+++ b/tox.ini
@@ -111,11 +111,11 @@ commands =
          flake8 {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
          # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
          #       cell-var-from-loop.
+         # W291,W293: Trailing whitespace.
          # ruff could probably replace flake8 and perhaps above tools as well...
-         ruff check --select=B023 {posargs:{env:DEFAULT_FILES}}
+         ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From 7b33931df2982bd35e0cdb38d0a1e1d77d34ce47 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:21:20 -0400
Subject: [PATCH 56/94] Replace flake8 with ruff.

---
 .gitignore |  2 ++
 .ruff.toml | 12 ++++++++++++
 setup.cfg  |  3 +++
 setup.py   |  3 +--
 tox.ini    |  7 +------
 5 files changed, 19 insertions(+), 8 deletions(-)
 create mode 100644 .ruff.toml

diff --git a/.gitignore b/.gitignore
index 7c7fa2afd..0cf688c54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,5 @@ zope.interface-*.egg
 # This is the plaintext of the private environment needed for some CircleCI
 # operations.  It's never supposed to be checked in.
 secret-env-plain
+
+.ruff_cache
\ No newline at end of file
diff --git a/.ruff.toml b/.ruff.toml
new file mode 100644
index 000000000..75ff62c2d
--- /dev/null
+++ b/.ruff.toml
@@ -0,0 +1,12 @@
+select = [
+    # Pyflakes checks
+    "F",
+    # Prohibit tabs:
+    "W191",
+    # No trailing whitespace:
+    "W291",
+    "W293",
+    # Make sure we bind closure variables in a loop (equivalent to pylint
+    # cell-var-from-loop):
+    "B023",
+]
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index f4539279e..9415b3ab4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,6 +6,9 @@ develop = update_version develop
 bdist_egg = update_version bdist_egg
 bdist_wheel = update_version bdist_wheel
 
+# This has been replaced by ruff (see .ruff.toml), which has same checks as
+# flake8 plus many more, and is also faster. However, we're keeping this config
+# in case people still use flake8 in IDEs, etc..
 [flake8]
 # Enforce all pyflakes constraints, and also prohibit tabs for indentation.
 # Reference:
diff --git a/setup.py b/setup.py
index 854a333f1..3358aa6c6 100644
--- a/setup.py
+++ b/setup.py
@@ -399,12 +399,11 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "gpg",
           ],
           "test": [
-              "flake8",
               # Pin a specific pyflakes so we don't have different folks
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it
               # intentionally.
-              "pyflakes == 3.0.1",
+              "ruff==0.0.261",
               "coverage ~= 5.0",
               "mock",
               "tox ~= 3.0",
diff --git a/tox.ini b/tox.ini
index a191d6078..2daf8dca3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -108,14 +108,9 @@ setenv =
 	 # entire codebase, including various pieces of supporting code.
 	 DEFAULT_FILES=src integration static misc setup.py
 commands =
-         flake8 {posargs:{env:DEFAULT_FILES}}
+         ruff check {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
-         # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
-         #       cell-var-from-loop.
-         # W291,W293: Trailing whitespace.
-         # ruff could probably replace flake8 and perhaps above tools as well...
-         ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From 6517cd4a48338bc13d991299d361e8c5b65fed22 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:22:27 -0400
Subject: [PATCH 57/94] Fix lint found by ruff.

---
 misc/checkers/check_load.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/misc/checkers/check_load.py b/misc/checkers/check_load.py
index d509b89ae..01a9ed832 100644
--- a/misc/checkers/check_load.py
+++ b/misc/checkers/check_load.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 """
 this is a load-generating client program. It does all of its work through a
 given tahoe node (specified by URL), and performs random reads and writes

From c05afb19dfd7ffb5f053aa2a52972403ccb4fa43 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:33:31 -0400
Subject: [PATCH 58/94] Don't install code, it's not necessary.

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 2daf8dca3..99487bc83 100644
--- a/tox.ini
+++ b/tox.ini
@@ -99,8 +99,10 @@ commands =
 
 [testenv:codechecks]
 basepython = python3
+skip_install = true
 deps =
      ruff
+     towncrier
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME
 setenv =

From 1371ffe9dc7e6a6b0346daad1603f6414bbd1fc7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:14:26 -0400
Subject: [PATCH 59/94] Just have ruff in one place.

---
 setup.py | 5 -----
 tox.ini  | 4 +++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index 3358aa6c6..2418c6dbe 100644
--- a/setup.py
+++ b/setup.py
@@ -399,11 +399,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "gpg",
           ],
           "test": [
-              # Pin a specific pyflakes so we don't have different folks
-              # disagreeing on what is or is not a lint issue.  We can bump
-              # this version from time to time, but we will do it
-              # intentionally.
-              "ruff==0.0.261",
               "coverage ~= 5.0",
               "mock",
               "tox ~= 3.0",
diff --git a/tox.ini b/tox.ini
index 99487bc83..5f18b6b95 100644
--- a/tox.ini
+++ b/tox.ini
@@ -101,7 +101,9 @@ commands =
 basepython = python3
 skip_install = true
 deps =
-     ruff
+     # Pin a specific version so we get consistent outcomes; update this
+     # occasionally:
+     ruff == 0.0.263
      towncrier
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME

From ebed5100b9cf2a208875eb2977da23364559881d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:16:12 -0400
Subject: [PATCH 60/94] Switch to longer timeout so it's unlikely to impact
 users.

---
 newsfragments/4012.bugfix            | 2 +-
 src/allmydata/scripts/common_http.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
index 97dfe6aad..24d5bb49a 100644
--- a/newsfragments/4012.bugfix
+++ b/newsfragments/4012.bugfix
@@ -1 +1 @@
-The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
+The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index a2cae5a85..46676b3f5 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -54,9 +54,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536)
+        c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536)
+        c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From 558e3bf79785fef5a8c3525f323590b9c7c15e36 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:46:57 -0400
Subject: [PATCH 61/94] Fix unnecessary conversion.

---
 src/allmydata/storage/http_client.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index fc165bedd..f786b8f30 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -807,11 +807,10 @@ class StorageClientImmutables(object):
             url,
         )
         if response.code == http.OK:
-            body = cast(
+            return cast(
                 Set[int],
                 await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
             )
-            return set(body)
         else:
             raise ClientException(response.code)
 

From 3d2e4d0798b874c493dc6b86487b527e36aa3324 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:26:58 -0400
Subject: [PATCH 62/94] note about port selection

---
 integration/conftest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/conftest.py b/integration/conftest.py
index f65c84141..f670c6486 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -289,6 +289,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             request,
             (
                 'create-introducer',
+                # The control port should agree with the configuration of the
+                # Tor network we bootstrap with chutney.
                 '--tor-control-port', 'tcp:localhost:8007',
                 '--hide-ip',
                 '--listen=tor',

From c595eea33e78eac579b3f3b63163a0354348a0d6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:27:51 -0400
Subject: [PATCH 63/94] always set the "start time" timeout

in both the "we installed it ourselves" and the "we found an existing
installation" cases.
---
 integration/conftest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index f670c6486..eaf740190 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -511,7 +511,6 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         chutney_dir,
         {
             "PYTHONPATH": join(chutney_dir, "lib"),
-            "CHUTNEY_START_TIME": "600",  # default is 60
         }
     )
 
@@ -534,6 +533,10 @@ def tor_network(reactor, temp_dir, chutney, request):
 
     env = environ.copy()
     env.update(chutney_env)
+    env.update({
+        # default is 60, probably too short for reliable automated use.
+        "CHUTNEY_START_TIME": "600",
+    })
     chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
     def chutney(argv):
         proto = _DumpOutputProtocol(None)

From ba387453cf95ff04a322d8ba4531d81a5f31d7b2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:30:53 -0400
Subject: [PATCH 64/94] it's a bug fix!

it's user-facing!
---
 newsfragments/3999.bugfix | 1 +
 newsfragments/3999.minor  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 newsfragments/3999.bugfix
 delete mode 100644 newsfragments/3999.minor

diff --git a/newsfragments/3999.bugfix b/newsfragments/3999.bugfix
new file mode 100644
index 000000000..a8a8396f4
--- /dev/null
+++ b/newsfragments/3999.bugfix
@@ -0,0 +1 @@
+A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.
\ No newline at end of file
diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor
deleted file mode 100644
index e69de29bb..000000000

From 825bcf3f3b8ca5a924d8c3075db653f9e5bf3c99 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:31:04 -0400
Subject: [PATCH 65/94] revert reformatting

---
 integration/conftest.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index eaf740190..b0d8da90f 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -507,12 +507,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
     )
     pytest_twisted.blockon(proto.done)
 
-    return (
-        chutney_dir,
-        {
-            "PYTHONPATH": join(chutney_dir, "lib"),
-        }
-    )
+    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
 
 
 @pytest.fixture(scope='session')

From fbb5f4c359800e606cc3d29d39d80896292e4c40 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:31:10 -0400
Subject: [PATCH 66/94] slightly clarified comment

---
 integration/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index b0d8da90f..cb590ef6f 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -547,7 +547,7 @@ def tor_network(reactor, temp_dir, chutney, request):
     # now, as per Chutney's README, we have to create the network
     pytest_twisted.blockon(chutney(("configure", basic_network)))
 
-    # ensure we will tear down the network right before we start it
+    # before we start the network, ensure we will tear down at the end
     def cleanup():
         print("Tearing down Chutney Tor network")
         try:

From f9a1eedaeadb818315bef8158902a47c863bbc65 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 12:31:37 -0400
Subject: [PATCH 67/94] Make timeout optional, enable it only for integration
 tests.

---
 integration/conftest.py              |  6 ++++++
 newsfragments/4012.bugfix            |  1 -
 newsfragments/4012.minor             |  0
 src/allmydata/scripts/common_http.py | 11 +++++++++--
 4 files changed, 15 insertions(+), 3 deletions(-)
 delete mode 100644 newsfragments/4012.bugfix
 create mode 100644 newsfragments/4012.minor

diff --git a/integration/conftest.py b/integration/conftest.py
index 879649588..d76b2a9c7 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -4,6 +4,7 @@ Ported to Python 3.
 
 from __future__ import annotations
 
+import os
 import sys
 import shutil
 from time import sleep
@@ -49,6 +50,11 @@ from .util import (
 )
 
 
+# No reason for HTTP requests to take longer than two minutes in the
+# integration tests. See allmydata/scripts/common_http.py for usage.
+os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
+
+
 # pytest customization hooks
 
 def pytest_addoption(parser):
diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
deleted file mode 100644
index 24d5bb49a..000000000
--- a/newsfragments/4012.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
diff --git a/newsfragments/4012.minor b/newsfragments/4012.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 46676b3f5..f138b9c07 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -53,10 +53,17 @@ def do_http(method, url, body=b""):
         assert body.seek
         assert body.read
     scheme, host, port, path = parse_url(url)
+
+    # For testing purposes, allow setting a timeout on HTTP requests. If this
+    # ever become a user-facing feature, this should probably be a CLI option?
+    timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None)
+    if timeout is not None:
+        timeout = float(timeout)
+
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536)
+        c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536)
+        c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From c0e49064ce64eb2860dba6c2957a86485c1a1e41 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 09:50:02 -0400
Subject: [PATCH 68/94] Attempt to get more information about client unready
 state

---
 integration/util.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 177983e2e..39e5dfa6d 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -604,19 +604,27 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             print("waiting because '{}'".format(e))
             time.sleep(1)
             continue
+        servers = js['servers']
 
-        if len(js['servers']) < minimum_number_of_servers:
-            print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})")
+        if len(servers) < minimum_number_of_servers:
+            print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})")
             time.sleep(1)
             continue
+
+        print(
+            f"Now: {time.ctime()}\n"
+            f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}"
+        )
+
         server_times = [
             server['last_received_data']
-            for server in js['servers']
+            for server in servers
         ]
         # if any times are null/None that server has never been
         # contacted (so it's down still, probably)
-        if any(t is None for t in server_times):
-            print("waiting because at least one server not contacted")
+        never_received_data = server_times.count(None)
+        if never_received_data > 0:
+            print(f"waiting because {never_received_data} server(s) not contacted")
             time.sleep(1)
             continue
 

From 8f1d1cc1a0db48a1aa219b9a8814ef4201206098 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:23:06 -0400
Subject: [PATCH 69/94] Include node name in the logging output from
 subprocesses.

---
 integration/conftest.py |  6 +++---
 integration/test_i2p.py |  4 ++--
 integration/util.py     | 11 ++++++-----
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index d76b2a9c7..69d1934b0 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -161,7 +161,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     )
     pytest_twisted.blockon(out_protocol.done)
 
-    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
+    twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer")
     twistd_process = reactor.spawnProcess(
         twistd_protocol,
         which('twistd')[0],
@@ -244,7 +244,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
@@ -320,7 +320,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "tor_introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 96619a93a..4d4dbe620 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -35,7 +35,7 @@ if sys.platform.startswith('win'):
 @pytest.fixture
 def i2p_network(reactor, temp_dir, request):
     """Fixture to start up local i2pd."""
-    proto = util._MagicTextProtocol("ephemeral keys")
+    proto = util._MagicTextProtocol("ephemeral keys", "i2pd")
     reactor.spawnProcess(
         proto,
         which("docker"),
@@ -99,7 +99,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = util._MagicTextProtocol('introducer running')
+    protocol = util._MagicTextProtocol('introducer running', "introducer")
     transport = util._tahoe_runner_optional_coverage(
         protocol,
         reactor,
diff --git a/integration/util.py b/integration/util.py
index 05fef8fed..b1692a7a3 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -12,7 +12,7 @@ import sys
 import time
 import json
 from os import mkdir, environ
-from os.path import exists, join
+from os.path import exists, join, basename
 from io import StringIO, BytesIO
 from subprocess import check_output
 
@@ -129,8 +129,9 @@ class _MagicTextProtocol(ProcessProtocol):
     and then .callback()s on self.done and .errback's if the process exits
     """
 
-    def __init__(self, magic_text):
+    def __init__(self, magic_text: str, name: str) -> None:
         self.magic_seen = Deferred()
+        self.name = f"{name}: "
         self.exited = Deferred()
         self._magic_text = magic_text
         self._output = StringIO()
@@ -140,7 +141,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def outReceived(self, data):
         data = str(data, sys.stdout.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
@@ -148,7 +149,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         data = str(data, sys.stderr.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
 
 
 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@@ -282,7 +283,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
     """
     if magic_text is None:
         magic_text = "client running"
-    protocol = _MagicTextProtocol(magic_text)
+    protocol = _MagicTextProtocol(magic_text, basename(node_dir))
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 86a513282f67556e1791faa7ab09ac4fc3f5b6b7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:36:39 -0400
Subject: [PATCH 70/94] Include Foolscap logging in node output in integration
 tests.

---
 integration/conftest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration/conftest.py b/integration/conftest.py
index 69d1934b0..cdc65f9e8 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -54,6 +54,10 @@ from .util import (
 # integration tests. See allmydata/scripts/common_http.py for usage.
 os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
 
+# Make Foolscap logging go into Twisted logging, so that integration test logs
+# include extra information
+# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst):
+os.environ["FLOGTOTWISTED"] = "1"
 
 # pytest customization hooks
 

From 9faf742b411e5dddd11e098eee7421b7154e3b25 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:36:59 -0400
Subject: [PATCH 71/94] News file.

---
 newsfragments/4018.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4018.minor

diff --git a/newsfragments/4018.minor b/newsfragments/4018.minor
new file mode 100644
index 000000000..e69de29bb

From 3d0c872f4c3751d48c5cf1f2392ec431b11235f8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:44:10 -0400
Subject: [PATCH 72/94] restrict CI jobs to the wheelhouse

---
 .circleci/setup-virtualenv.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index feccbbf23..7087c5120 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -26,12 +26,7 @@ shift || :
 
 # Tell pip where it can find any existing wheels.
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
-
-# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
-# between the time dependencies change and the images are re-built and (b) the
-# upcoming-deprecations job wants to install some dependencies from github and
-# it's awkward to get that done any earlier than the tox run.  So, we don't
-# set it.
+export PIP_NO_INDEX="1"
 
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \

From f9269158baf5103e014bbd439944690f3138c1af Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:46:58 -0400
Subject: [PATCH 73/94] news fragment

---
 newsfragments/4019.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4019.minor

diff --git a/newsfragments/4019.minor b/newsfragments/4019.minor
new file mode 100644
index 000000000..e69de29bb

From 4d5b9f2d0c88e411b0cb032a46b8b681d984703c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:48:46 -0400
Subject: [PATCH 74/94] match the version in the docker image

it is maybe wrong that we pin a specific version here and also only include a
specific version (probably some interpretation of "the most recent release")
in the docker image...
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 5f18b6b95..6e56496d4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,7 +36,7 @@ deps =
      # happening at the time.  The versions selected here are just the current
      # versions at the time.  Bumping them to keep up with future releases is
      # fine as long as those releases are known to actually work.
-     pip==22.0.3
+     pip==22.3.1
      setuptools==60.9.1
      wheel==0.37.1
      subunitreporter==22.2.0

From 58ccecff5414e7ceded6aaac3666e289aa54dd5b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:17:19 -0400
Subject: [PATCH 75/94] Take a step towards unifying dependency pins used by
 tox env and Docker image building

---
 .circleci/populate-wheelhouse.sh | 21 +++--------------
 setup.py                         | 39 +++++++++++++++++++++++++++++---
 tox.ini                          | 19 +---------------
 3 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 374ca0adb..f103a6af8 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -3,18 +3,6 @@
 # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
-# Basic Python packages that you just need to have around to do anything,
-# practically speaking.
-BASIC_DEPS="pip wheel"
-
-# Python packages we need to support the test infrastructure.  *Not* packages
-# Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0"
-
-# Python packages we need to generate test reports for CI infrastructure.
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
-
 # The filesystem location of the wheelhouse which we'll populate with wheels
 # for all of our dependencies.
 WHEELHOUSE_PATH="$1"
@@ -41,15 +29,12 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
-    "${PROJECT_ROOT}"[test] \
-    ${BASIC_DEPS} \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
+    "${PROJECT_ROOT}"[testenv] \
+    "${PROJECT_ROOT}"[test]
 
 # Not strictly wheelhouse population but ... Note we omit basic deps here.
 # They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
 # have to ask.
 "${PIP}" \
     install \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
+    "${PROJECT_ROOT}"[testenv]
diff --git a/setup.py b/setup.py
index 2418c6dbe..6e16381e6 100644
--- a/setup.py
+++ b/setup.py
@@ -398,10 +398,44 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "dulwich",
               "gpg",
           ],
+
+          # Here are the dependencies required to set up a reproducible test
+          # environment.  This could be for CI or local development.  These
+          # are *not* library dependencies of the test suite itself.  They are
+          # the tools we use to run the test suite at all.
+          "testenv": [
+              # Pin all of these versions for the same reason you ever want to
+              # pin anything: to prevent new releases with regressions from
+              # introducing spurious failures into CI runs for whatever
+              # development work is happening at the time.  The versions
+              # selected here are just the current versions at the time.
+              # Bumping them to keep up with future releases is fine as long
+              # as those releases are known to actually work.
+
+              # XXX For the moment, unpinned so we use whatever is in the
+              # image.  The images vary in what versions they have. :/
+              "pip", # ==22.0.3",
+              "wheel", # ==0.37.1"
+              "setuptools", # ==60.9.1",
+              "tox", # ~=3.0",
+              "subunitreporter", # ==22.2.0",
+              "python-subunit", # ==1.4.2",
+              "junitxml", # ==0.7",
+              "coverage", # ~= 5.0",
+
+              # As an exception, we don't pin certifi because it contains CA
+              # certificates which necessarily change over time.  Pinning this
+              # is guaranteed to cause things to break eventually as old
+              # certificates expire and as new ones are used in the wild that
+              # aren't present in whatever version we pin.  Hopefully there
+              # won't be functionality regressions in new releases of this
+              # package that cause us the kind of suffering we're trying to
+              # avoid with the above pins.
+              "certifi",
+          ],
+
           "test": [
-              "coverage ~= 5.0",
               "mock",
-              "tox ~= 3.0",
               "pytest",
               "pytest-twisted",
               "hypothesis >= 3.6.1",
@@ -410,7 +444,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "fixtures",
               "beautifulsoup4",
               "html5lib",
-              "junitxml",
               # Pin old version until
               # https://github.com/paramiko/paramiko/issues/1961 is fixed.
               "paramiko < 2.9",
diff --git a/tox.ini b/tox.ini
index 6e56496d4..3b7a96503 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,24 +30,7 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
 # available to those systems.  Installing it ahead of time (with pip) avoids
 # this problem.
 deps =
-     # Pin all of these versions for the same reason you ever want to pin
-     # anything: to prevent new releases with regressions from introducing
-     # spurious failures into CI runs for whatever development work is
-     # happening at the time.  The versions selected here are just the current
-     # versions at the time.  Bumping them to keep up with future releases is
-     # fine as long as those releases are known to actually work.
-     pip==22.3.1
-     setuptools==60.9.1
-     wheel==0.37.1
-     subunitreporter==22.2.0
-     # As an exception, we don't pin certifi because it contains CA
-     # certificates which necessarily change over time.  Pinning this is
-     # guaranteed to cause things to break eventually as old certificates
-     # expire and as new ones are used in the wild that aren't present in
-     # whatever version we pin.  Hopefully there won't be functionality
-     # regressions in new releases of this package that cause us the kind of
-     # suffering we're trying to avoid with the above pins.
-     certifi
+     .[testenv]
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.

From 66d3de059432a3e1d12b14b50b66ac2ea263929d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:31:26 -0400
Subject: [PATCH 76/94] narrowly pin these dependencies

This will break because these are not the versions on all Docker CI images but
we need to pin them to rebuild those images with the correct versions.

Rebuilding the images might break CI for all other branches.  But!  It's
broken already, so it's not like it's any worse.
---
 setup.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/setup.py b/setup.py
index 6e16381e6..127d17328 100644
--- a/setup.py
+++ b/setup.py
@@ -411,17 +411,14 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # selected here are just the current versions at the time.
               # Bumping them to keep up with future releases is fine as long
               # as those releases are known to actually work.
-
-              # XXX For the moment, unpinned so we use whatever is in the
-              # image.  The images vary in what versions they have. :/
-              "pip", # ==22.0.3",
-              "wheel", # ==0.37.1"
-              "setuptools", # ==60.9.1",
-              "tox", # ~=3.0",
-              "subunitreporter", # ==22.2.0",
-              "python-subunit", # ==1.4.2",
-              "junitxml", # ==0.7",
-              "coverage", # ~= 5.0",
+              "pip==22.0.3",
+              "wheel==0.37.1"
+              "setuptools==60.9.1",
+              "tox~=3.0",
+              "subunitreporter==22.2.0",
+              "python-subunit==1.4.2",
+              "junitxml==0.7",
+              "coverage ~= 5.0",
 
               # As an exception, we don't pin certifi because it contains CA
               # certificates which necessarily change over time.  Pinning this

From 29961a08b2c4097ee527043216a41b17a7da048d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:40:49 -0400
Subject: [PATCH 77/94] typo in the requirements list...

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 127d17328..b1d54b43c 100644
--- a/setup.py
+++ b/setup.py
@@ -412,7 +412,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # Bumping them to keep up with future releases is fine as long
               # as those releases are known to actually work.
               "pip==22.0.3",
-              "wheel==0.37.1"
+              "wheel==0.37.1",
               "setuptools==60.9.1",
               "tox~=3.0",
               "subunitreporter==22.2.0",

From 0f200e422e3278091843f6384fa91cfe5bf1101c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 15:48:49 -0400
Subject: [PATCH 78/94] Give it more time.

---
 integration/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index b1692a7a3..c58fc2e93 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
 
 
 @run_in_thread
-def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
+def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
     `TahoeProcess` instance usually from a fixture e.g. `alice`) to be

From f6e4e862a9d1bb8cc16d27b791e76aec09a298e6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:50:50 -0400
Subject: [PATCH 79/94] Require that the actual test run step do this part

Keep this script to wheelhouse population.  We might be giving up a tiny bit
of performance here but let's make it work at all before we make it fast.
---
 .circleci/populate-wheelhouse.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index f103a6af8..239c8367b 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -31,10 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
     "${PROJECT_ROOT}"[test]
-
-# Not strictly wheelhouse population but ... Note we omit basic deps here.
-# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
-# have to ask.
-"${PIP}" \
-    install \
-    "${PROJECT_ROOT}"[testenv]

From 70caa22370b9646096e83cb18057287a3946698f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:51:45 -0400
Subject: [PATCH 80/94] have to do certifi in tox.ini

by the time setup.py is being processed it is too late for certifi to help
---
 setup.py | 10 ----------
 tox.ini  | 22 +++++++++++++++-------
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/setup.py b/setup.py
index b1d54b43c..599eb5898 100644
--- a/setup.py
+++ b/setup.py
@@ -419,16 +419,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "python-subunit==1.4.2",
               "junitxml==0.7",
               "coverage ~= 5.0",
-
-              # As an exception, we don't pin certifi because it contains CA
-              # certificates which necessarily change over time.  Pinning this
-              # is guaranteed to cause things to break eventually as old
-              # certificates expire and as new ones are used in the wild that
-              # aren't present in whatever version we pin.  Hopefully there
-              # won't be functionality regressions in new releases of this
-              # package that cause us the kind of suffering we're trying to
-              # avoid with the above pins.
-              "certifi",
           ],
 
           "test": [
diff --git a/tox.ini b/tox.ini
index 3b7a96503..609a78b13 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,14 +23,22 @@ minversion = 2.4
 
 [testenv]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
-# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
-# a package to be installed (with setuptools) then it'll fail on certain
-# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
-# requirements (TLS >= 1.2) are incompatible with the old TLS clients
-# available to those systems.  Installing it ahead of time (with pip) avoids
-# this problem.
 deps =
-     .[testenv]
+    # We pull in certify *here* to avoid bug #2913. Basically if a
+    # `setup_requires=...` causes a package to be installed (with setuptools)
+    # then it'll fail on certain platforms (travis's OX-X 10.12, Slackware
+    # 14.2) because PyPI's TLS requirements (TLS >= 1.2) are incompatible with
+    # the old TLS clients available to those systems.  Installing it ahead of
+    # time (with pip) avoids this problem.
+    #
+    # We don't pin an exact version of it because it contains CA certificates
+    # which necessarily change over time.  Pinning this is guaranteed to cause
+    # things to break eventually as old certificates expire and as new ones
+    # are used in the wild that aren't present in whatever version we pin.
+    # Hopefully there won't be functionality regressions in new releases of
+    # this package that cause us the kind of suffering we're trying to avoid
+    # with the above pins.
+    certifi
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.

From 17706f582ee54532b7a117b3b97ffce3e0108b7a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:52:05 -0400
Subject: [PATCH 81/94] use tox testenv `extras` to request testenv too

---
 tox.ini | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 609a78b13..2edb15a0b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -43,9 +43,14 @@ deps =
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.
 usedevelop = False
-# We use extras=test to get things like "mock" that are required for our unit
-# tests.
-extras = test
+
+extras =
+       # Get general testing environment dependencies so we can run the tests
+       # how we like.
+       testenv
+
+       # And get all of the test suite's actual direct Python dependencies.
+       test
 
 setenv =
        # Define TEST_SUITE in the environment as an aid to constructing the

From f48eb81d9d741857b6fe6cbabab0e04942238036 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:57:51 -0400
Subject: [PATCH 82/94] restrict werkzeug more, at least for the moment

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 599eb5898..d823029e2 100644
--- a/setup.py
+++ b/setup.py
@@ -141,8 +141,10 @@ install_requires = [
 
     # HTTP server and client
     "klein",
+
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    "werkzeug != 2.2.0",
+    # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
+    "werkzeug != 2.2.0, != 2.3.0, != 2.3.1",
     "treq",
     "cbor2",
 

From 44cd746ce480c3a1641ebfe55350d15698625323 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 28 Apr 2023 11:43:26 -0400
Subject: [PATCH 83/94] Limit klein version for now.

---
 setup.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 2418c6dbe..751c7ebc3 100644
--- a/setup.py
+++ b/setup.py
@@ -141,8 +141,10 @@ install_requires = [
 
     # HTTP server and client
     "klein",
-    # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    "werkzeug != 2.2.0",
+    # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 and 2.3 is
+    # incompatible with klein 21.8 and earlier; see
+    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4020 for the latter.
+    "werkzeug != 2.2.0,<2.3",
     "treq",
     "cbor2",
 

From c15dd6c9f0fb9d43c9a17db523d6f726f67ee593 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 28 Apr 2023 11:43:48 -0400
Subject: [PATCH 84/94] This wasn't the issue.

---
 integration/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index c58fc2e93..b1692a7a3 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
 
 
 @run_in_thread
-def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1):
+def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
     `TahoeProcess` instance usually from a fixture e.g. `alice`) to be

From f0b98aead562495f7f2019dd886b4e61c5fe68f9 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:32:42 -0400
Subject: [PATCH 85/94] You don't need tox *inside* your test environment.

You need tox to *manage* your test environment (this is the premise, at least).
---
 .circleci/setup-virtualenv.sh | 4 ++++
 setup.py                      | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 7087c5120..7fc6dc528 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -28,6 +28,10 @@ shift || :
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 export PIP_NO_INDEX="1"
 
+# Get tox inside the bootstrap virtualenv since we use tox to manage the rest
+# of the environment.
+"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0"
+
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \
      -c "${PROJECT_ROOT}"/tox.ini \
diff --git a/setup.py b/setup.py
index d823029e2..bac93a4bb 100644
--- a/setup.py
+++ b/setup.py
@@ -416,7 +416,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "pip==22.0.3",
               "wheel==0.37.1",
               "setuptools==60.9.1",
-              "tox~=3.0",
               "subunitreporter==22.2.0",
               "python-subunit==1.4.2",
               "junitxml==0.7",

From d67016d1b99ed1750c01af0caa4c137a768f31ce Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:39:49 -0400
Subject: [PATCH 86/94] Get the right version of tox in the wheelhouse

---
 .circleci/populate-wheelhouse.sh | 3 ++-
 .circleci/setup-virtualenv.sh    | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 239c8367b..f7ce361a8 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -30,4 +30,5 @@ LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
-    "${PROJECT_ROOT}"[test]
+    "${PROJECT_ROOT}"[test] \
+    "tox~=3.0"
diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 7fc6dc528..3f0074da3 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -30,7 +30,7 @@ export PIP_NO_INDEX="1"
 
 # Get tox inside the bootstrap virtualenv since we use tox to manage the rest
 # of the environment.
-"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0"
+"${BOOTSTRAP_VENV}"/bin/pip install tox
 
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \

From a088b1d8125404b53f76cd7408d29df08cb9ab2a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:49:14 -0400
Subject: [PATCH 87/94] don't bother to make a wheel of tox, just install it

---
 .circleci/populate-wheelhouse.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index f7ce361a8..14d421652 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -30,5 +30,8 @@ LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
-    "${PROJECT_ROOT}"[test] \
-    "tox~=3.0"
+    "${PROJECT_ROOT}"[test]
+
+# Put tox right into the bootstrap environment because everyone is going to
+# need to use it.
+"${PIP}" install "tox~=3.0"

From 29c0ca59748507f95035de4aff1faaa65995102b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:51:22 -0400
Subject: [PATCH 88/94] put the tox installation near other software
 installation

---
 .circleci/create-virtualenv.sh   | 4 ++++
 .circleci/populate-wheelhouse.sh | 4 ----
 .circleci/setup-virtualenv.sh    | 4 ----
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh
index 810ce5ae2..7327d0859 100755
--- a/.circleci/create-virtualenv.sh
+++ b/.circleci/create-virtualenv.sh
@@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 # above, it may still not be able to get us a compatible version unless we
 # explicitly ask for one.
 "${PIP}" install --upgrade setuptools==44.0.0 wheel
+
+# Just about every user of this image wants to use tox from the bootstrap
+# virtualenv so go ahead and install it now.
+"${PIP}" install "tox~=3.0"
diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 14d421652..239c8367b 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -31,7 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
     "${PROJECT_ROOT}"[test]
-
-# Put tox right into the bootstrap environment because everyone is going to
-# need to use it.
-"${PIP}" install "tox~=3.0"
diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 3f0074da3..7087c5120 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -28,10 +28,6 @@ shift || :
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 export PIP_NO_INDEX="1"
 
-# Get tox inside the bootstrap virtualenv since we use tox to manage the rest
-# of the environment.
-"${BOOTSTRAP_VENV}"/bin/pip install tox
-
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \
      -c "${PROJECT_ROOT}"/tox.ini \

From 04ef5a02b2a27e7dc224b11f51615369d99f7e74 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 14:12:01 -0400
Subject: [PATCH 89/94] eh ... these things moved into the tox-managed venv

not intentional but not sure what a _good_ fix is, so try this.
---
 .circleci/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index 6d7a881fe..b1e45af9b 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -93,5 +93,5 @@ if [ -n "${ARTIFACTS}" ]; then
 
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+    "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

From fa034781b46f2d3ae5351a9c57c7d55825fdebfe Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 14:29:21 -0400
Subject: [PATCH 90/94] Perhaps this is the correct way to locate the
 tox-managed venv

---
 .circleci/run-tests.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index b1e45af9b..d897cc729 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -79,9 +79,10 @@ else
     alternative="false"
 fi
 
+WORKDIR=/tmp/tahoe-lafs.tox
 ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \
     -c ${PROJECT_ROOT}/tox.ini \
-    --workdir /tmp/tahoe-lafs.tox \
+    --workdir "${WORKDIR}" \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
     ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
 
@@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then
 
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+
+    "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

From 3c660aff5d23849cdc6cdd788a455a14f2bb7881 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 1 May 2023 09:19:01 -0400
Subject: [PATCH 91/94] a comment about the other test extra

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index bac93a4bb..49004b1ff 100644
--- a/setup.py
+++ b/setup.py
@@ -422,6 +422,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "coverage ~= 5.0",
           ],
 
+          # Here are the library dependencies of the test suite.
           "test": [
               "mock",
               "pytest",

From 0af84c9ac1fdf07cb644d381b693dfcafcdf594e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 1 May 2023 09:28:46 -0400
Subject: [PATCH 92/94] news fragment

---
 newsfragments/4020.installation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/4020.installation

diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation
new file mode 100644
index 000000000..8badf4b3c
--- /dev/null
+++ b/newsfragments/4020.installation
@@ -0,0 +1 @@
+werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein.

From b21b15f3954ae328fbaafd7d53839b62c4329d4b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:56:59 -0400
Subject: [PATCH 93/94] Blocking newer werkzeug is a temporary measure.

---
 newsfragments/4020.installation | 1 -
 newsfragments/4020.minor        | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 newsfragments/4020.installation
 create mode 100644 newsfragments/4020.minor

diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation
deleted file mode 100644
index 8badf4b3c..000000000
--- a/newsfragments/4020.installation
+++ /dev/null
@@ -1 +0,0 @@
-werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein.
diff --git a/newsfragments/4020.minor b/newsfragments/4020.minor
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/newsfragments/4020.minor
@@ -0,0 +1 @@
+

From 4ca056b51c3dcc9f13f424bf133bd9dc34de8d93 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:57:35 -0400
Subject: [PATCH 94/94] Be more general, 2.3.2 just came out for example.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 49004b1ff..4f28b4438 100644
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@ install_requires = [
 
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
     # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
-    "werkzeug != 2.2.0, != 2.3.0, != 2.3.1",
+    "werkzeug != 2.2.0, < 2.3",
     "treq",
     "cbor2",