From 2d34b6a998f3f5eb454e915415e94e1387a1ee06 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Apr 2022 16:29:00 -0400
Subject: [PATCH 001/362] Log the request and response in the server.

---
 src/allmydata/storage/http_server.py | 36 ++++++++++++++++++++++------
 1 file changed, 29 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 7c4860d57..935390d10 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -8,6 +8,7 @@ from functools import wraps
 from base64 import b64decode
 import binascii
 
+from eliot import start_action
 from zope.interface import implementer
 from klein import Klein
 from twisted.web import http
@@ -83,8 +84,9 @@ def _extract_secrets(
 
 def _authorization_decorator(required_secrets):
     """
-    Check the ``Authorization`` header, and extract ``X-Tahoe-Authorization``
-    headers and pass them in.
+    1. Check the ``Authorization`` header matches server swissnum.
+    2. Extract ``X-Tahoe-Authorization`` headers and pass them in.
+    3. Log the request and response.
     """
 
     def decorator(f):
@@ -106,7 +108,22 @@ def _authorization_decorator(required_secrets):
             except ClientSecretsException:
                 request.setResponseCode(http.BAD_REQUEST)
                 return b"Missing required secrets"
-            return f(self, request, secrets, *args, **kwargs)
+            with start_action(
+                action_type="allmydata:storage:http-server:request",
+                method=request.method,
+                path=request.path,
+            ) as ctx:
+                try:
+                    result = f(self, request, secrets, *args, **kwargs)
+                except _HTTPError as e:
+                    # This isn't an error necessarily for logging purposes,
+                    # it's an implementation detail, an easier way to set
+                    # response codes.
+                    ctx.add_success_fields(response_code=e.code)
+                    ctx.finish()
+                    raise
+                ctx.add_success_fields(response_code=request.code)
+                return result
 
         return route
 
@@ -239,19 +256,24 @@ class _HTTPError(Exception):
 # https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml. Notably, #6.258
 # indicates a set.
 _SCHEMAS = {
-    "allocate_buckets": Schema("""
+    "allocate_buckets": Schema(
+        """
     message = {
       share-numbers: #6.258([* uint])
       allocated-size: uint
     }
-    """),
-    "advise_corrupt_share": Schema("""
+    """
+    ),
+    "advise_corrupt_share": Schema(
+        """
     message = {
       reason: tstr
     }
-    """)
+    """
+    ),
 }
 
+
 class HTTPServer(object):
     """
     A HTTP interface to the storage server.

From 2722e81d2b4c8bffeebbab3be9654f1cb40736ad Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 26 Apr 2022 16:33:48 -0400
Subject: [PATCH 002/362] News file.

---
 newsfragments/3880.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3880.minor

diff --git a/newsfragments/3880.minor b/newsfragments/3880.minor
new file mode 100644
index 000000000..e69de29bb

From bd631665f4c5d66d16574dbab036c76529a16bca Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 4 May 2022 10:21:16 -0400
Subject: [PATCH 003/362] Add logging of HTTP requests from client.

---
 src/allmydata/storage/http_client.py | 88 +++++++++++++++-------------
 1 file changed, 48 insertions(+), 40 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index da350e0c6..b4bde6a95 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -4,6 +4,7 @@ HTTP client that talks to the HTTP storage server.
 
 from __future__ import annotations
 
+from eliot import start_action, register_exception_extractor
 from typing import Union, Optional, Sequence, Mapping
 from base64 import b64encode
 
@@ -55,6 +56,8 @@ class ClientException(Exception):
         Exception.__init__(self, code, *additional_args)
         self.code = code
 
+register_exception_extractor(ClientException, lambda e: {"response_code": e.code})
+
 
 # Schemas for server responses.
 #
@@ -280,6 +283,7 @@ class StorageClient(object):
         )
         return headers
 
+    @inlineCallbacks
     def request(
         self,
         method,
@@ -299,37 +303,40 @@ class StorageClient(object):
         If ``message_to_serialize`` is set, it will be serialized (by default
         with CBOR) and set as the request body.
         """
-        headers = self._get_headers(headers)
+        with start_action(action_type="allmydata:storage:http-client:request", method=method, url=str(url)) as ctx:
+            headers = self._get_headers(headers)
 
-        # Add secrets:
-        for secret, value in [
-            (Secrets.LEASE_RENEW, lease_renew_secret),
-            (Secrets.LEASE_CANCEL, lease_cancel_secret),
-            (Secrets.UPLOAD, upload_secret),
-            (Secrets.WRITE_ENABLER, write_enabler_secret),
-        ]:
-            if value is None:
-                continue
-            headers.addRawHeader(
-                "X-Tahoe-Authorization",
-                b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
-            )
-
-        # Note we can accept CBOR:
-        headers.addRawHeader("Accept", CBOR_MIME_TYPE)
-
-        # If there's a request message, serialize it and set the Content-Type
-        # header:
-        if message_to_serialize is not None:
-            if "data" in kwargs:
-                raise TypeError(
-                    "Can't use both `message_to_serialize` and `data` "
-                    "as keyword arguments at the same time"
+            # Add secrets:
+            for secret, value in [
+                (Secrets.LEASE_RENEW, lease_renew_secret),
+                (Secrets.LEASE_CANCEL, lease_cancel_secret),
+                (Secrets.UPLOAD, upload_secret),
+                (Secrets.WRITE_ENABLER, write_enabler_secret),
+            ]:
+                if value is None:
+                    continue
+                headers.addRawHeader(
+                    "X-Tahoe-Authorization",
+                    b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
                 )
-            kwargs["data"] = dumps(message_to_serialize)
-            headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
-        return self._treq.request(method, url, headers=headers, **kwargs)
+            # Note we can accept CBOR:
+            headers.addRawHeader("Accept", CBOR_MIME_TYPE)
+
+            # If there's a request message, serialize it and set the Content-Type
+            # header:
+            if message_to_serialize is not None:
+                if "data" in kwargs:
+                    raise TypeError(
+                        "Can't use both `message_to_serialize` and `data` "
+                        "as keyword arguments at the same time"
+                    )
+                kwargs["data"] = dumps(message_to_serialize)
+                headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
+
+            response = yield self._treq.request(method, url, headers=headers, **kwargs)
+            ctx.add_success_fields(response_code=response.code)
+            return response
 
 
 class StorageClientGeneral(object):
@@ -538,18 +545,19 @@ class StorageClientImmutables(object):
         """
         Return the set of shares for a given storage index.
         """
-        url = self._client.relative_url(
-            "/v1/immutable/{}/shares".format(_encode_si(storage_index))
-        )
-        response = yield self._client.request(
-            "GET",
-            url,
-        )
-        if response.code == http.OK:
-            body = yield _decode_cbor(response, _SCHEMAS["list_shares"])
-            returnValue(set(body))
-        else:
-            raise ClientException(response.code)
+        with start_action(action_type="allmydata:storage:http-client:immutable:list-shares", storage_index=storage_index) as ctx:
+            url = self._client.relative_url(
+                "/v1/immutable/{}/shares".format(_encode_si(storage_index))
+            )
+            response = yield self._client.request(
+                "GET",
+                url,
+            )
+            if response.code == http.OK:
+                body = yield _decode_cbor(response, _SCHEMAS["list_shares"])
+                return set(body)
+            else:
+                raise ClientException(response.code)
 
     @inlineCallbacks
     def add_or_renew_lease(

From 669296d5d68a7705698bcc2fe15fa56491948fdc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 26 Jan 2023 11:44:53 -0500
Subject: [PATCH 004/362] News file.

---
 newsfragments/3935.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3935.minor

diff --git a/newsfragments/3935.minor b/newsfragments/3935.minor
new file mode 100644
index 000000000..e69de29bb

From badba97ff20a961153ddd86490c9646e042df172 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Fri, 17 Feb 2023 16:20:29 -0600
Subject: [PATCH 005/362] Type annotations added for wormholetesting.py

---
 src/allmydata/test/cli/wormholetesting.py | 35 +++++++++++------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 744f9d75a..7cf9d7eff 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -32,8 +32,7 @@ For example::
 
 from __future__ import annotations
 
-from typing import Iterator, Optional, List, Tuple
-from collections.abc import Awaitable
+from typing import Iterator, Optional, List, Tuple, Any, TextIO
 from inspect import getargspec
 from itertools import count
 from sys import stderr
@@ -66,18 +65,18 @@ class MemoryWormholeServer(object):
 
     def create(
         self,
-        appid,
-        relay_url,
-        reactor,
-        versions={},
-        delegate=None,
-        journal=None,
-        tor=None,
-        timing=None,
-        stderr=stderr,
-        _eventual_queue=None,
-        _enable_dilate=False,
-    ):
+        appid: str,
+        relay_url: str,
+        reactor: Any,
+        versions: Any={},
+        delegate: Optional[Any]=None,
+        journal: Optional[Any]=None,
+        tor: Optional[Any]=None,
+        timing: Optional[Any]=None,
+        stderr: TextIO=stderr,
+        _eventual_queue: Optional[Any]=None,
+        _enable_dilate: bool=False,
+    )-> _MemoryWormhole:
         """
         Create a wormhole.  It will be able to connect to other wormholes created
         by this instance (and constrained by the normal appid/relay_url
@@ -184,7 +183,7 @@ class _WormholeApp(object):
 
         return code
 
-    def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]:
+    def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]:
         """
         Return a ``Deferred`` which fires with the next wormhole to be associated
         with the given code.  This is used to let the first end of a wormhole
@@ -262,7 +261,7 @@ class _MemoryWormhole(object):
             return d
         return succeed(self._code)
 
-    def get_welcome(self):
+    def get_welcome(self) -> Deferred[str]:
         return succeed("welcome")
 
     def send_message(self, payload: WormholeMessage) -> None:
@@ -276,8 +275,8 @@ class _MemoryWormhole(object):
             )
         d = self._view.wormhole_by_code(self._code, exclude=self)
 
-        def got_wormhole(wormhole):
-            msg = wormhole._payload.get()
+        def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[_MemoryWormhole]:
+            msg: Deferred[_MemoryWormhole] = wormhole._payload.get()
             return msg
 
         d.addCallback(got_wormhole)

From 86dbcb21ce4f27e779b3d8febc633c1cbf3fd97e Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Fri, 17 Feb 2023 16:24:32 -0600
Subject: [PATCH 006/362] Refactored verify function to update deprecated
 getargspec function with getfullargspec and maintained strictness

---
 src/allmydata/test/cli/wormholetesting.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 7cf9d7eff..9ce199545 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -33,7 +33,7 @@ For example::
 from __future__ import annotations
 
 from typing import Iterator, Optional, List, Tuple, Any, TextIO
-from inspect import getargspec
+from inspect import getfullargspec
 from itertools import count
 from sys import stderr
 
@@ -133,18 +133,24 @@ class TestingHelper(object):
         return wormhole
 
 
-def _verify():
+def _verify() -> None:
     """
     Roughly confirm that the in-memory wormhole creation function matches the
     interface of the real implementation.
     """
     # Poor man's interface verification.
 
-    a = getargspec(create)
-    b = getargspec(MemoryWormholeServer.create)
+    a = getfullargspec(create)
+    b = getfullargspec(MemoryWormholeServer.create)
+
     # I know it has a `self` argument at the beginning.  That's okay.
     b = b._replace(args=b.args[1:])
-    assert a == b, "{} != {}".format(a, b)
+
+    # Just compare the same information to check function signature
+    assert a.varkw == b.varkw
+    assert a.args == b.args
+    assert a.varargs == b.varargs
+    assert a.kwonlydefaults == b.kwonlydefaults
 
 
 _verify()

From be9d76e2b8cffda206afe066fe00be2db8dd6759 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Fri, 17 Feb 2023 16:24:52 -0600
Subject: [PATCH 007/362] Added newsfragment

---
 newsfragments/3970.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3970.minor

diff --git a/newsfragments/3970.minor b/newsfragments/3970.minor
new file mode 100644
index 000000000..e69de29bb

From 95bb7afba7c0c883ff7c9099dc9cd839daef3c6c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Feb 2023 10:42:06 -0500
Subject: [PATCH 008/362] Sketch of happy eyeballs.

---
 src/allmydata/storage_client.py | 32 ++++++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 8e9ad3656..8191014e8 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -47,7 +47,7 @@ from zope.interface import (
 )
 from twisted.python.failure import Failure
 from twisted.web import http
-from twisted.internet.task import LoopingCall
+from twisted.internet.task import LoopingCall, deferLater
 from twisted.internet import defer, reactor
 from twisted.application import service
 from twisted.plugin import (
@@ -935,6 +935,21 @@ class NativeStorageServer(service.MultiService):
         self._reconnector.reset()
 
 
+async def _pick_a_http_server(reactor, nurls: list[DecodedURL]) -> DecodedURL:
+    """Pick the first server we successfully talk to."""
+    while True:
+        try:
+            _, index = await defer.DeferredList([
+                StorageClientGeneral(
+                    StorageClient.from_nurl(nurl, reactor)
+                ).get_version() for nurl in nurls
+            ], consumeErrors=True, fireOnOneCallback=True)
+            return nurls[index]
+        except Exception as e:
+            log.err(e, "Failed to connect to any of the HTTP NURLs for server")
+            await deferLater(reactor, 1, lambda: None)
+
+
 @implementer(IServer)
 class HTTPNativeStorageServer(service.MultiService):
     """
@@ -962,10 +977,8 @@ class HTTPNativeStorageServer(service.MultiService):
         ) = _parse_announcement(server_id, furl, announcement)
         # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs?
         # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935
-        nurl = DecodedURL.from_text(announcement[ANONYMOUS_STORAGE_NURLS][0])
-        self._istorage_server = _HTTPStorageServer.from_http_client(
-            StorageClient.from_nurl(nurl, reactor)
-        )
+        self._nurls = [DecodedURL.from_text(u) for u in announcement[ANONYMOUS_STORAGE_NURLS]]
+        self._istorage_server = None
 
         self._connection_status = connection_status.ConnectionStatus.unstarted()
         self._version = None
@@ -1033,7 +1046,14 @@ class HTTPNativeStorageServer(service.MultiService):
         version = self.get_version()
         return _available_space_from_version(version)
 
-    def start_connecting(self, trigger_cb):
+    @async_to_deferred
+    async def start_connecting(self, trigger_cb):
+        # The problem with this scheme is that while picking the HTTP server to
+        # talk to, we don't have connection status updates...
+        nurl = await _pick_a_http_server(reactor, self._nurls)
+        self._istorage_server = _HTTPStorageServer.from_http_client(
+            StorageClient.from_nurl(nurl, reactor)
+        )
         self._lc = LoopingCall(self._connect)
         self._lc.start(1, True)
 

From 2ac6580c269268e891a1f79f23a81f0d56917512 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Feb 2023 10:56:38 -0500
Subject: [PATCH 009/362] Welcome to the world of tomorrow.

---
 src/allmydata/test/test_storage_client.py | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 1a84f35ec..0657a814e 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -1,16 +1,8 @@
 """
-Ported from Python 3.
+Tests for allmydata.storage_client.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
 
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-from six import ensure_text
+from __future__ import annotations
 
 from json import (
     loads,
@@ -475,7 +467,7 @@ class StoragePluginWebPresence(AsyncTestCase):
                 # config validation policy).
                 "tub.port": tubport_endpoint,
                 "tub.location": tubport_location,
-                "web.port": ensure_text(webport_endpoint),
+                "web.port": str(webport_endpoint),
             },
             storage_plugin=self.storage_plugin,
             basedir=self.basedir,

From 32768e310ae5316cc2e6fc0f0f969368c1ff3eee Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Feb 2023 11:30:47 -0500
Subject: [PATCH 010/362] Unit test for _pick_a_http_server.

---
 src/allmydata/storage_client.py           | 48 ++++++++++-------
 src/allmydata/test/test_storage_client.py | 64 ++++++++++++++++++++++-
 2 files changed, 93 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 8191014e8..436335431 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -33,7 +33,7 @@ Ported to Python 3.
 from __future__ import annotations
 
 from six import ensure_text
-from typing import Union
+from typing import Union, Callable, Any
 import re, time, hashlib
 from os import urandom
 from configparser import NoSectionError
@@ -935,19 +935,25 @@ class NativeStorageServer(service.MultiService):
         self._reconnector.reset()
 
 
-async def _pick_a_http_server(reactor, nurls: list[DecodedURL]) -> DecodedURL:
-    """Pick the first server we successfully talk to."""
+async def _pick_a_http_server(
+        reactor,
+        nurls: list[DecodedURL],
+        request: Callable[[DecodedURL, Any], defer.Deferred[Any]]
+) -> DecodedURL:
+    """Pick the first server we successfully send a request to."""
     while True:
-        try:
-            _, index = await defer.DeferredList([
-                StorageClientGeneral(
-                    StorageClient.from_nurl(nurl, reactor)
-                ).get_version() for nurl in nurls
-            ], consumeErrors=True, fireOnOneCallback=True)
+        result = await defer.DeferredList([
+                request(reactor, nurl) for nurl in nurls
+        ], consumeErrors=True, fireOnOneCallback=True)
+        # Apparently DeferredList is an awful awful API. If everything fails,
+        # you get back a list of (False, Failure), if it succeeds, you get a
+        # tuple of (value, index).
+        if isinstance(result, list):
+             await deferLater(reactor, 1, lambda: None)
+        else:
+            assert isinstance(result, tuple)
+            _, index = result
             return nurls[index]
-        except Exception as e:
-            log.err(e, "Failed to connect to any of the HTTP NURLs for server")
-            await deferLater(reactor, 1, lambda: None)
 
 
 @implementer(IServer)
@@ -975,9 +981,10 @@ class HTTPNativeStorageServer(service.MultiService):
             self._short_description,
             self._long_description
         ) = _parse_announcement(server_id, furl, announcement)
-        # TODO need some way to do equivalent of Happy Eyeballs for multiple NURLs?
-        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3935
-        self._nurls = [DecodedURL.from_text(u) for u in announcement[ANONYMOUS_STORAGE_NURLS]]
+        self._nurls = [
+            DecodedURL.from_text(u)
+            for u in announcement[ANONYMOUS_STORAGE_NURLS]
+        ]
         self._istorage_server = None
 
         self._connection_status = connection_status.ConnectionStatus.unstarted()
@@ -1048,9 +1055,14 @@ class HTTPNativeStorageServer(service.MultiService):
 
     @async_to_deferred
     async def start_connecting(self, trigger_cb):
-        # The problem with this scheme is that while picking the HTTP server to
-        # talk to, we don't have connection status updates...
-        nurl = await _pick_a_http_server(reactor, self._nurls)
+        # TODO file a bug: The problem with this scheme is that while picking
+        # the HTTP server to talk to, we don't have connection status
+        # updates...
+        def request(reactor, nurl: DecodedURL):
+            return StorageClientGeneral(
+                StorageClient.from_nurl(nurl, reactor)
+            ).get_version()
+        nurl = await _pick_a_http_server(reactor, self._nurls, request)
         self._istorage_server = _HTTPStorageServer.from_http_client(
             StorageClient.from_nurl(nurl, reactor)
         )
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 0657a814e..38ef8c1d3 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -7,8 +7,10 @@ from __future__ import annotations
 from json import (
     loads,
 )
-
 import hashlib
+from typing import Union, Any
+
+from hyperlink import DecodedURL
 from fixtures import (
     TempDir,
 )
@@ -52,6 +54,7 @@ from twisted.internet.defer import (
 from twisted.python.filepath import (
     FilePath,
 )
+from twisted.internet.task import Clock
 
 from foolscap.api import (
     Tub,
@@ -80,12 +83,14 @@ from allmydata.webish import (
     WebishServer,
 )
 from allmydata.util import base32, yamlutil
+from allmydata.util.deferredutil import async_to_deferred
 from allmydata.storage_client import (
     IFoolscapStorageServer,
     NativeStorageServer,
     StorageFarmBroker,
     _FoolscapStorage,
     _NullStorage,
+    _pick_a_http_server
 )
 from ..storage.server import (
     StorageServer,
@@ -731,3 +736,60 @@ storage:
 
         yield done
         self.assertTrue(done.called)
+
+
+class PickHTTPServerTests(unittest.SynchronousTestCase):
+    """Tests for ``_pick_a_http_server``."""
+
+    def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> Deferred[DecodedURL]:
+        """
+        Given mapping of URLs to list of (delay, result), return the URL of the
+        first selected server.
+        """
+        clock = Clock()
+
+        def request(reactor, url):
+            delay, value = url_to_results[url].pop(0)
+            result = Deferred()
+            def add_result_value():
+                if isinstance(value, Exception):
+                    result.errback(value)
+                else:
+                    result.callback(value)
+            reactor.callLater(delay, add_result_value)
+            return result
+
+        d = async_to_deferred(_pick_a_http_server)(
+            clock, list(url_to_results.keys()), request
+        )
+        for i in range(1000):
+            clock.advance(0.1)
+        return d
+
+    def test_first_successful_connect_is_picked(self):
+        """
+        Given multiple good URLs, the first one that connects is chosen.
+        """
+        earliest_url = DecodedURL.from_text("http://a")
+        latest_url = DecodedURL.from_text("http://b")
+        d = self.loop_until_result({
+            latest_url: [(2, None)],
+            earliest_url: [(1, None)]
+        })
+        self.assertEqual(self.successResultOf(d), earliest_url)
+
+    def test_failures_are_retried(self):
+        """
+        If the initial requests all fail, ``_pick_a_http_server`` keeps trying
+        until success.
+        """
+        eventually_good_url = DecodedURL.from_text("http://good")
+        bad_url = DecodedURL.from_text("http://bad")
+        d = self.loop_until_result({
+            eventually_good_url: [
+                (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None)
+            ],
+            bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())]
+        })
+        self.flushLoggedErrors(ZeroDivisionError, RuntimeError)
+        self.assertEqual(self.successResultOf(d), eventually_good_url)

From 74e77685a35905bc9bdd0609ac73309a9da7f10d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 10:07:57 -0500
Subject: [PATCH 011/362] Get rid of DeferredList.

---
 src/allmydata/storage_client.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 436335431..0d2443d64 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -942,18 +942,31 @@ async def _pick_a_http_server(
 ) -> DecodedURL:
     """Pick the first server we successfully send a request to."""
     while True:
-        result = await defer.DeferredList([
-                request(reactor, nurl) for nurl in nurls
-        ], consumeErrors=True, fireOnOneCallback=True)
-        # Apparently DeferredList is an awful awful API. If everything fails,
-        # you get back a list of (False, Failure), if it succeeds, you get a
-        # tuple of (value, index).
-        if isinstance(result, list):
-             await deferLater(reactor, 1, lambda: None)
+        result : defer.Deferred[Union[DecodedURL, None]] = defer.Deferred()
+
+        def succeeded(nurl: DecodedURL, result=result):
+            # Only need the first successful NURL:
+            if result.called:
+                return
+            result.callback(nurl)
+
+        def failed(failure, failures=[], result=result):
+            log.err(failure)
+            failures.append(None)
+            if len(failures) == len(nurls):
+                # All our potential NURLs failed...
+                result.callback(None)
+
+        for index, nurl in enumerate(nurls):
+            request(reactor, nurl).addCallback(
+                lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed)
+
+        first_nurl = await result
+        if first_nurl is None:
+            # Failed to connect to any of the NURLs:
+            await deferLater(reactor, 1, lambda: None)
         else:
-            assert isinstance(result, tuple)
-            _, index = result
-            return nurls[index]
+            return first_nurl
 
 
 @implementer(IServer)

From f41f4a5e0cc1fde83e7f184b6dd9c08775994d4b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 10:10:25 -0500
Subject: [PATCH 012/362] Correct type.

---
 src/allmydata/storage_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 0d2443d64..ac2a3cf5e 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -938,7 +938,7 @@ class NativeStorageServer(service.MultiService):
 async def _pick_a_http_server(
         reactor,
         nurls: list[DecodedURL],
-        request: Callable[[DecodedURL, Any], defer.Deferred[Any]]
+        request: Callable[[Any, DecodedURL], defer.Deferred[Any]]
 ) -> DecodedURL:
     """Pick the first server we successfully send a request to."""
     while True:
@@ -951,7 +951,7 @@ async def _pick_a_http_server(
             result.callback(nurl)
 
         def failed(failure, failures=[], result=result):
-            log.err(failure)
+            log.err(failure, "Failed to connect to NURL")
             failures.append(None)
             if len(failures) == len(nurls):
                 # All our potential NURLs failed...

From 99de5fa54c95331086d4ad92a6cc46b1823ccec7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 10:12:25 -0500
Subject: [PATCH 013/362] Link to follow-up ticket.

---
 src/allmydata/storage_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index ac2a3cf5e..420be8461 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1068,9 +1068,9 @@ class HTTPNativeStorageServer(service.MultiService):
 
     @async_to_deferred
     async def start_connecting(self, trigger_cb):
-        # TODO file a bug: The problem with this scheme is that while picking
+        # TODO The problem with this scheme is that while picking
         # the HTTP server to talk to, we don't have connection status
-        # updates...
+        # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
         def request(reactor, nurl: DecodedURL):
             return StorageClientGeneral(
                 StorageClient.from_nurl(nurl, reactor)

From b6e20dfa812be98ffb569aa023f1081cf83938af Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 13:26:30 -0500
Subject: [PATCH 014/362] Slightly longer timeout.

---
 src/allmydata/storage_client.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 420be8461..93c890c56 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -963,8 +963,9 @@ async def _pick_a_http_server(
 
         first_nurl = await result
         if first_nurl is None:
-            # Failed to connect to any of the NURLs:
-            await deferLater(reactor, 1, lambda: None)
+            # Failed to connect to any of the NURLs, try again in a few
+            # seconds:
+            await deferLater(reactor, 5, lambda: None)
         else:
             return first_nurl
 

From b95a1d2b79a465a70a65e08511221cfe80bc9dc0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 13:27:41 -0500
Subject: [PATCH 015/362] Nicer type annotations.

---
 src/allmydata/storage_client.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 93c890c56..c2468c679 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -33,7 +33,7 @@ Ported to Python 3.
 from __future__ import annotations
 
 from six import ensure_text
-from typing import Union, Callable, Any
+from typing import Union, Callable, Any, Optional
 import re, time, hashlib
 from os import urandom
 from configparser import NoSectionError
@@ -942,7 +942,7 @@ async def _pick_a_http_server(
 ) -> DecodedURL:
     """Pick the first server we successfully send a request to."""
     while True:
-        result : defer.Deferred[Union[DecodedURL, None]] = defer.Deferred()
+        result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred()
 
         def succeeded(nurl: DecodedURL, result=result):
             # Only need the first successful NURL:
@@ -1300,7 +1300,7 @@ class _HTTPBucketWriter(object):
         return self.finished
 
 
-def _ignore_404(failure: Failure) -> Union[Failure, None]:
+def _ignore_404(failure: Failure) -> Optional[Failure]:
     """
     Useful for advise_corrupt_share(), since it swallows unknown share numbers
     in Foolscap.

From 96e1e9ffac0f2b60d1a4db14761939ad96d9467e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 23 Feb 2023 19:45:01 -0500
Subject: [PATCH 016/362] Move where choosing a NURL happens.

---
 src/allmydata/storage_client.py | 36 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index c2468c679..53131c88a 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1067,19 +1067,7 @@ class HTTPNativeStorageServer(service.MultiService):
         version = self.get_version()
         return _available_space_from_version(version)
 
-    @async_to_deferred
-    async def start_connecting(self, trigger_cb):
-        # TODO The problem with this scheme is that while picking
-        # the HTTP server to talk to, we don't have connection status
-        # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
-        def request(reactor, nurl: DecodedURL):
-            return StorageClientGeneral(
-                StorageClient.from_nurl(nurl, reactor)
-            ).get_version()
-        nurl = await _pick_a_http_server(reactor, self._nurls, request)
-        self._istorage_server = _HTTPStorageServer.from_http_client(
-            StorageClient.from_nurl(nurl, reactor)
-        )
+    def start_connecting(self, trigger_cb):
         self._lc = LoopingCall(self._connect)
         self._lc.start(1, True)
 
@@ -1113,7 +1101,24 @@ class HTTPNativeStorageServer(service.MultiService):
     def try_to_connect(self):
         self._connect()
 
-    def _connect(self):
+    @async_to_deferred
+    async def _connect(self):
+        if self._istorage_server is None:
+            # We haven't selected a server yet, so let's do so.
+
+            # TODO The problem with this scheme is that while picking
+            # the HTTP server to talk to, we don't have connection status
+            # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
+            def request(reactor, nurl: DecodedURL):
+                return StorageClientGeneral(
+                    StorageClient.from_nurl(nurl, reactor)
+                ).get_version()
+
+            nurl = await _pick_a_http_server(reactor, self._nurls, request)
+            self._istorage_server = _HTTPStorageServer.from_http_client(
+                StorageClient.from_nurl(nurl, reactor)
+            )
+
         result = self._istorage_server.get_version()
 
         def remove_connecting_deferred(result):
@@ -1127,6 +1132,9 @@ class HTTPNativeStorageServer(service.MultiService):
             self._failed_to_connect
         )
 
+        # TODO Make sure LoopingCall waits for the above timeout for looping again:
+        #return self._connecting_deferred
+
     def stopService(self):
         if self._connecting_deferred is not None:
             self._connecting_deferred.cancel()

From e09d19463dfffc9f6b68ff99593fb96f2bdf5233 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Feb 2023 09:53:28 -0500
Subject: [PATCH 017/362] Logging errors breaks some tests.

---
 src/allmydata/storage_client.py           | 5 ++++-
 src/allmydata/test/test_storage_client.py | 1 -
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 53131c88a..a2726fe09 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -951,7 +951,10 @@ async def _pick_a_http_server(
             result.callback(nurl)
 
         def failed(failure, failures=[], result=result):
-            log.err(failure, "Failed to connect to NURL")
+            # Logging errors breaks a bunch of tests, and it's not a _bug_ to
+            # have a failed connection, it's often expected and transient. More
+            # of a warning, really?
+            log.msg("Failed to connect to NURL: {}".format(failure))
             failures.append(None)
             if len(failures) == len(nurls):
                 # All our potential NURLs failed...
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 38ef8c1d3..d7420b62f 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -791,5 +791,4 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
             ],
             bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())]
         })
-        self.flushLoggedErrors(ZeroDivisionError, RuntimeError)
         self.assertEqual(self.successResultOf(d), eventually_good_url)

From 3d0b17bc1c197a73b212b6b5eac0ad3b3ee43297 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Feb 2023 11:37:18 -0500
Subject: [PATCH 018/362] Make cancellation more likely to happen.

---
 src/allmydata/storage_client.py           | 88 ++++++++++++++---------
 src/allmydata/test/test_storage_client.py | 28 ++++----
 2 files changed, 72 insertions(+), 44 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index a2726fe09..549062d63 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -47,7 +47,7 @@ from zope.interface import (
 )
 from twisted.python.failure import Failure
 from twisted.web import http
-from twisted.internet.task import LoopingCall, deferLater
+from twisted.internet.task import LoopingCall
 from twisted.internet import defer, reactor
 from twisted.application import service
 from twisted.plugin import (
@@ -935,42 +935,52 @@ class NativeStorageServer(service.MultiService):
         self._reconnector.reset()
 
 
-async def _pick_a_http_server(
+def _pick_a_http_server(
         reactor,
         nurls: list[DecodedURL],
         request: Callable[[Any, DecodedURL], defer.Deferred[Any]]
-) -> DecodedURL:
-    """Pick the first server we successfully send a request to."""
-    while True:
-        result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred()
+) -> defer.Deferred[Optional[DecodedURL]]:
+    """Pick the first server we successfully send a request to.
 
-        def succeeded(nurl: DecodedURL, result=result):
-            # Only need the first successful NURL:
-            if result.called:
-                return
-            result.callback(nurl)
+    Fires with ``None`` if no server was found, or with the ``DecodedURL`` of
+    the first successfully-connected server.
+    """
 
-        def failed(failure, failures=[], result=result):
-            # Logging errors breaks a bunch of tests, and it's not a _bug_ to
-            # have a failed connection, it's often expected and transient. More
-            # of a warning, really?
-            log.msg("Failed to connect to NURL: {}".format(failure))
-            failures.append(None)
-            if len(failures) == len(nurls):
-                # All our potential NURLs failed...
-                result.callback(None)
+    to_cancel : list[defer.Deferred] = []
 
-        for index, nurl in enumerate(nurls):
-            request(reactor, nurl).addCallback(
-                lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed)
+    def cancel(result: Optional[defer.Deferred]):
+        for d in to_cancel:
+            if not d.called:
+                d.cancel()
+        if result is not None:
+            result.errback(defer.CancelledError())
 
-        first_nurl = await result
-        if first_nurl is None:
-            # Failed to connect to any of the NURLs, try again in a few
-            # seconds:
-            await deferLater(reactor, 5, lambda: None)
-        else:
-            return first_nurl
+    result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred(canceller=cancel)
+
+    def succeeded(nurl: DecodedURL, result=result):
+        # Only need the first successful NURL:
+        if result.called:
+            return
+        result.callback(nurl)
+        # No point in continuing other requests if we're connected:
+        cancel(None)
+
+    def failed(failure, failures=[], result=result):
+        # Logging errors breaks a bunch of tests, and it's not a _bug_ to
+        # have a failed connection, it's often expected and transient. More
+        # of a warning, really?
+        log.msg("Failed to connect to NURL: {}".format(failure))
+        failures.append(None)
+        if len(failures) == len(nurls):
+            # All our potential NURLs failed...
+            result.callback(None)
+
+    for index, nurl in enumerate(nurls):
+        d = request(reactor, nurl)
+        to_cancel.append(d)
+        d.addCallback(lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed)
+
+    return result
 
 
 @implementer(IServer)
@@ -1117,8 +1127,22 @@ class HTTPNativeStorageServer(service.MultiService):
                     StorageClient.from_nurl(nurl, reactor)
                 ).get_version()
 
-            nurl = await _pick_a_http_server(reactor, self._nurls, request)
-            self._istorage_server = _HTTPStorageServer.from_http_client(
+            # LoopingCall.stop() doesn't cancel Deferreds, unfortunately:
+            # https://github.com/twisted/twisted/issues/11814 Thus we want
+            # store the Deferred so it gets cancelled.
+            picking = _pick_a_http_server(reactor, self._nurls, request)
+            self._connecting_deferred = picking
+            try:
+                nurl = await picking
+            finally:
+                self._connecting_deferred = None
+
+            if nurl is None:
+                # We failed to find a server to connect to. Perhaps the next
+                # iteration of the loop will succeed.
+                return
+            else:
+                self._istorage_server = _HTTPStorageServer.from_http_client(
                 StorageClient.from_nurl(nurl, reactor)
             )
 
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index d7420b62f..a51e44a82 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -83,7 +83,6 @@ from allmydata.webish import (
     WebishServer,
 )
 from allmydata.util import base32, yamlutil
-from allmydata.util.deferredutil import async_to_deferred
 from allmydata.storage_client import (
     IFoolscapStorageServer,
     NativeStorageServer,
@@ -741,7 +740,7 @@ storage:
 class PickHTTPServerTests(unittest.SynchronousTestCase):
     """Tests for ``_pick_a_http_server``."""
 
-    def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> Deferred[DecodedURL]:
+    def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> tuple[int, DecodedURL]:
         """
         Given mapping of URLs to list of (delay, result), return the URL of the
         first selected server.
@@ -759,12 +758,15 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
             reactor.callLater(delay, add_result_value)
             return result
 
-        d = async_to_deferred(_pick_a_http_server)(
-            clock, list(url_to_results.keys()), request
-        )
-        for i in range(1000):
-            clock.advance(0.1)
-        return d
+        iterations = 0
+        while True:
+            iterations += 1
+            d = _pick_a_http_server(clock, list(url_to_results.keys()), request)
+            for i in range(100):
+                clock.advance(0.1)
+            result = self.successResultOf(d)
+            if result is not None:
+                return iterations, result
 
     def test_first_successful_connect_is_picked(self):
         """
@@ -772,11 +774,12 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
         """
         earliest_url = DecodedURL.from_text("http://a")
         latest_url = DecodedURL.from_text("http://b")
-        d = self.loop_until_result({
+        iterations, result = self.loop_until_result({
             latest_url: [(2, None)],
             earliest_url: [(1, None)]
         })
-        self.assertEqual(self.successResultOf(d), earliest_url)
+        self.assertEqual(iterations, 1)
+        self.assertEqual(result, earliest_url)
 
     def test_failures_are_retried(self):
         """
@@ -785,10 +788,11 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
         """
         eventually_good_url = DecodedURL.from_text("http://good")
         bad_url = DecodedURL.from_text("http://bad")
-        d = self.loop_until_result({
+        iterations, result = self.loop_until_result({
             eventually_good_url: [
                 (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None)
             ],
             bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())]
         })
-        self.assertEqual(self.successResultOf(d), eventually_good_url)
+        self.assertEqual(iterations, 3)
+        self.assertEqual(result, eventually_good_url)

From af51b022284f2b7284a806de7beb2223f5ad9961 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 27 Feb 2023 15:05:52 -0600
Subject: [PATCH 019/362] Revert wait_for_wormhole function return type back to
 Awaitable for forward compatibility when we move to async def

---
 src/allmydata/test/cli/wormholetesting.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 9ce199545..b30b92fe1 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -38,7 +38,7 @@ from itertools import count
 from sys import stderr
 
 from attrs import frozen, define, field, Factory
-from twisted.internet.defer import Deferred, DeferredQueue, succeed
+from twisted.internet.defer import Deferred, DeferredQueue, succeed, Awaitable
 from wormhole._interfaces import IWormhole
 from wormhole.wormhole import create
 from zope.interface import implementer
@@ -189,7 +189,7 @@ class _WormholeApp(object):
 
         return code
 
-    def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]:
+    def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]:
         """
         Return a ``Deferred`` which fires with the next wormhole to be associated
         with the given code.  This is used to let the first end of a wormhole
@@ -281,8 +281,8 @@ class _MemoryWormhole(object):
             )
         d = self._view.wormhole_by_code(self._code, exclude=self)
 
-        def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[_MemoryWormhole]:
-            msg: Deferred[_MemoryWormhole] = wormhole._payload.get()
+        def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[WormholeMessage]:
+            msg: Deferred[WormholeMessage] = wormhole._payload.get()
             return msg
 
         d.addCallback(got_wormhole)

From 582876197a724dd8c24b06160345d122832e03b6 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 27 Feb 2023 15:14:58 -0600
Subject: [PATCH 020/362] Added default check to verify to ensure strictness

---
 src/allmydata/test/cli/wormholetesting.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index b30b92fe1..d4e53a342 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -151,6 +151,7 @@ def _verify() -> None:
     assert a.args == b.args
     assert a.varargs == b.varargs
     assert a.kwonlydefaults == b.kwonlydefaults
+    assert a.defaults == b.defaults
 
 
 _verify()

From e9c3a227a17f3c23efbc9ac8ee0907101c0e8f62 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 14:17:25 -0500
Subject: [PATCH 021/362] File follow-up ticket.

---
 src/allmydata/storage_client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 388f8b4b8..686ee8e59 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1248,7 +1248,8 @@ class HTTPNativeStorageServer(service.MultiService):
         )
 
         # TODO Make sure LoopingCall waits for the above timeout for looping again:
-        #return self._connecting_deferred
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3981
+        #return self._connecting_deferred or maye await it
 
     def stopService(self):
         if self._connecting_deferred is not None:

From 75da037d673a3db8db50a9472758f89960591f2d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 14:25:04 -0500
Subject: [PATCH 022/362] Add race() implementation from
 https://github.com/twisted/twisted/pull/11818

---
 src/allmydata/test/test_deferredutil.py | 160 ++++++++++++++++++++++--
 src/allmydata/util/deferredutil.py      | 100 ++++++++++++++-
 2 files changed, 247 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py
index a37dfdd6f..47121b4cb 100644
--- a/src/allmydata/test/test_deferredutil.py
+++ b/src/allmydata/test/test_deferredutil.py
@@ -1,23 +1,16 @@
 """
 Tests for allmydata.util.deferredutil.
-
-Ported to Python 3.
 """
 
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
+from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
+from hypothesis.strategies import integers
+from hypothesis import given
 
 from allmydata.util import deferredutil
+from allmydata.util.deferredutil import race, MultiFailure
 
 
 class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin):
@@ -157,3 +150,148 @@ class AsyncToDeferred(unittest.TestCase):
 
         result = f(1, 0)
         self.assertIsInstance(self.failureResultOf(result).value, ZeroDivisionError)
+
+
+
+def _setupRaceState(numDeferreds: int) -> tuple[list[int], list[Deferred[object]]]:
+    """
+    Create a list of Deferreds and a corresponding list of integers
+    tracking how many times each Deferred has been cancelled.  Without
+    additional steps the Deferreds will never fire.
+    """
+    cancelledState = [0] * numDeferreds
+
+    ds: list[Deferred[object]] = []
+    for n in range(numDeferreds):
+
+        def cancel(d: Deferred, n: int = n) -> None:
+            cancelledState[n] += 1
+
+        ds.append(Deferred(canceller=cancel))
+
+    return cancelledState, ds
+
+
+class RaceTests(unittest.SynchronousTestCase):
+    """
+    Tests for L{race}.
+    """
+
+    @given(
+        beforeWinner=integers(min_value=0, max_value=3),
+        afterWinner=integers(min_value=0, max_value=3),
+    )
+    def test_success(self, beforeWinner: int, afterWinner: int) -> None:
+        """
+        When one of the L{Deferred}s passed to L{race} fires successfully,
+        the L{Deferred} return by L{race} fires with the index of that
+        L{Deferred} and its result and cancels the rest of the L{Deferred}s.
+        @param beforeWinner: A randomly selected number of Deferreds to
+            appear before the "winning" Deferred in the list passed in.
+        @param beforeWinner: A randomly selected number of Deferreds to
+            appear after the "winning" Deferred in the list passed in.
+        """
+        cancelledState, ds = _setupRaceState(beforeWinner + 1 + afterWinner)
+
+        raceResult = race(ds)
+        expected = object()
+        ds[beforeWinner].callback(expected)
+
+        # The result should be the index and result of the only Deferred that
+        # fired.
+        self.assertEqual(
+            self.successResultOf(raceResult),
+            (beforeWinner, expected),
+        )
+        # All Deferreds except the winner should have been cancelled once.
+        expectedCancelledState = [1] * beforeWinner + [0] + [1] * afterWinner
+        self.assertEqual(
+            cancelledState,
+            expectedCancelledState,
+        )
+
+    @given(
+        beforeWinner=integers(min_value=0, max_value=3),
+        afterWinner=integers(min_value=0, max_value=3),
+    )
+    def test_failure(self, beforeWinner: int, afterWinner: int) -> None:
+        """
+        When all of the L{Deferred}s passed to L{race} fire with failures,
+        the L{Deferred} return by L{race} fires with L{MultiFailure} wrapping
+        all of their failures.
+        @param beforeWinner: A randomly selected number of Deferreds to
+            appear before the "winning" Deferred in the list passed in.
+        @param beforeWinner: A randomly selected number of Deferreds to
+            appear after the "winning" Deferred in the list passed in.
+        """
+        cancelledState, ds = _setupRaceState(beforeWinner + 1 + afterWinner)
+
+        failure = Failure(Exception("The test demands failures."))
+        raceResult = race(ds)
+        for d in ds:
+            d.errback(failure)
+
+        actualFailure = self.failureResultOf(raceResult, MultiFailure)
+        self.assertEqual(
+            actualFailure.value.failures,
+            [failure] * len(ds),
+        )
+        self.assertEqual(
+            cancelledState,
+            [0] * len(ds),
+        )
+
+    @given(
+        beforeWinner=integers(min_value=0, max_value=3),
+        afterWinner=integers(min_value=0, max_value=3),
+    )
+    def test_resultAfterCancel(self, beforeWinner: int, afterWinner: int) -> None:
+        """
+        If one of the Deferreds fires after it was cancelled its result
+        goes nowhere.  In particular, it does not cause any errors to be
+        logged.
+        """
+        # Ensure we have a Deferred to win and at least one other Deferred
+        # that can ignore cancellation.
+        ds: list[Deferred[None]] = [
+            Deferred() for n in range(beforeWinner + 2 + afterWinner)
+        ]
+
+        raceResult = race(ds)
+        ds[beforeWinner].callback(None)
+        ds[beforeWinner + 1].callback(None)
+
+        self.successResultOf(raceResult)
+        self.assertEqual(len(self.flushLoggedErrors()), 0)
+
+    def test_resultFromCancel(self) -> None:
+        """
+        If one of the input Deferreds has a cancel function that fires it
+        with success, nothing bad happens.
+        """
+        winner: Deferred[object] = Deferred()
+        ds: list[Deferred[object]] = [
+            winner,
+            Deferred(canceller=lambda d: d.callback(object())),
+        ]
+        expected = object()
+        raceResult = race(ds)
+        winner.callback(expected)
+
+        self.assertEqual(self.successResultOf(raceResult), (0, expected))
+
+    @given(
+        numDeferreds=integers(min_value=1, max_value=3),
+    )
+    def test_cancel(self, numDeferreds: int) -> None:
+        """
+        If the result of L{race} is cancelled then all of the L{Deferred}s
+        passed in are cancelled.
+        """
+        cancelledState, ds = _setupRaceState(numDeferreds)
+
+        raceResult = race(ds)
+        raceResult.cancel()
+
+        self.assertEqual(cancelledState, [1] * numDeferreds)
+        self.failureResultOf(raceResult, MultiFailure)
diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 782663e8b..83de411ce 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -1,15 +1,18 @@
 """
 Utilities for working with Twisted Deferreds.
-
-Ported to Python 3.
 """
 
+from __future__ import annotations
+
 import time
 from functools import wraps
 
 from typing import (
     Callable,
     Any,
+    Sequence,
+    TypeVar,
+    Optional,
 )
 
 from foolscap.api import eventually
@@ -17,6 +20,7 @@ from eliot.twisted import (
     inline_callbacks,
 )
 from twisted.internet import defer, reactor, error
+from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
 
 from allmydata.util import log
@@ -234,3 +238,95 @@ def async_to_deferred(f):
         return defer.Deferred.fromCoroutine(f(*args, **kwargs))
 
     return not_async
+
+
+class MultiFailure(Exception):
+    """
+    More than one failure occurred.
+    """
+
+    def __init__(self, failures: Sequence[Failure]) -> None:
+        super(MultiFailure, self).__init__()
+        self.failures = failures
+
+
+_T = TypeVar("_T")
+
+# Eventually this should be in Twisted upstream:
+# https://github.com/twisted/twisted/pull/11818
+def race(ds: Sequence[Deferred[_T]]) -> Deferred[tuple[int, _T]]:
+    """
+    Select the first available result from the sequence of Deferreds and
+    cancel the rest.
+    @return: A cancellable L{Deferred} that fires with the index and output of
+        the element of C{ds} to have a success result first, or that fires
+        with L{MultiFailure} holding a list of their failures if they all
+        fail.
+    """
+    # Keep track of the Deferred for the action which completed first.  When
+    # it completes, all of the other Deferreds will get cancelled but this one
+    # shouldn't be.  Even though it "completed" it isn't really done - the
+    # caller will still be using it for something.  If we cancelled it,
+    # cancellation could propagate down to them.
+    winner: Optional[Deferred] = None
+
+    # The cancellation function for the Deferred this function returns.
+    def cancel(result: Deferred) -> None:
+        # If it is cancelled then we cancel all of the Deferreds for the
+        # individual actions because there is no longer the possibility of
+        # delivering any of their results anywhere.  We don't have to fire
+        # `result` because the Deferred will do that for us.
+        for d in to_cancel:
+            d.cancel()
+
+    # The Deferred that this function will return.  It will fire with the
+    # index and output of the action that completes first, or None if all of
+    # the actions fail.  If it is cancelled, all of the actions will be
+    # cancelled.
+    final_result: Deferred[tuple[int, _T]] = Deferred(canceller=cancel)
+
+    # A callback for an individual action.
+    def succeeded(this_output: _T, this_index: int) -> None:
+        # If it is the first action to succeed then it becomes the "winner",
+        # its index/output become the externally visible result, and the rest
+        # of the action Deferreds get cancelled.  If it is not the first
+        # action to succeed (because some action did not support
+        # cancellation), just ignore the result.  It is uncommon for this
+        # callback to be entered twice.  The only way it can happen is if one
+        # of the input Deferreds has a cancellation function that fires the
+        # Deferred with a success result.
+        nonlocal winner
+        if winner is None:
+            # This is the first success.  Act on it.
+            winner = to_cancel[this_index]
+
+            # Cancel the rest.
+            for d in to_cancel:
+                if d is not winner:
+                    d.cancel()
+
+            # Fire our Deferred
+            final_result.callback((this_index, this_output))
+
+    # Keep track of how many actions have failed.  If they all fail we need to
+    # deliver failure notification on our externally visible result.
+    failure_state = []
+
+    def failed(failure: Failure, this_index: int) -> None:
+        failure_state.append((this_index, failure))
+        if len(failure_state) == len(to_cancel):
+            # Every operation failed.
+            failure_state.sort()
+            failures = [f for (ignored, f) in failure_state]
+            final_result.errback(MultiFailure(failures))
+
+    # Copy the sequence of Deferreds so we know it doesn't get mutated out
+    # from under us.
+    to_cancel = list(ds)
+    for index, d in enumerate(ds):
+        # Propagate the position of this action as well as the argument to f
+        # to the success callback so we can cancel the right Deferreds and
+        # propagate the result outwards.
+        d.addCallbacks(succeeded, failed, callbackArgs=(index,), errbackArgs=(index,))
+
+    return final_result

From 0093edcd938b6f18692094a2442bcf236f42da39 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 14:36:37 -0500
Subject: [PATCH 023/362] Refactor to use race().

---
 src/allmydata/storage_client.py | 41 +++++++++------------------------
 1 file changed, 11 insertions(+), 30 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 686ee8e59..169d34e32 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -82,7 +82,7 @@ from allmydata.util.observer import ObserverList
 from allmydata.util.rrefutil import add_version_to_remote_reference
 from allmydata.util.hashutil import permute_server_hash
 from allmydata.util.dictutil import BytesKeyDict, UnicodeKeyDict
-from allmydata.util.deferredutil import async_to_deferred
+from allmydata.util.deferredutil import async_to_deferred, race
 from allmydata.storage.http_client import (
     StorageClient, StorageClientImmutables, StorageClientGeneral,
     ClientException as HTTPClientException, StorageClientMutables,
@@ -1017,42 +1017,23 @@ def _pick_a_http_server(
     Fires with ``None`` if no server was found, or with the ``DecodedURL`` of
     the first successfully-connected server.
     """
+    queries = race([
+        request(reactor, nurl).addCallback(lambda _, nurl=nurl: nurl)
+        for nurl in nurls
+    ])
 
-    to_cancel : list[defer.Deferred] = []
-
-    def cancel(result: Optional[defer.Deferred]):
-        for d in to_cancel:
-            if not d.called:
-                d.cancel()
-        if result is not None:
-            result.errback(defer.CancelledError())
-
-    result : defer.Deferred[Optional[DecodedURL]] = defer.Deferred(canceller=cancel)
-
-    def succeeded(nurl: DecodedURL, result=result):
-        # Only need the first successful NURL:
-        if result.called:
-            return
-        result.callback(nurl)
-        # No point in continuing other requests if we're connected:
-        cancel(None)
-
-    def failed(failure, failures=[], result=result):
+    def failed(failure: Failure):
         # Logging errors breaks a bunch of tests, and it's not a _bug_ to
         # have a failed connection, it's often expected and transient. More
         # of a warning, really?
         log.msg("Failed to connect to NURL: {}".format(failure))
-        failures.append(None)
-        if len(failures) == len(nurls):
-            # All our potential NURLs failed...
-            result.callback(None)
+        return None
 
-    for index, nurl in enumerate(nurls):
-        d = request(reactor, nurl)
-        to_cancel.append(d)
-        d.addCallback(lambda _, nurl=nurl: nurl).addCallbacks(succeeded, failed)
+    def succeeded(result: tuple[int, DecodedURL]):
+        _, nurl = result
+        return nurl
 
-    return result
+    return queries.addCallbacks(succeeded, failed)
 
 
 @implementer(IServer)

From 4db65ea9369b80d92b4a6d4c0c8ae9c7a7b8916c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 14:53:43 -0500
Subject: [PATCH 024/362] Make tests test _pick_a_http_server more directly.

---
 src/allmydata/test/test_storage_client.py | 46 ++++++++++-------------
 1 file changed, 19 insertions(+), 27 deletions(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index a51e44a82..c919440d8 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -8,7 +8,7 @@ from json import (
     loads,
 )
 import hashlib
-from typing import Union, Any
+from typing import Union, Any, Optional
 
 from hyperlink import DecodedURL
 from fixtures import (
@@ -740,15 +740,15 @@ storage:
 class PickHTTPServerTests(unittest.SynchronousTestCase):
     """Tests for ``_pick_a_http_server``."""
 
-    def loop_until_result(self, url_to_results: dict[DecodedURL, list[tuple[float, Union[Exception, Any]]]]) -> tuple[int, DecodedURL]:
+    def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Optional[DecodedURL]:
         """
-        Given mapping of URLs to list of (delay, result), return the URL of the
-        first selected server.
+        Given mapping of URLs to (delay, result), return the URL of the
+        first selected server, or None.
         """
         clock = Clock()
 
         def request(reactor, url):
-            delay, value = url_to_results[url].pop(0)
+            delay, value = url_to_results[url]
             result = Deferred()
             def add_result_value():
                 if isinstance(value, Exception):
@@ -758,15 +758,10 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
             reactor.callLater(delay, add_result_value)
             return result
 
-        iterations = 0
-        while True:
-            iterations += 1
-            d = _pick_a_http_server(clock, list(url_to_results.keys()), request)
-            for i in range(100):
-                clock.advance(0.1)
-            result = self.successResultOf(d)
-            if result is not None:
-                return iterations, result
+        d = _pick_a_http_server(clock, list(url_to_results.keys()), request)
+        for i in range(100):
+            clock.advance(0.1)
+        return self.successResultOf(d)
 
     def test_first_successful_connect_is_picked(self):
         """
@@ -774,25 +769,22 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
         """
         earliest_url = DecodedURL.from_text("http://a")
         latest_url = DecodedURL.from_text("http://b")
-        iterations, result = self.loop_until_result({
-            latest_url: [(2, None)],
-            earliest_url: [(1, None)]
+        bad_url = DecodedURL.from_text("http://bad")
+        result = self.pick_result({
+            latest_url: (2, None),
+            earliest_url: (1, None),
+            bad_url: (0.5, RuntimeError()),
         })
-        self.assertEqual(iterations, 1)
         self.assertEqual(result, earliest_url)
 
     def test_failures_are_retried(self):
         """
-        If the initial requests all fail, ``_pick_a_http_server`` keeps trying
-        until success.
+        If the requests all fail, ``_pick_a_http_server`` returns ``None``.
         """
         eventually_good_url = DecodedURL.from_text("http://good")
         bad_url = DecodedURL.from_text("http://bad")
-        iterations, result = self.loop_until_result({
-            eventually_good_url: [
-                (1, ZeroDivisionError()), (0.1, ZeroDivisionError()), (1, None)
-            ],
-            bad_url: [(0.1, RuntimeError()), (0.1, RuntimeError()), (0.1, RuntimeError())]
+        result = self.pick_result({
+            eventually_good_url: (1, ZeroDivisionError()),
+            bad_url: (0.1, RuntimeError())
         })
-        self.assertEqual(iterations, 3)
-        self.assertEqual(result, eventually_good_url)
+        self.assertEqual(result, None)

From 3702ad62335451d8148a70bc7e36fa93d3d52fa5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 14:54:53 -0500
Subject: [PATCH 025/362] Fix indentation.

---
 src/allmydata/storage_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 169d34e32..4073d9b41 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1212,8 +1212,8 @@ class HTTPNativeStorageServer(service.MultiService):
                 return
             else:
                 self._istorage_server = _HTTPStorageServer.from_http_client(
-                StorageClient.from_nurl(nurl, reactor)
-            )
+                    StorageClient.from_nurl(nurl, reactor)
+                )
 
         result = self._istorage_server.get_version()
 

From 8ccbd37d29906cef62d8db22573878534a783fdd Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Wed, 8 Mar 2023 15:16:03 -0600
Subject: [PATCH 026/362] Fix implicit re-export error by importing IWormhole
 from wormhole library directly

---
 src/allmydata/test/cli/test_invite.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py
index 07756eeed..94d4395ff 100644
--- a/src/allmydata/test/cli/test_invite.py
+++ b/src/allmydata/test/cli/test_invite.py
@@ -19,7 +19,8 @@ from ...util.jsonbytes import dumps_bytes
 from ..common_util import run_cli
 from ..no_network import GridTestMixin
 from .common import CLITestMixin
-from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server
+from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server
+from wormhole._interfaces import IWormhole
 
 # Logically:
 #   JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]]

From 10b3eabed41baedd47e3b4f9ce55aec92699003a Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Wed, 8 Mar 2023 15:19:08 -0600
Subject: [PATCH 027/362] Apply per file flags corresponding to --strict to
 wormholetesting.py

---
 mypy.ini                                  | 2 +-
 src/allmydata/test/cli/wormholetesting.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/mypy.ini b/mypy.ini
index e6e7d16ff..27e9f6154 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -7,4 +7,4 @@ show_error_codes = True
 warn_unused_configs =True
 no_implicit_optional = True
 warn_redundant_casts = True
-strict_equality = True
\ No newline at end of file
+strict_equality = True
diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index d4e53a342..a0050a75b 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -29,6 +29,7 @@ For example::
     import wormhole
     run(peerA(wormhole))
 """
+# mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate
 
 from __future__ import annotations
 

From 4f47a18c6af89e92c81641c9bcc96bb30398c355 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Wed, 8 Mar 2023 15:29:50 -0600
Subject: [PATCH 028/362] Comments added for inline mypy config. Individual
 flags used as --strict flag can only be used on a per-module basis.

---
 src/allmydata/test/cli/wormholetesting.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index a0050a75b..6fb2b791c 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -30,6 +30,10 @@ For example::
     run(peerA(wormhole))
 """
 # mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate
+# This inline mypy config applies a per-file mypy config for this file.
+# It applies the '--strict' list of flags to this file.
+# If you want to test using CLI run the command remove the inline config above and run:
+# "mypy --follow-imports silent --strict src/allmydata/test/cli/wormholetesting.py"
 
 from __future__ import annotations
 

From b43150ba85c7a3b5d1cb5952f9a235f9493c04d7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 8 Mar 2023 16:48:08 -0500
Subject: [PATCH 029/362] Add future import.

---
 src/allmydata/test/test_deferredutil.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py
index 47121b4cb..34358d0c8 100644
--- a/src/allmydata/test/test_deferredutil.py
+++ b/src/allmydata/test/test_deferredutil.py
@@ -2,6 +2,8 @@
 Tests for allmydata.util.deferredutil.
 """
 
+from __future__ import annotations
+
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
 from twisted.internet.defer import Deferred

From 74ff8cd08041a1107b05771778310449bf4d99f8 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 11:04:52 -0500
Subject: [PATCH 030/362] Per-file configuration for wormholetesting.py moved
 from inline mypy configuration moved to mypy.ini file

---
 mypy.ini                                  | 16 ++++++++++++++++
 src/allmydata/test/cli/wormholetesting.py |  5 -----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/mypy.ini b/mypy.ini
index 27e9f6154..c391c5594 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -8,3 +8,19 @@ warn_unused_configs =True
 no_implicit_optional = True
 warn_redundant_casts = True
 strict_equality = True
+
+[mypy-allmydata.test.cli.wormholetesting]
+warn_unused_configs = True
+disallow_any_generics = True
+disallow_subclassing_any = True
+disallow_untyped_calls = True
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+check_untyped_defs = True
+disallow_untyped_decorators = True
+warn_redundant_casts = True
+warn_unused_ignores = True
+warn_return_any = True
+no_implicit_reexport = True
+strict_equality = True
+strict_concatenate = True
diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 6fb2b791c..d4e53a342 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -29,11 +29,6 @@ For example::
     import wormhole
     run(peerA(wormhole))
 """
-# mypy: warn-unused-configs, disallow-any-generics, disallow-subclassing-any, disallow-untyped-calls, disallow-untyped-defs, disallow-incomplete-defs, check-untyped-defs, disallow-untyped-decorators, warn-redundant-casts, warn-unused-ignores, warn-return-any, no-implicit-reexport, strict-equality, strict-concatenate
-# This inline mypy config applies a per-file mypy config for this file.
-# It applies the '--strict' list of flags to this file.
-# If you want to test using CLI run the command remove the inline config above and run:
-# "mypy --follow-imports silent --strict src/allmydata/test/cli/wormholetesting.py"
 
 from __future__ import annotations
 

From 61c835c8a05c15b7eabe29b453633b7c4da022e8 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 11:17:01 -0500
Subject: [PATCH 031/362] Added missing space between return type

---
 src/allmydata/test/cli/wormholetesting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index d4e53a342..be94a7981 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -76,7 +76,7 @@ class MemoryWormholeServer(object):
         stderr: TextIO=stderr,
         _eventual_queue: Optional[Any]=None,
         _enable_dilate: bool=False,
-    )-> _MemoryWormhole:
+    ) -> _MemoryWormhole:
         """
         Create a wormhole.  It will be able to connect to other wormholes created
         by this instance (and constrained by the normal appid/relay_url

From b58dd2bb3bed375258f611eb0af39f6c08f64684 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 12:27:53 -0500
Subject: [PATCH 032/362] Remove flags that are unused

---
 mypy.ini | 2 --
 1 file changed, 2 deletions(-)

diff --git a/mypy.ini b/mypy.ini
index c391c5594..7acc0ddc5 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -10,7 +10,6 @@ warn_redundant_casts = True
 strict_equality = True
 
 [mypy-allmydata.test.cli.wormholetesting]
-warn_unused_configs = True
 disallow_any_generics = True
 disallow_subclassing_any = True
 disallow_untyped_calls = True
@@ -18,7 +17,6 @@ disallow_untyped_defs = True
 disallow_incomplete_defs = True
 check_untyped_defs = True
 disallow_untyped_decorators = True
-warn_redundant_casts = True
 warn_unused_ignores = True
 warn_return_any = True
 no_implicit_reexport = True

From 041a634d27f1f2adfdc82471e60192aaecb1fbfc Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 13:08:32 -0500
Subject: [PATCH 033/362] Fix private interface import to test_invite

---
 src/allmydata/test/cli/test_invite.py     | 4 ++--
 src/allmydata/test/cli/wormholetesting.py | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py
index 94d4395ff..31992a54d 100644
--- a/src/allmydata/test/cli/test_invite.py
+++ b/src/allmydata/test/cli/test_invite.py
@@ -19,8 +19,8 @@ from ...util.jsonbytes import dumps_bytes
 from ..common_util import run_cli
 from ..no_network import GridTestMixin
 from .common import CLITestMixin
-from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server
-from wormhole._interfaces import IWormhole
+from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server, IWormhole
+
 
 # Logically:
 #   JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]]
diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index be94a7981..9fbe8b63e 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -32,6 +32,8 @@ For example::
 
 from __future__ import annotations
 
+__all__ = ['IWormhole']
+
 from typing import Iterator, Optional, List, Tuple, Any, TextIO
 from inspect import getfullargspec
 from itertools import count
@@ -76,7 +78,7 @@ class MemoryWormholeServer(object):
         stderr: TextIO=stderr,
         _eventual_queue: Optional[Any]=None,
         _enable_dilate: bool=False,
-    ) -> _MemoryWormhole:
+    )-> _MemoryWormhole:
         """
         Create a wormhole.  It will be able to connect to other wormholes created
         by this instance (and constrained by the normal appid/relay_url

From f1be1ca1de497f4b3ebb5d5795803a5a331dcba9 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 14:53:25 -0500
Subject: [PATCH 034/362] Added more elements to export list in
 wormholetesting.py

---
 src/allmydata/test/cli/wormholetesting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 9fbe8b63e..99e26e64b 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -32,7 +32,7 @@ For example::
 
 from __future__ import annotations
 
-__all__ = ['IWormhole']
+__all__ = ['MemoryWormholeServer', 'TestingHelper', 'memory_server', 'IWormhole']
 
 from typing import Iterator, Optional, List, Tuple, Any, TextIO
 from inspect import getfullargspec

From 1c926aeb869817c0a2aaa76786075b5459c396a2 Mon Sep 17 00:00:00 2001
From: dlee <daniel.zwlee@gmail.com>
Date: Mon, 13 Mar 2023 16:23:28 -0500
Subject: [PATCH 035/362] Add space to return type

---
 src/allmydata/test/cli/wormholetesting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 99e26e64b..eb6249a0d 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -78,7 +78,7 @@ class MemoryWormholeServer(object):
         stderr: TextIO=stderr,
         _eventual_queue: Optional[Any]=None,
         _enable_dilate: bool=False,
-    )-> _MemoryWormhole:
+    ) -> _MemoryWormhole:
         """
         Create a wormhole.  It will be able to connect to other wormholes created
         by this instance (and constrained by the normal appid/relay_url

From f8ea650b922ba5622debdf6a49121388e2a32e2d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 14 Mar 2023 12:02:32 -0400
Subject: [PATCH 036/362] Wait for current loop iteration to finish before
 moving on to next iteration.

---
 src/allmydata/storage_client.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 5dd906005..faa48710f 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1239,9 +1239,13 @@ class HTTPNativeStorageServer(service.MultiService):
             self._failed_to_connect
         )
 
-        # TODO Make sure LoopingCall waits for the above timeout for looping again:
-        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3981
-        #return self._connecting_deferred or maye await it
+        # We don't want to do another iteration of the loop until this
+        # iteration has finished, so wait here:
+        try:
+            if self._connecting_deferred is not None:
+                await self._connecting_deferred
+        except Exception as e:
+            log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
 
     def stopService(self):
         if self._connecting_deferred is not None:

From dd07a39399709b59d7c1c1e2e0cb5d3ac4d6ff65 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 14 Mar 2023 13:01:10 -0400
Subject: [PATCH 037/362] Don't bother with persistent connections when testing
 NURLs.

---
 src/allmydata/storage/http_client.py | 10 ++++++----
 src/allmydata/storage_client.py      |  6 +++++-
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 90bda7fc0..3edf5f835 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -311,18 +311,20 @@ class StorageClient(object):
 
     @classmethod
     def from_nurl(
-        cls,
-        nurl: DecodedURL,
-        reactor,
+        cls, nurl: DecodedURL, reactor, persistent=True, retryAutomatically=True
     ) -> StorageClient:
         """
         Create a ``StorageClient`` for the given NURL.
+
+        ``persistent`` and ``retryAutomatically`` arguments are passed to the
+        new HTTPConnectionPool.
         """
         assert nurl.fragment == "v=1"
         assert nurl.scheme == "pb"
         swissnum = nurl.path[0].encode("ascii")
         certificate_hash = nurl.user.encode("ascii")
-        pool = HTTPConnectionPool(reactor)
+        pool = HTTPConnectionPool(reactor, persistent=persistent)
+        pool.retryAutomatically = retryAutomatically
         pool.maxPersistentPerHost = 20
 
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index faa48710f..2888b10e7 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1203,8 +1203,12 @@ class HTTPNativeStorageServer(service.MultiService):
             # the HTTP server to talk to, we don't have connection status
             # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
             def request(reactor, nurl: DecodedURL):
+                # Since we're just using this one off to check if the NURL
+                # works, no need for persistent pool or other fanciness.
                 return StorageClientGeneral(
-                    StorageClient.from_nurl(nurl, reactor)
+                    StorageClient.from_nurl(
+                        nurl, reactor, persistent=False, retryAutomatically=False
+                    )
                 ).get_version()
 
             # LoopingCall.stop() doesn't cancel Deferreds, unfortunately:

From 24212b412cba52481695b85e057da1ac04a96d67 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 14 Mar 2023 13:01:45 -0400
Subject: [PATCH 038/362] Fix 3.11 runs.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 3e2dacbb2..538cded80 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,6 +10,7 @@ python =
     3.8: py38-coverage
     3.9: py39-coverage
     3.10: py310-coverage
+    3.11: py311-coverage
     pypy-3.8: pypy38
     pypy-3.9: pypy39
 

From 52f43cefea6b447cda47cd5e75df4937190f8bd6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 15 Mar 2023 15:44:45 -0400
Subject: [PATCH 039/362] Add 3.11.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 538cded80..382ba973e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -18,7 +18,7 @@ python =
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration
+envlist = typechecks,codechecks,py{38,39,310,311}-{coverage},pypy27,pypy38,pypy39,integration
 minversion = 2.4
 
 [testenv]

From a24e6bd7f94311b185072132b3372fd4a0afb723 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 16 Mar 2023 16:31:28 -0400
Subject: [PATCH 040/362] Try to rewrite test_get_put.py::test_large_file into
 system-style test.

---
 src/allmydata/test/test_system.py | 32 +++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index d11a6e866..b7873f14f 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -15,11 +15,15 @@ from past.builtins import chr as byteschr, long
 from six import ensure_text
 
 import os, re, sys, time, json
+from subprocess import check_call
+from pathlib import Path
+from tempfile import mkdtemp
 
 from bs4 import BeautifulSoup
 
 from twisted.trial import unittest
 from twisted.internet import defer
+from twisted.internet.threads import deferToThread
 
 from allmydata import uri
 from allmydata.storage.mutable import MutableShareFile
@@ -1830,6 +1834,34 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(_got_lit_filenode)
         return d
 
+    async def test_immutable_upload_download(self):
+        """
+        A reproducer for issue 3988: upload a large file and then download it.
+        """
+        DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" * 1_000_000
+        await self.set_up_nodes()
+
+        async def run(*args):
+            await deferToThread(check_call, ["tahoe", "--node-directory", self.getdir("client0")] + list(args))
+
+        for c in self.clients:
+            c.encoding_params['k'] = 2
+            c.encoding_params['happy'] = 3
+            c.encoding_params['n'] = 4
+
+        await run("create-alias", "getput")
+
+        tmp_path = Path(mkdtemp())
+        tempfile = tmp_path / "input"
+
+        with tempfile.open("wb") as f:
+            f.write(DATA)
+        await run("put", str(tempfile), "getput:largefile")
+
+        outfile = tmp_path / "out"
+        await run("get", "getput:largefile", str(outfile))
+        self.assertEqual(outfile.read_bytes(), tempfile.read_bytes())
+
 
 class Connections(SystemTestMixin, unittest.TestCase):
     FORCE_FOOLSCAP_FOR_STORAGE = True

From a3ebd21b25c29fe8a871ae53967e0ed3f29be5d4 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Mar 2023 15:30:14 -0400
Subject: [PATCH 041/362] implement retry ourselves, don't depend on tenacity

---
 setup.py                          |  1 -
 src/allmydata/test/test_iputil.py | 55 ++++++++++++++++++++++++-------
 2 files changed, 44 insertions(+), 12 deletions(-)

diff --git a/setup.py b/setup.py
index 82ff45764..152c49f0e 100644
--- a/setup.py
+++ b/setup.py
@@ -413,7 +413,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "beautifulsoup4",
               "html5lib",
               "junitxml",
-              "tenacity",
               # Pin old version until
               # https://github.com/paramiko/paramiko/issues/1961 is fixed.
               "paramiko < 2.9",
diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index 081c80ee3..c060fcc04 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -4,18 +4,14 @@ Tests for allmydata.util.iputil.
 Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2, native_str
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
 import os, socket
 import gc
+from functools import wraps
 
+from typing import TypeVar, Callable
+from typing_extensions import TypeAlias
 from testtools.matchers import (
     MatchesAll,
     IsInstance,
@@ -25,8 +21,6 @@ from testtools.matchers import (
 
 from twisted.trial import unittest
 
-from tenacity import retry, stop_after_attempt
-
 from foolscap.api import Tub
 
 from allmydata.util import iputil, gcutil
@@ -39,6 +33,45 @@ from .common import (
     SyncTestCase,
 )
 
+T = TypeVar("T")
+
+TestFunction: TypeAlias = Callable[[], T]
+Decorator: TypeAlias = Callable[[TestFunction[T]], TestFunction[T]]
+
+def retry(stop: Callable[[], bool]) -> Decorator[T]:
+    """
+    Call a function until the predicate says to stop or the function stops
+    raising an exception.
+
+    :param stop: A callable to call after the decorated function raises an
+        exception.  The decorated function will be called again if ``stop``
+        returns ``False``.
+
+    :return: A decorator function.
+    """
+    def decorate(f: TestFunction[T]) -> TestFunction[T]:
+        @wraps(f)
+        def decorator(self) -> T:
+            while True:
+                try:
+                    return f(self)
+                except Exception:
+                    if stop():
+                        raise
+        return decorator
+    return decorate
+
+def stop_after_attempt(limit: int) -> Callable[[], bool]:
+    """
+    Stop after ``limit`` calls.
+    """
+    counter = 0
+    def check():
+        nonlocal counter
+        counter += 1
+        return counter < limit
+    return check
+
 class ListenOnUsed(unittest.TestCase):
     """Tests for listenOnUnused."""
 
@@ -127,7 +160,7 @@ class GetLocalAddressesSyncTests(SyncTestCase):
                 IsInstance(list),
                 AllMatch(
                     MatchesAll(
-                        IsInstance(native_str),
+                        IsInstance(str),
                         MatchesPredicate(
                             lambda addr: socket.inet_pton(socket.AF_INET, addr),
                             "%r is not an IPv4 address.",

From a9f34655686764e633be4a6ccb8f2d79b841a291 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Mar 2023 15:31:07 -0400
Subject: [PATCH 042/362] news fragment

---
 newsfragments/3989.installation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3989.installation

diff --git a/newsfragments/3989.installation b/newsfragments/3989.installation
new file mode 100644
index 000000000..a2155b65c
--- /dev/null
+++ b/newsfragments/3989.installation
@@ -0,0 +1 @@
+tenacity is no longer a dependency.

From 5cf892b441daf77ad5efada4a785dfa4a0e2ecf6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Mar 2023 15:32:13 -0400
Subject: [PATCH 043/362] Also remove it from the Nix packaging

---
 nix/tahoe-lafs.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index 380260c70..5986db420 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -58,7 +58,6 @@ let
     pytest
     pytest-timeout
     pytest-twisted
-    tenacity
     testtools
     towncrier
   ];

From 6a4346587cf06f7603572796daf4851bd98a1415 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Mar 2023 15:46:27 -0400
Subject: [PATCH 044/362] Fix the type annotations

---
 src/allmydata/test/test_iputil.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py
index c060fcc04..26274830f 100644
--- a/src/allmydata/test/test_iputil.py
+++ b/src/allmydata/test/test_iputil.py
@@ -11,7 +11,6 @@ import gc
 from functools import wraps
 
 from typing import TypeVar, Callable
-from typing_extensions import TypeAlias
 from testtools.matchers import (
     MatchesAll,
     IsInstance,
@@ -33,12 +32,10 @@ from .common import (
     SyncTestCase,
 )
 
-T = TypeVar("T")
+T = TypeVar("T", contravariant=True)
+U = TypeVar("U", covariant=True)
 
-TestFunction: TypeAlias = Callable[[], T]
-Decorator: TypeAlias = Callable[[TestFunction[T]], TestFunction[T]]
-
-def retry(stop: Callable[[], bool]) -> Decorator[T]:
+def retry(stop: Callable[[], bool]) -> Callable[[Callable[[T], U]], Callable[[T], U]]:
     """
     Call a function until the predicate says to stop or the function stops
     raising an exception.
@@ -49,9 +46,9 @@ def retry(stop: Callable[[], bool]) -> Decorator[T]:
 
     :return: A decorator function.
     """
-    def decorate(f: TestFunction[T]) -> TestFunction[T]:
+    def decorate(f: Callable[[T], U]) -> Callable[[T], U]:
         @wraps(f)
-        def decorator(self) -> T:
+        def decorator(self: T) -> U:
             while True:
                 try:
                     return f(self)

From 61d9d82c55644b3a786b3b05725438a3cff02a18 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:02:35 -0400
Subject: [PATCH 045/362] Make await_client_ready() non-blocking.

---
 integration/conftest.py                  |  6 +++---
 integration/test_get_put.py              |  4 ----
 integration/test_servers_of_happiness.py |  2 +-
 integration/test_tor.py                  |  4 ++--
 integration/util.py                      | 11 +++++------
 newsfragments/3988.minor                 |  0
 6 files changed, 11 insertions(+), 16 deletions(-)
 create mode 100644 newsfragments/3988.minor

diff --git a/integration/conftest.py b/integration/conftest.py
index dc0107eea..33e7998c1 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -393,7 +393,7 @@ def alice(
             finalize=False,
         )
     )
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))
 
     # 1. Create a new RW directory cap:
     cli(process, "create-alias", "test")
@@ -424,7 +424,7 @@ alice-key ssh-rsa {ssh_public_key} {rwcap}
 
     # 4. Restart the node with new SFTP config.
     pytest_twisted.blockon(process.restart_async(reactor, request))
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))
     print(f"Alice pid: {process.transport.pid}")
     return process
 
@@ -439,7 +439,7 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques
             storage=False,
         )
     )
-    await_client_ready(process)
+    pytest_twisted.blockon(await_client_ready(process))
     return process
 
 
diff --git a/integration/test_get_put.py b/integration/test_get_put.py
index 1b6c30072..927ec622b 100644
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@@ -85,10 +85,6 @@ def test_large_file(alice, get_put_alias, tmp_path):
     assert outfile.read_bytes() == tempfile.read_bytes()
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"),
-    reason="reconfigure() has issues on Windows"
-)
 @ensureDeferred
 async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
     """
diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index b9de0c075..c63642066 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -31,7 +31,7 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
         happy=7,
         total=10,
     )
-    util.await_client_ready(edna)
+    yield util.await_client_ready(edna)
 
     node_dir = join(temp_dir, 'edna')
 
diff --git a/integration/test_tor.py b/integration/test_tor.py
index c78fa8098..901858347 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -42,8 +42,8 @@ if PY2:
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    util.await_client_ready(carol, minimum_number_of_servers=2)
-    util.await_client_ready(dave, minimum_number_of_servers=2)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
diff --git a/integration/util.py b/integration/util.py
index 04c925abf..c2befe47b 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -570,6 +570,10 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
     We will try for up to `timeout` seconds for the above conditions
     to be true. Otherwise, an exception is raised
     """
+    return deferToThread(_await_client_ready_blocking, tahoe, timeout, liveness, minimum_number_of_servers)
+
+
+def _await_client_ready_blocking(tahoe, timeout, liveness, minimum_number_of_servers):
     start = time.time()
     while (time.time() - start) < float(timeout):
         try:
@@ -792,16 +796,11 @@ async def reconfigure(reactor, request, node: TahoeProcess,
             )
 
     if changed:
-        # TODO reconfigure() seems to have issues on Windows. If you need to
-        # use it there, delete this assert and try to figure out what's going
-        # on...
-        assert not sys.platform.startswith("win")
-
         # restart the node
         print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
         await node.restart_async(reactor, request)
         print("Restarted.  Waiting for ready state.")
-        await_client_ready(node)
+        await await_client_ready(node)
         print("Ready.")
     else:
         print("Config unchanged, not restarting.")
diff --git a/newsfragments/3988.minor b/newsfragments/3988.minor
new file mode 100644
index 000000000..e69de29bb

From aba60d271956d5100375c14b507a720582459860 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:08:22 -0400
Subject: [PATCH 046/362] Run blocking tests in a thread.

---
 integration/test_get_put.py |  2 ++
 integration/test_web.py     | 11 +++++++++++
 2 files changed, 13 insertions(+)

diff --git a/integration/test_get_put.py b/integration/test_get_put.py
index 927ec622b..6b87c9b62 100644
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@@ -50,6 +50,7 @@ def test_put_from_stdin(alice, get_put_alias, tmpdir):
     assert read_bytes(tempfile) == DATA
 
 
+@run_in_thread
 def test_get_to_stdout(alice, get_put_alias, tmpdir):
     """
     It's possible to upload a file, and then download it to stdout.
@@ -67,6 +68,7 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir):
     assert p.wait() == 0
 
 
+@run_in_thread
 def test_large_file(alice, get_put_alias, tmp_path):
     """
     It's possible to upload and download a larger file.
diff --git a/integration/test_web.py b/integration/test_web.py
index 95a09a5f5..b3c4a8e5f 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -18,6 +18,7 @@ import allmydata.uri
 from allmydata.util import jsonbytes as json
 
 from . import util
+from .util import run_in_thread
 
 import requests
 import html5lib
@@ -25,6 +26,7 @@ from bs4 import BeautifulSoup
 
 from pytest_twisted import ensureDeferred
 
+@run_in_thread
 def test_index(alice):
     """
     we can download the index file
@@ -32,6 +34,7 @@ def test_index(alice):
     util.web_get(alice, u"")
 
 
+@run_in_thread
 def test_index_json(alice):
     """
     we can download the index file as json
@@ -41,6 +44,7 @@ def test_index_json(alice):
     json.loads(data)
 
 
+@run_in_thread
 def test_upload_download(alice):
     """
     upload a file, then download it via readcap
@@ -70,6 +74,7 @@ def test_upload_download(alice):
     assert str(data, "utf-8") == FILE_CONTENTS
 
 
+@run_in_thread
 def test_put(alice):
     """
     use PUT to create a file
@@ -89,6 +94,7 @@ def test_put(alice):
     assert cap.needed_shares == int(cfg.get_config("client", "shares.needed"))
 
 
+@run_in_thread
 def test_helper_status(storage_nodes):
     """
     successfully GET the /helper_status page
@@ -101,6 +107,7 @@ def test_helper_status(storage_nodes):
     assert str(dom.h1.string) == u"Helper Status"
 
 
+@run_in_thread
 def test_deep_stats(alice):
     """
     create a directory, do deep-stats on it and prove the /operations/
@@ -417,6 +424,7 @@ async def test_directory_deep_check(reactor, request, alice):
     assert dom is not None, "Operation never completed"
 
 
+@run_in_thread
 def test_storage_info(storage_nodes):
     """
     retrieve and confirm /storage URI for one storage node
@@ -428,6 +436,7 @@ def test_storage_info(storage_nodes):
     )
 
 
+@run_in_thread
 def test_storage_info_json(storage_nodes):
     """
     retrieve and confirm /storage?t=json URI for one storage node
@@ -442,6 +451,7 @@ def test_storage_info_json(storage_nodes):
     assert data[u"stats"][u"storage_server.reserved_space"] == 1000000000
 
 
+@run_in_thread
 def test_introducer_info(introducer):
     """
     retrieve and confirm /introducer URI for the introducer
@@ -460,6 +470,7 @@ def test_introducer_info(introducer):
     assert "subscription_summary" in data
 
 
+@run_in_thread
 def test_mkdir_with_children(alice):
     """
     create a directory using ?t=mkdir-with-children

From ded5b20924537bbda2471581606cc9adbe7f87ce Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:20:39 -0400
Subject: [PATCH 047/362] Lint fix.

---
 integration/test_get_put.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/test_get_put.py b/integration/test_get_put.py
index 6b87c9b62..f121d6284 100644
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@@ -4,7 +4,6 @@ and stdout.
 """
 
 from subprocess import Popen, PIPE, check_output, check_call
-import sys
 
 import pytest
 from pytest_twisted import ensureDeferred

From cce5d3adff757d67fdde04da8cd10a86e5a61176 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:24:10 -0400
Subject: [PATCH 048/362] Don't actually need this.

---
 src/allmydata/test/test_system.py | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index b7873f14f..c997ac734 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -1834,34 +1834,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         d.addCallback(_got_lit_filenode)
         return d
 
-    async def test_immutable_upload_download(self):
-        """
-        A reproducer for issue 3988: upload a large file and then download it.
-        """
-        DATA = b"abc123 this is not utf-8 decodable \xff\x00\x33 \x11" * 1_000_000
-        await self.set_up_nodes()
-
-        async def run(*args):
-            await deferToThread(check_call, ["tahoe", "--node-directory", self.getdir("client0")] + list(args))
-
-        for c in self.clients:
-            c.encoding_params['k'] = 2
-            c.encoding_params['happy'] = 3
-            c.encoding_params['n'] = 4
-
-        await run("create-alias", "getput")
-
-        tmp_path = Path(mkdtemp())
-        tempfile = tmp_path / "input"
-
-        with tempfile.open("wb") as f:
-            f.write(DATA)
-        await run("put", str(tempfile), "getput:largefile")
-
-        outfile = tmp_path / "out"
-        await run("get", "getput:largefile", str(outfile))
-        self.assertEqual(outfile.read_bytes(), tempfile.read_bytes())
-
 
 class Connections(SystemTestMixin, unittest.TestCase):
     FORCE_FOOLSCAP_FOR_STORAGE = True

From 815066c4de7db05c5df86ca1b9ab7dbeb08cfab2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:25:52 -0400
Subject: [PATCH 049/362] Just use the utility.

---
 integration/util.py | 54 ++++++++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 28 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index c2befe47b..05fef8fed 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -430,6 +430,31 @@ class FileShouldVanishException(Exception):
         )
 
 
+def run_in_thread(f):
+    """Decorator for integration tests that runs code in a thread.
+
+    Because we're using pytest_twisted, tests that rely on the reactor are
+    expected to return a Deferred and use async APIs so the reactor can run.
+
+    In the case of the integration test suite, it launches nodes in the
+    background using Twisted APIs.  The nodes stdout and stderr is read via
+    Twisted code.  If the reactor doesn't run, reads don't happen, and
+    eventually the buffers fill up, and the nodes block when they try to flush
+    logs.
+
+    We can switch to Twisted APIs (treq instead of requests etc.), but
+    sometimes it's easier or expedient to just have a blocking test.  So this
+    decorator allows you to run the test in a thread, and the reactor can keep
+    running in the main thread.
+
+    See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
+    """
+    @wraps(f)
+    def test(*args, **kwargs):
+        return deferToThread(lambda: f(*args, **kwargs))
+    return test
+
+
 def await_file_contents(path, contents, timeout=15, error_if=None):
     """
     wait up to `timeout` seconds for the file at `path` (any path-like
@@ -555,6 +580,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
     return resp.content
 
 
+@run_in_thread
 def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
@@ -570,10 +596,6 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
     We will try for up to `timeout` seconds for the above conditions
     to be true. Otherwise, an exception is raised
     """
-    return deferToThread(_await_client_ready_blocking, tahoe, timeout, liveness, minimum_number_of_servers)
-
-
-def _await_client_ready_blocking(tahoe, timeout, liveness, minimum_number_of_servers):
     start = time.time()
     while (time.time() - start) < float(timeout):
         try:
@@ -626,30 +648,6 @@ def generate_ssh_key(path):
         f.write(s.encode("ascii"))
 
 
-def run_in_thread(f):
-    """Decorator for integration tests that runs code in a thread.
-
-    Because we're using pytest_twisted, tests that rely on the reactor are
-    expected to return a Deferred and use async APIs so the reactor can run.
-
-    In the case of the integration test suite, it launches nodes in the
-    background using Twisted APIs.  The nodes stdout and stderr is read via
-    Twisted code.  If the reactor doesn't run, reads don't happen, and
-    eventually the buffers fill up, and the nodes block when they try to flush
-    logs.
-
-    We can switch to Twisted APIs (treq instead of requests etc.), but
-    sometimes it's easier or expedient to just have a blocking test.  So this
-    decorator allows you to run the test in a thread, and the reactor can keep
-    running in the main thread.
-
-    See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug.
-    """
-    @wraps(f)
-    def test(*args, **kwargs):
-        return deferToThread(lambda: f(*args, **kwargs))
-    return test
-
 @frozen
 class CHK:
     """

From 23b977a4b1626683116c71e08ed1a4f33381d5f0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 20 Mar 2023 15:27:16 -0400
Subject: [PATCH 050/362] Undo unnecessary imports.

---
 src/allmydata/test/test_system.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index c997ac734..d11a6e866 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -15,15 +15,11 @@ from past.builtins import chr as byteschr, long
 from six import ensure_text
 
 import os, re, sys, time, json
-from subprocess import check_call
-from pathlib import Path
-from tempfile import mkdtemp
 
 from bs4 import BeautifulSoup
 
 from twisted.trial import unittest
 from twisted.internet import defer
-from twisted.internet.threads import deferToThread
 
 from allmydata import uri
 from allmydata.storage.mutable import MutableShareFile

From 900b4a3c989f0ef707b0ab39b72063aa3f991a7c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 21 Mar 2023 08:55:41 -0400
Subject: [PATCH 051/362] Package a version of collections-extended compatible
 with Python 3.11

---
 nix/collections-extended.nix | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 nix/collections-extended.nix

diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix
new file mode 100644
index 000000000..05254fc1b
--- /dev/null
+++ b/nix/collections-extended.nix
@@ -0,0 +1,12 @@
+# Package a version that's compatible with Python 3.11.  This can go away once
+# https://github.com/mlenzen/collections-extended/pull/199 is merged and
+# included in a version of nixpkgs we depend on.
+{ fetchFromGitHub, collections-extended }:
+collections-extended.overrideAttrs (old: {
+  src = fetchFromGitHub {
+    owner = "mlenzen";
+    repo = "collections-extended";
+    rev = "8b93390636d58d28012b8e9d22334ee64ca37d73";
+    hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0=";
+  };
+})

From 41d5538921e07d291e3f88f6ffaf60d8c2e68daa Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 21 Mar 2023 08:56:05 -0400
Subject: [PATCH 052/362] Fix `maturin build` when using PyPy for the pycddl
 package

---
 nix/pycddl.nix | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/nix/pycddl.nix b/nix/pycddl.nix
index 703f00595..4c68830d4 100644
--- a/nix/pycddl.nix
+++ b/nix/pycddl.nix
@@ -27,7 +27,7 @@
 #
 # 8. run `nix-build`.  it should succeed.  if it does not, seek assistance.
 #
-{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
+{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }:
 buildPythonPackage rec {
   pname = "pycddl";
   version = "0.4.0";
@@ -38,6 +38,12 @@ buildPythonPackage rec {
     sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
   };
 
+  # Without this, when building for PyPy, `maturin build` seems to fail to
+  # find the interpreter at all and then fails early in the build process with
+  # an error saying "unsupported Python interpreter".  We can easily point
+  # directly at the relevant interpreter, so do that.
+  maturinBuildFlags = [ "--interpreter" python.executable ];
+
   nativeBuildInputs = with rustPlatform; [
     maturinBuildHook
     cargoSetupHook

From dd8f6d408d27908de924ce400c7c717536095f0b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 21 Mar 2023 08:56:50 -0400
Subject: [PATCH 053/362] Remove the non-unit test dependencies from the unit
 test inputs

---
 nix/tahoe-lafs.nix | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index 5986db420..2e1c4aa39 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -48,19 +48,15 @@ let
     zope_interface
   ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
 
-  pythonCheckDependencies = with pythonPackages; [
+  unitTestDependencies = with pythonPackages; [
     beautifulsoup4
     fixtures
     hypothesis
     mock
-    paramiko
     prometheus-client
-    pytest
-    pytest-timeout
-    pytest-twisted
     testtools
-    towncrier
   ];
+
 in
 buildPythonPackage {
   inherit pname version;
@@ -68,7 +64,7 @@ buildPythonPackage {
   propagatedBuildInputs = pythonPackageDependencies;
 
   inherit doCheck;
-  checkInputs = pythonCheckDependencies;
+  checkInputs = unitTestDependencies;
   checkPhase = ''
     export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
     python -m twisted.trial -j $NIX_BUILD_CORES allmydata

From 35b921b11d1fdf23d8039a7cf4b526eb56474995 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 21 Mar 2023 08:57:21 -0400
Subject: [PATCH 054/362] Put Python package overrides in one place, and add a
 lot more of them

These packaging changes fix issues against CPython 3.11 or PyPy.
---
 default.nix              |   6 +-
 nix/python-overrides.nix | 133 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 134 insertions(+), 5 deletions(-)
 create mode 100644 nix/python-overrides.nix

diff --git a/default.nix b/default.nix
index b87a6730a..d616f63b8 100644
--- a/default.nix
+++ b/default.nix
@@ -32,11 +32,7 @@ in
 
 }:
 with (pkgs.${pythonVersion}.override {
-  packageOverrides = self: super: {
-    # Some dependencies aren't packaged in nixpkgs so supply our own packages.
-    pycddl = self.callPackage ./nix/pycddl.nix { };
-    txi2p = self.callPackage ./nix/txi2p.nix { };
-  };
+  packageOverrides = import ./nix/python-overrides.nix;
 }).pkgs;
 callPackage ./nix/tahoe-lafs.nix {
   # Select whichever package extras were requested.
diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
new file mode 100644
index 000000000..87c42ad58
--- /dev/null
+++ b/nix/python-overrides.nix
@@ -0,0 +1,133 @@
+# Override various Python packages to create a package set that works for
+# Tahoe-LAFS on CPython and PyPy.
+self: super:
+let
+
+  # Run a function on a derivation if and only if we're building for PyPy.
+  onPyPy = f: drv: if super.isPyPy then f drv else drv;
+
+  # Disable a Python package's test suite.
+  dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; });
+
+  # Disable building a Python package's documentation.
+  dontBuildDocs = alsoDisable: drv: (drv.override ({
+    sphinxHook = null;
+  } // alsoDisable)).overrideAttrs ({ outputs, ... }: {
+    outputs = builtins.filter (x: "doc" != x) outputs;
+  });
+
+in {
+  # Some dependencies aren't packaged in nixpkgs so supply our own packages.
+  pycddl = self.callPackage ./pycddl.nix { };
+  txi2p = self.callPackage ./txi2p.nix { };
+
+  # collections-extended is currently broken for Python 3.11 in nixpkgs but
+  # we know where a working version lives.
+  collections-extended = self.callPackage ./collections-extended.nix {
+    inherit (super) collections-extended;
+  };
+
+  # greenlet is incompatible with PyPy but PyPy has a builtin equivalent.
+  # Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  greenlet = onPyPy (drv: null) super.greenlet;
+
+  # tornado and tk pull in a huge dependency trees for functionality we don't
+  # care about, also tkinter doesn't work on PyPy.
+  matplotlib = super.matplotlib.override { tornado = null; enableTk = false; };
+
+  tqdm = super.tqdm.override {
+    # ibid.
+    tkinter = null;
+    # pandas is only required by the part of the test suite covering
+    # integration with pandas that we don't care about.  pandas is a huge
+    # dependency.
+    pandas = null;
+  };
+
+  # The treq test suite depends on httpbin.  httpbin pulls in babel (flask ->
+  # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow).  babel fails
+  # its test suite and arrow segfaults.
+  treq = onPyPy dontCheck super.treq;
+
+  # the six test suite fails on PyPy because it depends on dbm which the
+  # nixpkgs PyPy build appears to be missing.  Maybe fixed in nixpkgs in
+  # a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  six = onPyPy dontCheck super.six;
+
+  # Building the docs requires sphinx which brings in a dependency on babel,
+  # the test suite of which fails.
+  pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl;
+
+  # Likewise for beautifulsoup4.
+  beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
+
+  # The autobahn test suite pulls in a vast number of dependencies for
+  # functionality we don't care about.  It might be nice to *selectively*
+  # disable just some of it but this is easier.
+  autobahn = onPyPy dontCheck super.autobahn;
+
+  # and python-dotenv tests pulls in a lot of dependencies, including jedi,
+  # which does not work on PyPy.
+  python-dotenv = onPyPy dontCheck super.python-dotenv;
+
+  # Upstream package unaccountably includes a sqlalchemy dependency ... but
+  # the project has no such dependency.  Fixed in nixpkgs in
+  # da10e809fff70fbe1d86303b133b779f09f56503.
+  aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; };
+
+  # By default, the sphinx docs are built, which pulls in a lot of
+  # dependencies - including jedi, which does not work on PyPy.
+  hypothesis =
+    (let h = super.hypothesis;
+     in
+       if (h.override.__functionArgs.enableDocumentation or false)
+       then h.override { enableDocumentation = false; }
+       else h).overrideAttrs ({ nativeBuildInputs, ... }: {
+         # The nixpkgs expression is missing the tzdata check input.
+         nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ];
+       });
+
+  # flaky's test suite depends on nose and nose appears to have Python 3
+  # incompatibilities (it includes `print` statements, for example).
+  flaky = onPyPy dontCheck super.flaky;
+
+  # Replace the deprecated way of running the test suite with the modern way.
+  # This also drops a bunch of unnecessary build-time dependencies, some of
+  # which are broken on PyPy.  Fixed in nixpkgs in
+  # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea.
+  zfec = (super.zfec.override {
+    setuptoolsTrial = null;
+  }).overrideAttrs (old: {
+    checkPhase = "trial zfec";
+  });
+
+  # collections-extended is packaged with poetry-core.  poetry-core test suite
+  # uses virtualenv and virtualenv test suite fails on PyPy.
+  poetry-core = onPyPy dontCheck super.poetry-core;
+
+  # The test suite fails with some rather irrelevant (to us) string comparison
+  # failure on PyPy.  Probably a PyPy bug but doesn't seem like we should
+  # care.
+  rich = onPyPy dontCheck super.rich;
+
+  # The pyutil test suite fails in some ... test ... for some deprecation
+  # functionality we don't care about.
+  pyutil = onPyPy dontCheck super.pyutil;
+
+  # testCall1 fails fairly inscrutibly on PyPy.  Perhaps someone can fix that,
+  # or we could at least just skip that one test.  Probably better to fix it
+  # since we actually depend directly and significantly on Foolscap.
+  foolscap = onPyPy dontCheck super.foolscap;
+
+  # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246
+  psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: {
+    # Upstream already disables some tests but there are even more that have
+    # build impurities that come from build system hardware configuration.
+    # Skip them too.
+    pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray;
+    disabledTests = disabledTests ++ [ "sensors_temperatures" ];
+  });
+
+  # CircleCI build systems don't have enough memory to run this test suite.
+  lz4 = dontCheck super.lz4;
+}

From a173df4561fb3fad1898f32421553962f7659ff1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 21 Mar 2023 09:29:12 -0400
Subject: [PATCH 055/362] news fragment

---
 newsfragments/3991.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3991.minor

diff --git a/newsfragments/3991.minor b/newsfragments/3991.minor
new file mode 100644
index 000000000..e69de29bb

From a8832b11b6e365564d1b53f35bd885353d550841 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 20 Jan 2023 14:29:17 -0500
Subject: [PATCH 056/362] Start adapting language to narrow down possible
 interpretations

---
 docs/proposed/http-storage-node-protocol.rst | 113 ++++++++++++++-----
 1 file changed, 84 insertions(+), 29 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index aee201cf5..397d64ec2 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -3,7 +3,7 @@
 Storage Node Protocol ("Great Black Swamp", "GBS")
 ==================================================
 
-The target audience for this document is Tahoe-LAFS developers.
+The target audience for this document is developers working on Tahoe-LAFS or on an alternate implementation intended to be interoperable.
 After reading this document,
 one should expect to understand how Tahoe-LAFS clients interact over the network with Tahoe-LAFS storage nodes.
 
@@ -64,6 +64,10 @@ Glossary
    lease renew secret
      a short secret string which storage servers required to be presented before allowing a particular lease to be renewed
 
+The key words
+"MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED",  "MAY", and "OPTIONAL"
+in this document are to be interpreted as described in RFC 2119.
+
 Motivation
 ----------
 
@@ -119,8 +123,8 @@ An HTTP-based protocol can make use of TLS in largely the same way to provide th
 Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation
 (rather than the standard "web" rules for validation).
 
-Requirements
-------------
+Design Requirements
+-------------------
 
 Security
 ~~~~~~~~
@@ -189,6 +193,9 @@ Solutions
 An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described below.
 This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol.
 
+Summary (Non-normative)
+!!!!!!!!!!!!!!!!!!!!!!!
+
 Communication with the storage node will take place using TLS.
 The TLS version and configuration will be dictated by an ongoing understanding of best practices.
 The storage node will present an x509 certificate during the TLS handshake.
@@ -240,7 +247,7 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the *
    They are encoded with Base32 for a length of 32 bytes.
    SPKI information discussed here is 32 bytes (SHA256 digest).
    They would be encoded in Base32 for a length of 52 bytes.
-   `base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
+   `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
    This would encode the SPKI information for a length of merely 43 bytes.
    SHA1,
    the current Foolscap hash function,
@@ -332,12 +339,15 @@ Details about the interface are encoded in the HTTP message body.
 Message Encoding
 ~~~~~~~~~~~~~~~~
 
-The preferred encoding for HTTP message bodies is `CBOR`_.
-A request may be submitted using an alternate encoding by declaring this in the ``Content-Type`` header.
-A request may indicate its preference for an alternate encoding in the response using the ``Accept`` header.
-These two headers are used in the typical way for an HTTP application.
+Clients and servers MUST use the ``Content-Type`` and ``Accept`` header fields as specified in `RFC 9110`_ for message body negotiation.
 
-The only other encoding support for which is currently recommended is JSON.
+The encoding for HTTP message bodies SHOULD be `CBOR`_.
+Clients submitting requests using this encoding MUST include a ``Content-Type: application/cbor`` request header field.
+A request MAY be submitted using an alternate encoding by declaring this in the ``Content-Type`` header field.
+A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field.
+A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field.
+
+Clients and servers SHOULD support ``application/json`` request and response message body encoding.
 For HTTP messages carrying binary share data,
 this is expected to be a particularly poor encoding.
 However,
@@ -350,10 +360,19 @@ Because of the simple types used throughout
 and the equivalence described in `RFC 7049`_
 these examples should be representative regardless of which of these two encodings is chosen.
 
-The one exception is sets.
-For CBOR messages, any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
-Tag 6.258 is used to indicate sets in CBOR; see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details.
-Sets will be represented as JSON lists in examples because JSON doesn't support sets.
+One exception to this rule is for sets.
+For CBOR messages,
+any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
+Tag 6.258 is used to indicate sets in CBOR;
+see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml>`_ for more details.
+The JSON encoding does not support sets.
+Sets MUST be represented as arrays in JSON-encoded messages.
+
+Another exception to this rule is for bytes.
+The CBOR encoding natively supports a bytes type while the JSON encoding does not.
+Bytes MUST be represented as strings giving the `Base64`_ representation of the original bytes value.
+
+Clients and servers MAY support additional request and response message body encodings.
 
 HTTP Design
 ~~~~~~~~~~~
@@ -368,29 +387,49 @@ one branch contains all of the share data;
 another branch contains all of the lease data;
 etc.
 
-An ``Authorization`` header in requests is required for all endpoints.
-The standard HTTP authorization protocol is used.
-The authentication *type* used is ``Tahoe-LAFS``.
-The swissnum from the NURL used to locate the storage service is used as the *credentials*.
-If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``401 UNAUTHORIZED`` response.
+Clients and servers MUST use the ``Authorization`` header field,
+as specified in `RFC 9110`_,
+for authorization of all requests to all endpoints specified here.
+The authentication *type* MUST be ``Tahoe-LAFS``.
+Clients MUST present the swissnum from the NURL used to locate the storage service as the *credentials*.
 
-There are also, for some endpoints, secrets sent via ``X-Tahoe-Authorization`` headers.
-If these are:
+If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
+
+Requests to certain endpoints MUST include additional secrets in the ``X-Tahoe-Authorization`` headers field.
+The endpoints which require these secrets are:
+
+* ``PUT /storage/v1/lease/:storage_index``:
+  The secrets included MUST be ``lease-renew-secret`` and ``lease-cancel-secret``.
+
+* ``POST /storage/v1/immutable/:storage_index``:
+  The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``upload-secret``.
+
+* ``PATCH /storage/v1/immutable/:storage_index/:share_number``:
+  The secrets included MUST be ``upload-secret``.
+
+* ``PUT /storage/v1/immutable/:storage_index/:share_number/abort``:
+  The secrets included MUST be ``upload-secret``.
+
+* ``POST /storage/v1/mutable/:storage_index/read-test-write``:
+  The secrets included MUST be ``lease-renew-secret``, ``lease-cancel-secret``, and ``write-enabler``.
+
+If these secrets are:
 
 1. Missing.
 2. The wrong length.
 3. Not the expected kind of secret.
 4. They are otherwise unparseable before they are actually semantically used.
 
-the server will respond with ``400 BAD REQUEST``.
+the server MUST respond with ``400 BAD REQUEST`` and perform no other processing of the message.
 401 is not used because this isn't an authorization problem, this is a "you sent garbage and should know better" bug.
 
-If authorization using the secret fails, then a ``401 UNAUTHORIZED`` response should be sent.
+If authorization using the secret fails,
+then the server MUST send a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
 
 Encoding
 ~~~~~~~~
 
-* ``storage_index`` should be base32 encoded (RFC3548) in URLs.
+* ``storage_index`` MUST be `Base32`_ encoded in URLs.
 
 General
 ~~~~~~~
@@ -398,11 +437,14 @@ General
 ``GET /storage/v1/version``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Retrieve information about the version of the storage server.
-Information is returned as an encoded mapping.
-For example::
+This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service.
+The response MUST represent a mapping from schema identifiers to the metadata.
 
-  { "http://allmydata.org/tahoe/protocols/storage/v1" :
+The only schema identifier specified is ``"http://allmydata.org/tahoe/protocols/storage/v1"``.
+The server MUST include an entry in the mapping with this key.
+The value for the key MUST be another mapping with the following keys and value types::
+
+  { "http://allmydata.org/tahoe/protocols/storage/v1":
     { "maximum-immutable-share-size": 1234,
       "maximum-mutable-share-size": 1235,
       "available-space": 123456,
@@ -414,6 +456,11 @@ For example::
     "application-version": "1.13.0"
     }
 
+The server SHOULD populate as many fields as possible with accurate information about itself.
+
+XXX Document every single field
+
+
 ``PUT /storage/v1/lease/:storage_index``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
@@ -926,10 +973,18 @@ otherwise it will read a byte which won't match `b""`::
 
      204 NO CONTENT
 
+.. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4
+
+.. _Base32: https://www.rfc-editor.org/rfc/rfc4648#section-6
+
+.. _RFC 4648: https://tools.ietf.org/html/rfc4648
+
 .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4
 
 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4
 
+.. _RFC 9110: https://tools.ietf.org/html/rfc9110
+
 .. _CBOR: http://cbor.io/
 
 .. [#]
@@ -974,7 +1029,7 @@ otherwise it will read a byte which won't match `b""`::
         spki_encoded = urlsafe_b64encode(spki_sha256)
         assert spki_encoded == tub_id
 
-   Note we use `base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.
+   Note we use `unpadded base64url`_ rather than the Foolscap- and Tahoe-LAFS-preferred Base32.
 
 .. [#]
    https://www.cvedetails.com/cve/CVE-2017-5638/
@@ -985,6 +1040,6 @@ otherwise it will read a byte which won't match `b""`::
 .. [#]
    https://efail.de/
 
-.. _base64url: https://tools.ietf.org/html/rfc7515#appendix-C
+.. _unpadded base64url: https://tools.ietf.org/html/rfc7515#appendix-C
 
 .. _attacking SHA1: https://en.wikipedia.org/wiki/SHA-1#Attacks

From 7b207383088ce1f866bc6442e07b5f675ceea4b7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 23 Jan 2023 10:40:41 -0500
Subject: [PATCH 057/362] some more edits

---
 docs/proposed/http-storage-node-protocol.rst | 128 ++++++++++---------
 1 file changed, 70 insertions(+), 58 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 397d64ec2..cff6dc67b 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -438,27 +438,28 @@ General
 !!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 This endpoint allows clients to retrieve some basic metadata about a storage server from the storage service.
-The response MUST represent a mapping from schema identifiers to the metadata.
+The response MUST validate against this CDDL schema::
 
-The only schema identifier specified is ``"http://allmydata.org/tahoe/protocols/storage/v1"``.
-The server MUST include an entry in the mapping with this key.
-The value for the key MUST be another mapping with the following keys and value types::
+  {'http://allmydata.org/tahoe/protocols/storage/v1' => {
+      'maximum-immutable-share-size' => uint
+      'maximum-mutable-share-size' => uint
+      'available-space' => uint
+      'tolerates-immutable-read-overrun' => bool
+      'delete-mutable-shares-with-zero-length-writev' => bool
+      'fills-holes-with-zero-bytes' => bool
+      'prevents-read-past-end-of-share-data' => bool
+      }
+   'application-version' => bstr
+  }
 
-  { "http://allmydata.org/tahoe/protocols/storage/v1":
-    { "maximum-immutable-share-size": 1234,
-      "maximum-mutable-share-size": 1235,
-      "available-space": 123456,
-      "tolerates-immutable-read-overrun": true,
-      "delete-mutable-shares-with-zero-length-writev": true,
-      "fills-holes-with-zero-bytes": true,
-      "prevents-read-past-end-of-share-data": true
-      },
-    "application-version": "1.13.0"
-    }
+The server SHOULD populate as many fields as possible with accurate information about its behavior.
 
-The server SHOULD populate as many fields as possible with accurate information about itself.
+For fields which relate to a specific API
+the semantics are documented below in the section for that API.
+For fields that are more general than a single API the semantics are as follows:
 
-XXX Document every single field
+* available-space:
+  The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests.
 
 
 ``PUT /storage/v1/lease/:storage_index``
@@ -518,21 +519,23 @@ Writing
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Initialize an immutable storage index with some buckets.
-The buckets may have share data written to them once.
-A lease is also created for the shares.
+The server MUST allow share data to be written to the buckets at most one time.
+The server MAY create a lease for the buckets.
 Details of the buckets to create are encoded in the request body.
 For example::
 
   {"share-numbers": [1, 7, ...], "allocated-size": 12345}
 
-The request must include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
+The server SHOULD accept a value for **allocated-size** that is less than or equal to the value for the server's version message's **maximum-immutable-share-size** value.
+
+The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
 For example::
 
    X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
    X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
    X-Tahoe-Authorization: upload-secret <base64-upload-secret>
 
-The response body includes encoded information about the created buckets.
+The response body MUST include encoded information about the created buckets.
 For example::
 
   {"already-have": [1, ...], "allocated": [7, ...]}
@@ -589,26 +592,28 @@ Rejected designs for upload secrets:
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Write data for the indicated share.
-The share number must belong to the storage index.
-The request body is the raw share data (i.e., ``application/octet-stream``).
-*Content-Range* requests are required; for large transfers this allows partially complete uploads to be resumed.
+The share number MUST belong to the storage index.
+The request body MUST be the raw share data (i.e., ``application/octet-stream``).
+The request MUST include a *Content-Range* header field;
+for large transfers this allows partially complete uploads to be resumed.
+
 For example,
 a 1MiB share can be divided in to eight separate 128KiB chunks.
 Each chunk can be uploaded in a separate request.
 Each request can include a *Content-Range* value indicating its placement within the complete share.
 If any one of these requests fails then at most 128KiB of upload work needs to be retried.
 
-The server must recognize when all of the data has been received and mark the share as complete
+The server MUST recognize when all of the data has been received and mark the share as complete
 (which it can do because it was informed of the size when the storage index was initialized).
 
-The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
+The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::
 
     X-Tahoe-Authorization: upload-secret <base64-upload-secret>
 
 Responses:
 
-* When a chunk that does not complete the share is successfully uploaded the response is ``OK``.
-  The response body indicates the range of share data that has yet to be uploaded.
+* When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``.
+  The response body MUST indicate the range of share data that has yet to be uploaded.
   That is::
 
     { "required":
@@ -620,11 +625,12 @@ Responses:
       ]
     }
 
-* When the chunk that completes the share is successfully uploaded the response is ``CREATED``.
+* When the chunk that completes the share is successfully uploaded the response MUST be ``CREATED``.
 * If the *Content-Range* for a request covers part of the share that has already,
   and the data does not match already written data,
-  the response is ``CONFLICT``.
-  At this point the only thing to do is abort the upload and start from scratch (see below).
+  the response MUST be ``CONFLICT``.
+  In this case the client MUST abort the upload.
+  The client MAY then restart the upload from scratch.
 
 Discussion
 ``````````
@@ -650,34 +656,32 @@ From RFC 7231::
 
 This cancels an *in-progress* upload.
 
-The request must include a ``X-Tahoe-Authorization`` header that includes the upload secret::
+The request MUST include a ``X-Tahoe-Authorization`` header that includes the upload secret::
 
     X-Tahoe-Authorization: upload-secret <base64-upload-secret>
 
-The response code:
-
-* When the upload is still in progress and therefore the abort has succeeded,
-  the response is ``OK``.
-  Future uploads can start from scratch with no pre-existing upload state stored on the server.
-* If the uploaded has already finished, the response is 405 (Method Not Allowed)
-  and no change is made.
+If there is an incomplete upload with a matching upload-secret then the server MUST consider the abort to have succeeded.
+In this case the response MUST be ``OK``.
+The server MUST respond to all future requests as if the operations related to this upload did not take place.
 
+If there is no incomplete upload with a matching upload-secret then the server MUST respond with ``Method Not Allowed`` (405).
+The server MUST make no client-visible changes to its state in this case.
 
 ``POST /storage/v1/immutable/:storage_index/:share_number/corrupt``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Advise the server the data read from the indicated share was corrupt. The
-request body includes an human-meaningful text string with details about the
-corruption. It also includes potentially important details about the share.
+Advise the server the data read from the indicated share was corrupt.
+The request body includes an human-meaningful text string with details about the corruption.
+It also includes potentially important details about the share.
 
 For example::
 
   {"reason": "expected hash abcd, got hash efgh"}
 
-.. share-type, storage-index, and share-number are inferred from the URL
-
-The response code is OK (200) by default, or NOT FOUND (404) if the share
-couldn't be found.
+The report pertains to the immutable share with a **storage index** and **share number** given in the request path.
+If the identified **storage index** and **share number** are known to the server then the response SHOULD be accepted and made available to server administrators.
+In this case the response SHOULD be ``OK``.
+If the response is not accepted then the response SHOULD be ``Not Found`` (404).
 
 Reading
 ~~~~~~~
@@ -685,26 +689,34 @@ Reading
 ``GET /storage/v1/immutable/:storage_index/shares``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Retrieve a list (semantically, a set) indicating all shares available for the
-indicated storage index. For example::
+Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index.
+For example::
 
   [1, 5]
 
-An unknown storage index results in an empty list.
+If the **storage index** in the request path is not known to the server then the response MUST include an empty list.
 
 ``GET /storage/v1/immutable/:storage_index/:share_number``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Read a contiguous sequence of bytes from one share in one bucket.
-The response body is the raw share data (i.e., ``application/octet-stream``).
-The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
-Interpretation and response behavior is as specified in RFC 7233 § 4.1.
-Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported.
+The response body MUST be the raw share data (i.e., ``application/octet-stream``).
+The ``Range`` header MAY be used to request exactly one ``bytes`` range,
+in which case the response code MUST be ``Partial Content`` (206).
+Interpretation and response behavior MUST be as specified in RFC 7233 § 4.1.
+Multiple ranges in a single request are *not* supported;
+open-ended ranges are also not supported.
+Clients MUST NOT send requests using these features.
 
-If the response reads beyond the end of the data, the response may be shorter than the requested range.
-The resulting ``Content-Range`` header will be consistent with the returned data.
+If the response reads beyond the end of the data,
+the response MUST be shorter than the requested range.
+It MUST contain all data in the share and then end.
+The resulting ``Content-Range`` header MUST be consistent with the returned data.
 
-If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
+The server MUST indicate this behavior by specifying **True** for **tolerates-immutable-read-overrun** in its version response.
+
+If the response to a query is an empty range,
+the server MUST send a ``No Content`` (204) response.
 
 Discussion
 ``````````
@@ -743,13 +755,13 @@ The first write operation on a mutable storage index creates it
 (that is,
 there is no separate "create this storage index" operation as there is for the immutable storage index type).
 
-The request must include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets::
+The request MUST include ``X-Tahoe-Authorization`` headers with write enabler and lease secrets::
 
     X-Tahoe-Authorization: write-enabler <base64-write-enabler-secret>
     X-Tahoe-Authorization: lease-cancel-secret <base64-lease-cancel-secret>
     X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
 
-The request body includes test, read, and write vectors for the operation.
+The request body MUST include test, read, and write vectors for the operation.
 For example::
 
    {

From 98a3691891bfbe4af2871a58fba0e586400e90cf Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 25 Jan 2023 09:55:40 -0500
Subject: [PATCH 058/362] Add more CDDL to the spec; remove some server version
 flags from it

---
 docs/proposed/http-storage-node-protocol.rst | 95 ++++++++++++++++----
 src/allmydata/storage/http_client.py         | 16 +++-
 src/allmydata/storage/http_server.py         | 21 ++++-
 3 files changed, 109 insertions(+), 23 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index cff6dc67b..838f88426 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -444,10 +444,6 @@ The response MUST validate against this CDDL schema::
       'maximum-immutable-share-size' => uint
       'maximum-mutable-share-size' => uint
       'available-space' => uint
-      'tolerates-immutable-read-overrun' => bool
-      'delete-mutable-shares-with-zero-length-writev' => bool
-      'fills-holes-with-zero-bytes' => bool
-      'prevents-read-past-end-of-share-data' => bool
       }
    'application-version' => bstr
   }
@@ -522,11 +518,18 @@ Initialize an immutable storage index with some buckets.
 The server MUST allow share data to be written to the buckets at most one time.
 The server MAY create a lease for the buckets.
 Details of the buckets to create are encoded in the request body.
+The request body MUST validate against this CDDL schema::
+
+  {
+    share-numbers: #6.258([0*256 uint])
+    allocated-size: uint
+  }
+
 For example::
 
   {"share-numbers": [1, 7, ...], "allocated-size": 12345}
 
-The server SHOULD accept a value for **allocated-size** that is less than or equal to the value for the server's version message's **maximum-immutable-share-size** value.
+The server SHOULD accept a value for **allocated-size** that is less than or equal to the lesser of the values of the server's version message's **maximum-immutable-share-size** or **available-space** values.
 
 The request MUST include ``X-Tahoe-Authorization`` HTTP headers that set the various secrets—upload, lease renewal, lease cancellation—that will be later used to authorize various operations.
 For example::
@@ -536,6 +539,13 @@ For example::
    X-Tahoe-Authorization: upload-secret <base64-upload-secret>
 
 The response body MUST include encoded information about the created buckets.
+The response body MUST validate against this CDDL schema::
+
+  {
+    already-have: #6.258([0*256 uint])
+    allocated: #6.258([0*256 uint])
+  }
+
 For example::
 
   {"already-have": [1, ...], "allocated": [7, ...]}
@@ -614,7 +624,13 @@ Responses:
 
 * When a chunk that does not complete the share is successfully uploaded the response MUST be ``OK``.
   The response body MUST indicate the range of share data that has yet to be uploaded.
-  That is::
+  The response body MUST validate against this CDDL schema::
+
+    {
+      required: [0* {begin: uint, end: uint}]
+    }
+
+  For example::
 
     { "required":
       [ { "begin": <byte position, inclusive>
@@ -673,6 +689,11 @@ The server MUST make no client-visible changes to its state in this case.
 Advise the server the data read from the indicated share was corrupt.
 The request body includes an human-meaningful text string with details about the corruption.
 It also includes potentially important details about the share.
+The request body MUST validate against this CDDL schema::
+
+  {
+    reason: tstr
+  }
 
 For example::
 
@@ -690,6 +711,10 @@ Reading
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Retrieve a list (semantically, a set) indicating all shares available for the indicated storage index.
+The response body MUST validate against this CDDL schema::
+
+  #6.258([0*256 uint])
+
 For example::
 
   [1, 5]
@@ -710,11 +735,9 @@ Clients MUST NOT send requests using these features.
 
 If the response reads beyond the end of the data,
 the response MUST be shorter than the requested range.
-It MUST contain all data in the share and then end.
+It MUST contain all data up to the end of the share and then end.
 The resulting ``Content-Range`` header MUST be consistent with the returned data.
 
-The server MUST indicate this behavior by specifying **True** for **tolerates-immutable-read-overrun** in its version response.
-
 If the response to a query is an empty range,
 the server MUST send a ``No Content`` (204) response.
 
@@ -762,6 +785,20 @@ The request MUST include ``X-Tahoe-Authorization`` headers with write enabler an
     X-Tahoe-Authorization: lease-renew-secret <base64-lease-renew-secret>
 
 The request body MUST include test, read, and write vectors for the operation.
+The request body MUST validate against this CDDL schema::
+
+  {
+    "test-write-vectors": {
+      0*256 share_number : {
+        "test": [0*30 {"offset": uint, "size": uint, "specimen": bstr}]
+        "write": [* {"offset": uint, "data": bstr}]
+        "new-length": uint / null
+      }
+    }
+    "read-vector": [0*30 {"offset": uint, "size": uint}]
+  }
+  share_number = uint
+
 For example::
 
    {
@@ -784,6 +821,14 @@ For example::
 
 The response body contains a boolean indicating whether the tests all succeed
 (and writes were applied) and a mapping giving read data (pre-write).
+The response body MUST validate against this CDDL schema::
+
+  {
+    "success": bool,
+    "data": {0*256 share_number: [0* bstr]}
+  }
+  share_number = uint
+
 For example::
 
   {
@@ -795,7 +840,7 @@ For example::
       }
   }
 
-A test vector or read vector that read beyond the boundaries of existing data will return nothing for any bytes past the end.
+A server MUST return nothing for any bytes beyond the end of existing data for a test vector or read vector that reads tries to read such data.
 As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length.
 
 Reading
@@ -805,23 +850,34 @@ Reading
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Retrieve a set indicating all shares available for the indicated storage index.
-For example (this is shown as list, since it will be list for JSON, but will be set for CBOR)::
+The response body MUST validate against this CDDL schema::
+
+  #6.258([0*256 uint])
+
+For example::
 
   [1, 5]
 
 ``GET /storage/v1/mutable/:storage_index/:share_number``
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``
+Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``.
 
-The ``Range`` header may be used to request exactly one ``bytes`` range, in which case the response code will be 206 (partial content).
-Interpretation and response behavior is as specified in RFC 7233 § 4.1.
-Multiple ranges in a single request are *not* supported; open-ended ranges are also not supported.
+The response body MUST be the raw share data (i.e., ``application/octet-stream``).
+The ``Range`` header MAY be used to request exactly one ``bytes`` range,
+in which case the response code MUST be ``Partial Content`` (206).
+Interpretation and response behavior MUST be specified in RFC 7233 § 4.1.
+Multiple ranges in a single request are *not* supported;
+open-ended ranges are also not supported.
+Clients MUST NOT send requests using these features.
 
-If the response reads beyond the end of the data, the response may be shorter than the requested range.
-The resulting ``Content-Range`` header will be consistent with the returned data.
+If the response reads beyond the end of the data,
+the response MUST be shorter than the requested range.
+It MUST contain all data up to the end of the share and then end.
+The resulting ``Content-Range`` header MUST be consistent with the returned data.
 
-If the response to a query is an empty range, the ``NO CONTENT`` (204) response code will be used.
+If the response to a query is an empty range,
+the server MUST send a ``No Content`` (204) response.
 
 
 ``POST /storage/v1/mutable/:storage_index/:share_number/corrupt``
@@ -833,6 +889,9 @@ Just like the immutable version.
 Sample Interactions
 -------------------
 
+This section contains examples of client/server interactions to help illuminate the above specification.
+This section is non-normative.
+
 Immutable Data
 ~~~~~~~~~~~~~~
 
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 90bda7fc0..2f4f8398e 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -70,15 +70,14 @@ class ClientException(Exception):
 # indicates a set.
 _SCHEMAS = {
     "get_version": Schema(
+        # Note that the single-quoted (`'`) string keys in this schema
+        # represent *byte* strings - per the CDDL specification.  Text strings
+        # are represented using strings with *double* quotes (`"`).
         """
         response = {'http://allmydata.org/tahoe/protocols/storage/v1' => {
                  'maximum-immutable-share-size' => uint
                  'maximum-mutable-share-size' => uint
                  'available-space' => uint
-                 'tolerates-immutable-read-overrun' => bool
-                 'delete-mutable-shares-with-zero-length-writev' => bool
-                 'fills-holes-with-zero-bytes' => bool
-                 'prevents-read-past-end-of-share-data' => bool
                  }
                  'application-version' => bstr
               }
@@ -447,6 +446,15 @@ class StorageClientGeneral(object):
         decoded_response = yield self._client.decode_cbor(
             response, _SCHEMAS["get_version"]
         )
+        # Add some features we know are true because the HTTP API
+        # specification requires them and because other parts of the storage
+        # client implementation assumes they will be present.
+        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update({
+            b'tolerates-immutable-read-overrun': True,
+            b'delete-mutable-shares-with-zero-length-writev': True,
+            b'fills-holes-with-zero-bytes': True,
+            b'prevents-read-past-end-of-share-data': True,
+        })
         returnValue(decoded_response)
 
     @inlineCallbacks
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index fd7fd1187..b7ca2d971 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -592,7 +592,26 @@ class HTTPServer(object):
     @_authorized_route(_app, set(), "/storage/v1/version", methods=["GET"])
     def version(self, request, authorization):
         """Return version information."""
-        return self._send_encoded(request, self._storage_server.get_version())
+        return self._send_encoded(request, self._get_version())
+
+    def _get_version(self) -> dict[str, Any]:
+        """
+        Get the HTTP version of the storage server's version response.
+
+        This differs from the Foolscap version by omitting certain obsolete
+        fields.
+        """
+        v = self._storage_server.get_version()
+        v1_identifier = b"http://allmydata.org/tahoe/protocols/storage/v1"
+        v1 = v[v1_identifier]
+        return {
+            v1_identifier: {
+                b"maximum-immutable-share-size": v1[b"maximum-immutable-share-size"],
+                b"maximum-mutable-share-size": v1[b"maximum-mutable-share-size"],
+                b"available-space": v1[b"available-space"],
+            },
+            b"application-version": v[b"application-version"],
+        }
 
     ##### Immutable APIs #####
 

From 48a2d4d31d86bc39c2e1caa06f6e9c3e6baac741 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Feb 2023 13:58:58 -0500
Subject: [PATCH 059/362] ``Authorization`` is the right header field

---
 src/allmydata/storage/http_common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py
index 123ce403b..e5f07898e 100644
--- a/src/allmydata/storage/http_common.py
+++ b/src/allmydata/storage/http_common.py
@@ -28,7 +28,7 @@ def get_content_type(headers: Headers) -> Optional[str]:
 
 
 def swissnum_auth_header(swissnum: bytes) -> bytes:
-    """Return value for ``Authentication`` header."""
+    """Return value for ``Authorization`` header."""
     return b"Tahoe-LAFS " + b64encode(swissnum).strip()
 
 

From 8645462f4e1b44507e9dcde63c05ff3ef9f30453 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 17 Feb 2023 14:00:03 -0500
Subject: [PATCH 060/362] Base64 encode the swissnum

Typically swissnums themselves are base32 encoded but there's no requirement
that this is the case.  Base64 encoding in the header ensures we can represent
whatever the value was.
---
 docs/proposed/http-storage-node-protocol.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 838f88426..7f678d271 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -391,7 +391,7 @@ Clients and servers MUST use the ``Authorization`` header field,
 as specified in `RFC 9110`_,
 for authorization of all requests to all endpoints specified here.
 The authentication *type* MUST be ``Tahoe-LAFS``.
-Clients MUST present the swissnum from the NURL used to locate the storage service as the *credentials*.
+Clients MUST present the `Base64`_-encoded representation of the swissnum from the NURL used to locate the storage service as the *credentials*.
 
 If credentials are not presented or the swissnum is not associated with a storage service then the server MUST issue a ``401 UNAUTHORIZED`` response and perform no other processing of the message.
 

From 369d26f0f8c4975c7855e43ce9033caf893c50f0 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 10 Mar 2023 11:17:09 -0500
Subject: [PATCH 061/362] There is a limit to the size of the corruption report
 a server must accept

---
 docs/proposed/http-storage-node-protocol.rst | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 7f678d271..4f5a53906 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -692,7 +692,7 @@ It also includes potentially important details about the share.
 The request body MUST validate against this CDDL schema::
 
   {
-    reason: tstr
+    reason: tstr .size (1..32765)
   }
 
 For example::
@@ -704,6 +704,11 @@ If the identified **storage index** and **share number** are known to the server
 In this case the response SHOULD be ``OK``.
 If the response is not accepted then the response SHOULD be ``Not Found`` (404).
 
+Discussion
+``````````
+
+The seemingly odd length limit on ``reason`` is chosen so that the *encoded* representation of the message is limited to 32768.
+
 Reading
 ~~~~~~~
 

From b27946c3c6b590667ee54f43f61bc72e57780d6d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 10 Mar 2023 11:17:23 -0500
Subject: [PATCH 062/362] trim overlong section marker

---
 docs/proposed/http-storage-node-protocol.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 4f5a53906..6e5b85716 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -864,7 +864,7 @@ For example::
   [1, 5]
 
 ``GET /storage/v1/mutable/:storage_index/:share_number``
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
 Read data from the indicated mutable shares, just like ``GET /storage/v1/immutable/:storage_index``.
 

From c3afab15ed43a729a8517e7ded6a6877b3c765f0 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 09:22:42 -0400
Subject: [PATCH 063/362] correct version type annotation

---
 src/allmydata/storage/http_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index b7ca2d971..5560d3a73 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -594,7 +594,7 @@ class HTTPServer(object):
         """Return version information."""
         return self._send_encoded(request, self._get_version())
 
-    def _get_version(self) -> dict[str, Any]:
+    def _get_version(self) -> dict[bytes, Any]:
         """
         Get the HTTP version of the storage server's version response.
 

From 7859ba733717dbc75b98554311cf7a59733ed5f7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 09:25:49 -0400
Subject: [PATCH 064/362] fix title level inconsistency

---
 docs/proposed/http-storage-node-protocol.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 6e5b85716..c9bdf3013 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -194,7 +194,7 @@ An HTTP-based protocol, dubbed "Great Black Swamp" (or "GBS"), is described belo
 This protocol aims to satisfy the above requirements at a lower level of complexity than the current Foolscap-based protocol.
 
 Summary (Non-normative)
-!!!!!!!!!!!!!!!!!!!!!!!
+~~~~~~~~~~~~~~~~~~~~~~~
 
 Communication with the storage node will take place using TLS.
 The TLS version and configuration will be dictated by an ongoing understanding of best practices.

From 5facd06725d2f0c11e497c84e4d90e90bc37dd95 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 10:42:30 -0400
Subject: [PATCH 065/362] adjust markup to clarify the encoding exceptions

---
 docs/proposed/http-storage-node-protocol.rst | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index c9bdf3013..f6d90526e 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -347,6 +347,8 @@ A request MAY be submitted using an alternate encoding by declaring this in the
 A request MAY indicate its preference for an alternate encoding in the response using the ``Accept`` header field.
 A request which includes no ``Accept`` header field MUST be interpreted in the same way as a request including a ``Accept: application/cbor`` header field.
 
+Clients and servers MAY support additional request and response message body encodings.
+
 Clients and servers SHOULD support ``application/json`` request and response message body encoding.
 For HTTP messages carrying binary share data,
 this is expected to be a particularly poor encoding.
@@ -360,7 +362,11 @@ Because of the simple types used throughout
 and the equivalence described in `RFC 7049`_
 these examples should be representative regardless of which of these two encodings is chosen.
 
-One exception to this rule is for sets.
+There are two exceptions to this rule.
+
+1. Sets
+!!!!!!!
+
 For CBOR messages,
 any sequence that is semantically a set (i.e. no repeated values allowed, order doesn't matter, and elements are hashable in Python) should be sent as a set.
 Tag 6.258 is used to indicate sets in CBOR;
@@ -368,12 +374,12 @@ see `the CBOR registry <https://www.iana.org/assignments/cbor-tags/cbor-tags.xht
 The JSON encoding does not support sets.
 Sets MUST be represented as arrays in JSON-encoded messages.
 
-Another exception to this rule is for bytes.
+2. Bytes
+!!!!!!!!
+
 The CBOR encoding natively supports a bytes type while the JSON encoding does not.
 Bytes MUST be represented as strings giving the `Base64`_ representation of the original bytes value.
 
-Clients and servers MAY support additional request and response message body encodings.
-
 HTTP Design
 ~~~~~~~~~~~
 

From c0a8be584398c2ddd85c7832cb5bc5a55e0bab00 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 10:44:09 -0400
Subject: [PATCH 066/362] nail it down

---
 docs/proposed/http-storage-node-protocol.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index f6d90526e..f9f2cd868 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -436,6 +436,7 @@ Encoding
 ~~~~~~~~
 
 * ``storage_index`` MUST be `Base32`_ encoded in URLs.
+* ``share_number`` MUST be a decimal representation
 
 General
 ~~~~~~~

From 6dc6d6f39f35fe6f51002b46b90727381b142e04 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 11:06:16 -0400
Subject: [PATCH 067/362] inline the actual base32 alphabet we use

---
 docs/proposed/http-storage-node-protocol.rst | 100 ++++++++++++++++++-
 1 file changed, 96 insertions(+), 4 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index f9f2cd868..21e27d7dd 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -244,9 +244,9 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the *
 .. note::
 
    Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate).
-   They are encoded with Base32 for a length of 32 bytes.
+   They are encoded with `Base32_` for a length of 32 bytes.
    SPKI information discussed here is 32 bytes (SHA256 digest).
-   They would be encoded in Base32 for a length of 52 bytes.
+   They would be encoded in `Base32`_ for a length of 52 bytes.
    `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible.
    This would encode the SPKI information for a length of merely 43 bytes.
    SHA1,
@@ -336,6 +336,100 @@ and shares.
 A particular resource is addressed by the HTTP request path.
 Details about the interface are encoded in the HTTP message body.
 
+String Encoding
+~~~~~~~~~~~~~~~
+
+.. _Base32:
+
+Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6.
+
+That is, the alphabet is:
+
+.. list-table:: Base32 Alphabet
+		:header-rows: 1
+
+   * - Value
+     - Encoding
+     - Value
+     - Encoding
+     - Value
+     - Encoding
+     - Value
+     - Encoding
+
+   * - 0
+     - a
+     - 9
+     - j
+     - 18
+     - s
+     - 27
+     - 3
+   * - 1
+     - b
+     - 10
+     - k
+     - 19
+     - t
+     - 28
+     - 4
+   * - 2
+     - c
+     - 11
+     - l
+     - 20
+     - u
+     - 29
+     - 5
+   * - 3
+     - d
+     - 12
+     - m
+     - 21
+     - v
+     - 30
+     - 6
+   * - 4
+     - e
+     - 13
+     - n
+     - 22
+     - w
+     - 31
+     - 7
+   * - 5
+     - f
+     - 14
+     - o
+     - 23
+     - x
+     -
+     -
+   * - 6
+     - g
+     - 15
+     - p
+     - 24
+     - y
+     -
+     -
+   * - 7
+     - h
+     - 16
+     - q
+     - 25
+     - z
+     -
+     -
+   * - 8
+     - i
+     - 17
+     - r
+     - 26
+     - 2
+     -
+     -
+
 Message Encoding
 ~~~~~~~~~~~~~~~~
 
@@ -1058,8 +1152,6 @@ otherwise it will read a byte which won't match `b""`::
 
 .. _Base64: https://www.rfc-editor.org/rfc/rfc4648#section-4
 
-.. _Base32: https://www.rfc-editor.org/rfc/rfc4648#section-6
-
 .. _RFC 4648: https://tools.ietf.org/html/rfc4648
 
 .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4

From 6771ca8ce4caf34cceacd806c8c7c45eb80af315 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 13:29:53 -0400
Subject: [PATCH 068/362] fix table markup

---
 docs/proposed/http-storage-node-protocol.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 21e27d7dd..ebe39578c 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -346,7 +346,7 @@ Where the specification refers to Base32 the meaning is *unpadded* Base32 encodi
 That is, the alphabet is:
 
 .. list-table:: Base32 Alphabet
-		:header-rows: 1
+   :header-rows: 1
 
    * - Value
      - Encoding

From fe0e159e52712c14557e0c188798f8286e33ca65 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 13:30:32 -0400
Subject: [PATCH 069/362] Give base32 a section heading

We don't have any other sections but ... :shrug:
---
 docs/proposed/http-storage-node-protocol.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index ebe39578c..f81b2bc79 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -341,6 +341,9 @@ String Encoding
 
 .. _Base32:
 
+Base32
+!!!!!!
+
 Where the specification refers to Base32 the meaning is *unpadded* Base32 encoding as specified by `RFC 4648`_ using a *lowercase variation* of the alphabet from Section 6.
 
 That is, the alphabet is:

From 6a0a895ee88e34da3c798acc19c5800af3fda414 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 13 Mar 2023 13:37:01 -0400
Subject: [PATCH 070/362] Encode the reason limit in the implementation as well

---
 src/allmydata/storage/http_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 5560d3a73..3ae16ae5c 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -273,7 +273,7 @@ _SCHEMAS = {
     "advise_corrupt_share": Schema(
         """
     request = {
-      reason: tstr
+      reason: tstr .size (1..32765)
     }
     """
     ),

From e98967731952208896e78cf8e1b697b367c6f8a0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Mar 2023 11:20:25 -0400
Subject: [PATCH 071/362] Pass in a pool instead of pool options.

---
 src/allmydata/storage/http_client.py | 11 ++++-------
 src/allmydata/storage_client.py      |  7 ++++---
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 3edf5f835..1d798fecc 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -311,21 +311,18 @@ class StorageClient(object):
 
     @classmethod
     def from_nurl(
-        cls, nurl: DecodedURL, reactor, persistent=True, retryAutomatically=True
+        cls, nurl: DecodedURL, reactor, pool: Optional[HTTPConnectionPool] = None
     ) -> StorageClient:
         """
         Create a ``StorageClient`` for the given NURL.
-
-        ``persistent`` and ``retryAutomatically`` arguments are passed to the
-        new HTTPConnectionPool.
         """
         assert nurl.fragment == "v=1"
         assert nurl.scheme == "pb"
         swissnum = nurl.path[0].encode("ascii")
         certificate_hash = nurl.user.encode("ascii")
-        pool = HTTPConnectionPool(reactor, persistent=persistent)
-        pool.retryAutomatically = retryAutomatically
-        pool.maxPersistentPerHost = 20
+        if pool is None:
+            pool = HTTPConnectionPool(reactor)
+            pool.maxPersistentPerHost = 20
 
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
             cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 2888b10e7..19d6ef4a7 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -43,6 +43,7 @@ from configparser import NoSectionError
 
 import attr
 from hyperlink import DecodedURL
+from twisted.web.client import HTTPConnectionPool
 from zope.interface import (
     Attribute,
     Interface,
@@ -1205,10 +1206,10 @@ class HTTPNativeStorageServer(service.MultiService):
             def request(reactor, nurl: DecodedURL):
                 # Since we're just using this one off to check if the NURL
                 # works, no need for persistent pool or other fanciness.
+                pool = HTTPConnectionPool(reactor, persistent=False)
+                pool.retryAutomatically = False
                 return StorageClientGeneral(
-                    StorageClient.from_nurl(
-                        nurl, reactor, persistent=False, retryAutomatically=False
-                    )
+                    StorageClient.from_nurl(nurl, reactor, pool)
                 ).get_version()
 
             # LoopingCall.stop() doesn't cancel Deferreds, unfortunately:

From b65bc9dca70048d6e1c41abbf47d93a4f4b30d3e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Mar 2023 11:22:43 -0400
Subject: [PATCH 072/362] Better explanation.

---
 src/allmydata/storage_client.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 19d6ef4a7..a52bb3f75 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1038,7 +1038,10 @@ def _pick_a_http_server(
         # Logging errors breaks a bunch of tests, and it's not a _bug_ to
         # have a failed connection, it's often expected and transient. More
         # of a warning, really?
-        log.msg("Failed to connect to NURL: {}".format(failure))
+        log.msg(
+            "Failed to connect to a storage server advertised by NURL: {}".format(
+                failure)
+        )
         return None
 
     def succeeded(result: tuple[int, DecodedURL]):

From 7ae8b50d14465a6b4fa3d7f5b35bfdbf5056f28a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Mar 2023 11:26:40 -0400
Subject: [PATCH 073/362] Async!

---
 src/allmydata/storage_client.py | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index a52bb3f75..dffa78bc4 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1019,11 +1019,12 @@ class NativeStorageServer(service.MultiService):
         self._reconnector.reset()
 
 
-def _pick_a_http_server(
+@async_to_deferred
+async def _pick_a_http_server(
         reactor,
         nurls: list[DecodedURL],
         request: Callable[[Any, DecodedURL], defer.Deferred[Any]]
-) -> defer.Deferred[Optional[DecodedURL]]:
+) -> Optional[DecodedURL]:
     """Pick the first server we successfully send a request to.
 
     Fires with ``None`` if no server was found, or with the ``DecodedURL`` of
@@ -1034,22 +1035,19 @@ def _pick_a_http_server(
         for nurl in nurls
     ])
 
-    def failed(failure: Failure):
+    try:
+        _, nurl = await queries
+        return nurl
+    except Exception as e:
         # Logging errors breaks a bunch of tests, and it's not a _bug_ to
         # have a failed connection, it's often expected and transient. More
         # of a warning, really?
         log.msg(
             "Failed to connect to a storage server advertised by NURL: {}".format(
-                failure)
+                e)
         )
         return None
 
-    def succeeded(result: tuple[int, DecodedURL]):
-        _, nurl = result
-        return nurl
-
-    return queries.addCallbacks(succeeded, failed)
-
 
 @implementer(IServer)
 class HTTPNativeStorageServer(service.MultiService):

From 14aeaea02223970c1cbc2afbb7499a22231fbb62 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Mar 2023 11:29:19 -0400
Subject: [PATCH 074/362] Another todo.

---
 src/allmydata/storage_client.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index dffa78bc4..c88613803 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1201,7 +1201,11 @@ class HTTPNativeStorageServer(service.MultiService):
         if self._istorage_server is None:
             # We haven't selected a server yet, so let's do so.
 
-            # TODO The problem with this scheme is that while picking
+            # TODO This is somewhat inefficient on startup: it takes two successful
+            # version() calls before we are live talking to a server, it could only
+            # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992
+
+            # TODO Another problem with this scheme is that while picking
             # the HTTP server to talk to, we don't have connection status
             # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
             def request(reactor, nurl: DecodedURL):

From 264269f409bb7e567c4632335ed1d8a3a20faef4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 21 Mar 2023 11:29:50 -0400
Subject: [PATCH 075/362] Better test name.

---
 src/allmydata/test/test_storage_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 8273966ce..91668e7ca 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -819,7 +819,7 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
         })
         self.assertEqual(result, earliest_url)
 
-    def test_failures_are_retried(self):
+    def test_failures_are_turned_into_none(self):
         """
         If the requests all fail, ``_pick_a_http_server`` returns ``None``.
         """

From 44f5057ed39cba4f853ad3aaf862244323b29858 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 08:07:59 -0400
Subject: [PATCH 076/362] fix link markup

---
 docs/proposed/http-storage-node-protocol.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index f81b2bc79..493cf8f58 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -244,7 +244,7 @@ When Bob's client issues HTTP requests to Alice's storage node it includes the *
 .. note::
 
    Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate).
-   They are encoded with `Base32_` for a length of 32 bytes.
+   They are encoded with `Base32`_ for a length of 32 bytes.
    SPKI information discussed here is 32 bytes (SHA256 digest).
    They would be encoded in `Base32`_ for a length of 52 bytes.
    `unpadded base64url`_ provides a more compact encoding of the information while remaining URL-compatible.

From 7c0b21916f376f139e2569242e443fea60c40723 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 08:35:17 -0400
Subject: [PATCH 077/362] specify the unit of `available-space`

---
 docs/proposed/http-storage-node-protocol.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 493cf8f58..3e74c94d6 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -560,6 +560,7 @@ For fields that are more general than a single API the semantics are as follows:
 
 * available-space:
   The server SHOULD use this field to advertise the amount of space that it currently considers unused and is willing to allocate for client requests.
+  The value is a number of bytes.
 
 
 ``PUT /storage/v1/lease/:storage_index``

From e7ed17af17c7c77daa24203954ff2ef2198875c6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 08:42:32 -0400
Subject: [PATCH 078/362] fix some editing errors about overreads and generally
 try to clarify

---
 docs/proposed/http-storage-node-protocol.rst | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst
index 3e74c94d6..5009a992e 100644
--- a/docs/proposed/http-storage-node-protocol.rst
+++ b/docs/proposed/http-storage-node-protocol.rst
@@ -950,8 +950,17 @@ For example::
       }
   }
 
-A server MUST return nothing for any bytes beyond the end of existing data for a test vector or read vector that reads tries to read such data.
-As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length.
+A client MAY send a test vector or read vector to bytes beyond the end of existing data.
+In this case a server MUST behave as if the test or read vector referred to exactly as much data exists.
+
+For example,
+consider the case where the server has 5 bytes of data for a particular share.
+If a client sends a read vector with an ``offset`` of 1 and a ``size`` of 4 then the server MUST respond with all of the data except the first byte.
+If a client sends a read vector with the same ``offset`` and a ``size`` of 5 (or any larger value) then the server MUST respond in the same way.
+
+Similarly,
+if there is no data at all,
+an empty byte string is returned no matter what the offset or length.
 
 Reading
 ~~~~~~~

From c49aa446552f3060b4f53bddd300e288be1eb21d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 09:04:15 -0400
Subject: [PATCH 079/362] Update the raw number and give a reference for
 interpretation

---
 docs/performance.rst             |  5 +++--
 docs/specifications/dirnodes.rst | 10 +++++-----
 src/allmydata/client.py          |  2 --
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/docs/performance.rst b/docs/performance.rst
index 6ddeb1fe8..a0487c72c 100644
--- a/docs/performance.rst
+++ b/docs/performance.rst
@@ -82,8 +82,9 @@ network: A
 
 memory footprint: N/K*A
 
-notes: Tahoe-LAFS generates a new RSA keypair for each mutable file that it
-publishes to a grid. This takes up to 1 or 2 seconds on a typical desktop PC.
+notes:
+Tahoe-LAFS generates a new RSA keypair for each mutable file that it publishes to a grid.
+This takes around 100 milliseconds on a relatively high-end laptop from 2021.
 
 Part of the process of encrypting, encoding, and uploading a mutable file to a
 Tahoe-LAFS grid requires that the entire file be in memory at once. For larger
diff --git a/docs/specifications/dirnodes.rst b/docs/specifications/dirnodes.rst
index 88fcd0fa9..c53d28a26 100644
--- a/docs/specifications/dirnodes.rst
+++ b/docs/specifications/dirnodes.rst
@@ -267,7 +267,7 @@ How well does this design meet the goals?
    value, so there are no opportunities for staleness
 9. monotonicity: VERY: the single point of access also protects against
    retrograde motion
-     
+
 
 
 Confidentiality leaks in the storage servers
@@ -332,8 +332,9 @@ MDMF design rules allow for efficient random-access reads from the middle of
 the file, which would give the index something useful to point at.
 
 The current SDMF design generates a new RSA public/private keypair for each
-directory. This takes considerable time and CPU effort, generally one or two
-seconds per directory. We have designed (but not yet built) a DSA-based
+directory. This takes some time and CPU effort (around 100 milliseconds on a
+relatively high-end 2021 laptop) per directory.
+We have designed (but not yet built) a DSA-based
 mutable file scheme which will use shared parameters to reduce the
 directory-creation effort to a bare minimum (picking a random number instead
 of generating two random primes).
@@ -363,7 +364,7 @@ single child, looking up a single child) would require pulling or pushing a
 lot of unrelated data, increasing network overhead (and necessitating
 test-and-set semantics for the modification side, which increases the chances
 that a user operation will fail, making it more challenging to provide
-promises of atomicity to the user). 
+promises of atomicity to the user).
 
 It would also make it much more difficult to enable the delegation
 ("sharing") of specific directories. Since each aggregate "realm" provides
@@ -469,4 +470,3 @@ Preventing delegation between communication parties is just as pointless as
 asking Bob to forget previously accessed files. However, there may be value
 to configuring the UI to ask Carol to not share files with Bob, or to
 removing all files from Bob's view at the same time his access is revoked.
-
diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 2adf59660..8a10fe9e7 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -175,8 +175,6 @@ class KeyGenerator(object):
         """I return a Deferred that fires with a (verifyingkey, signingkey)
         pair. The returned key will be 2048 bit"""
         keysize = 2048
-        # RSA key generation for a 2048 bit key takes between 0.8 and 3.2
-        # secs
         signer, verifier = rsa.create_signing_keypair(keysize)
         return defer.succeed( (verifier, signer) )
 

From c1de2efd2d97d4bc79afb40fe0f9dfe6c450b01b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 09:04:31 -0400
Subject: [PATCH 080/362] news fragment

---
 newsfragments/3993.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3993.minor

diff --git a/newsfragments/3993.minor b/newsfragments/3993.minor
new file mode 100644
index 000000000..e69de29bb

From 8d0869f6140f2c701aa856238c6a66243d301111 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 09:30:52 -0400
Subject: [PATCH 081/362] Factor some shared pieces of CircleCI configuration
 out

* Take DOCKERHUB_CONTEXT off of the single arbitrary job it was hung on and
  make it standalone.  This isolates it from future changes to that particular
  job.

* Take DOCKERHUB_AUTH out of `jobs` so it doesn't need a lot of extra
  boilerplate to pass schema validation.

* Give the "nixos" job a Python version parameter so it can be instantiated
  multiple times to test multiple Python versions.  Change the "NixOS
  unstable" instantiation to use Python 3.11 as a demonstration.

* Move a lot of the implementation of the "nixos" job into a "nix" executor
  and a "nix-build" command that, together, do the generic setup required to do
  any nix-based builds.
---
 .circleci/config.yml | 215 +++++++++++++++++++++++++++----------------
 1 file changed, 135 insertions(+), 80 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ab0573a3f..d39e3de56 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -11,19 +11,30 @@
 #
 version: 2.1
 
-# A template that can be shared between the two different image-building
+# Every job that pushes a Docker image from Docker Hub must authenticate to
+# it.  Define a couple yaml anchors that can be used to supply a the necessary credentials.
+
+# First is a CircleCI job context which makes Docker Hub credentials available
+# in the environment.
+#
+# Contexts are managed in the CircleCI web interface:
+#
+#  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
+dockerhub-context-template: &DOCKERHUB_CONTEXT
+  context: "dockerhub-auth"
+
+# Next is a Docker executor template that gets the credentials from the
+# environment and supplies them to the executor.
+dockerhub-auth-template: &DOCKERHUB_AUTH
+  - auth:
+      username: $DOCKERHUB_USERNAME
+      password: $DOCKERHUB_PASSWORD
+
+  # A template that can be shared between the two different image-building
 # workflows.
 .images: &IMAGES
   jobs:
-    # Every job that pushes a Docker image from Docker Hub needs to provide
-    # credentials.  Use this first job to define a yaml anchor that can be
-    # used to supply a CircleCI job context which makes Docker Hub credentials
-    # available in the environment.
-    #
-    # Contexts are managed in the CircleCI web interface:
-    #
-    #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-    - "build-image-debian-11": &DOCKERHUB_CONTEXT
+    - "build-image-debian-11":
         <<: *DOCKERHUB_CONTEXT
     - "build-image-ubuntu-20-04":
         <<: *DOCKERHUB_CONTEXT
@@ -71,12 +82,20 @@ workflows:
           {}
 
       - "nixos":
-          name: "NixOS 22.11"
+          name: "<<matrix.pythonVersion>>"
           nixpkgs: "22.11"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python310"
 
       - "nixos":
-          name: "NixOS unstable"
+          name: "<<matrix.pythonVersion>>"
           nixpkgs: "unstable"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python311"
 
       # Eventually, test against PyPy 3.8
       #- "pypy27-buster":
@@ -113,30 +132,7 @@ workflows:
     # Build as part of the workflow but only if requested.
     when: "<< pipeline.parameters.build-images >>"
 
-
 jobs:
-  dockerhub-auth-template:
-    # This isn't a real job.  It doesn't get scheduled as part of any
-    # workflow.  Instead, it's just a place we can hang a yaml anchor to
-    # finish the Docker Hub authentication configuration.  Workflow jobs using
-    # the DOCKERHUB_CONTEXT anchor will have access to the environment
-    # variables used here.  These variables will allow the Docker Hub image
-    # pull to be authenticated and hopefully avoid hitting and rate limits.
-    docker: &DOCKERHUB_AUTH
-      - image: "null"
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
-
-    steps:
-      - run:
-          name: "CircleCI YAML schema conformity"
-          command: |
-            # This isn't a real command.  We have to have something in this
-            # space, though, or the CircleCI yaml schema validator gets angry.
-            # Since this job is never scheduled this step is never run so the
-            # actual value here is irrelevant.
-
   codechecks:
     docker:
       - <<: *DOCKERHUB_AUTH
@@ -374,56 +370,32 @@ jobs:
           Reference the name of a niv-managed nixpkgs source (see `niv show`
           and nix/sources.json)
         type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"
 
-    docker:
-      # Run in a highly Nix-capable environment.
-      - <<: *DOCKERHUB_AUTH
-        image: "nixos/nix:2.10.3"
-
-    environment:
-      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
-      # allows us to push to CACHIX_NAME.  We only need this set for
-      # `cachix use` in this step.
-      CACHIX_NAME: "tahoe-lafs-opensource"
+    executor: "nix"
 
     steps:
-      - "run":
-          # Get cachix for Nix-friendly caching.
-          name: "Install Basic Dependencies"
-          command: |
-            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
-            nix-env \
-              --file $NIXPKGS \
-              --install \
-              -A cachix bash
-              # Activate it for "binary substitution".  This sets up
-              # configuration tht lets Nix download something from the cache
-              # instead of building it locally, if possible.
-              cachix use "${CACHIX_NAME}"
+      - "nix-build":
+          nixpkgs: "<<parameters.nixpkgs>>"
+          pythonVersion: "<<parameters.pythonVersion>>"
+          buildSteps:
+            - "run":
+                name: "Unit Test"
+                command: |
+                  # The dependencies are all built so we can allow more
+                  # parallelism here.
+                  source .circleci/lib.sh
+                  cache_if_able nix-build \
+                    --cores 8 \
+                    --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+                    --argstr pythonVersion "<<parameters.pythonVersion>>" \
+                    nix/unittests.nix
 
-      - "checkout"
 
-      - "run":
-          # The Nix package doesn't know how to do this part, unfortunately.
-          name: "Generate version"
-          command: |
-            nix-shell \
-              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
-              --run 'python setup.py update_version'
 
-      - "run":
-          name: "Test"
-          command: |
-            # CircleCI build environment looks like it has a zillion and a
-            # half cores.  Don't let Nix autodetect this high core count
-            # because it blows up memory usage and fails the test run.  Pick a
-            # number of cores that suites the build environment we're paying
-            # for (the free one!).
-            source .circleci/lib.sh
-            cache_if_able nix-build \
-              --cores 8 \
-              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
-              nix/tests.nix
 
   typechecks:
     docker:
@@ -527,7 +499,6 @@ jobs:
 
   # build-image-pypy27-buster:
   #   <<: *BUILD_IMAGE
-
   #   environment:
   #     DISTRO: "pypy"
   #     TAG: "buster"
@@ -535,3 +506,87 @@ jobs:
   #     # setting up PyPy 3 in the image building toolchain.  This value is just
   #     # for constructing the right Docker image tag.
   #     PYTHON_VERSION: "2"
+
+executors:
+  nix:
+    docker:
+      # Run in a highly Nix-capable environment.
+      - <<: *DOCKERHUB_AUTH
+        image: "nixos/nix:2.10.3"
+    environment:
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
+      # allows us to push to CACHIX_NAME.  We only need this set for
+      # `cachix use` in this step.
+      CACHIX_NAME: "tahoe-lafs-opensource"
+
+commands:
+  nix-build:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"
+      buildSteps:
+        description: >-
+          The build steps to execute after setting up the build environment.
+        type: "steps"
+
+    steps:
+      - "run":
+          # Get cachix for Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
+            nix-env \
+              --file $NIXPKGS \
+              --install \
+              -A cachix bash
+              # Activate it for "binary substitution".  This sets up
+              # configuration tht lets Nix download something from the cache
+              # instead of building it locally, if possible.
+              cachix use "${CACHIX_NAME}"
+
+      - "checkout"
+
+      - "run":
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build Dependencies"
+          command: |
+            # CircleCI build environment looks like it has a zillion and a
+            # half cores.  Don't let Nix autodetect this high core count
+            # because it blows up memory usage and fails the test run.  Pick a
+            # number of cores that suits the build environment we're paying
+            # for (the free one!).
+            source .circleci/lib.sh
+            # nix-shell will build all of the dependencies of the target but
+            # not the target itself.
+            cache_if_able nix-shell \
+              --run "" \
+              --cores 3 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - "run":
+          name: "Build Package"
+          command: |
+            source .circleci/lib.sh
+            cache_if_able nix-build \
+              --cores 4 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - steps: "<<parameters.buildSteps>>"

From bc424dc1d1abbed3edc44eb053908ea0ca58efd6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 09:36:31 -0400
Subject: [PATCH 082/362] news fragment

---
 newsfragments/3994.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3994.minor

diff --git a/newsfragments/3994.minor b/newsfragments/3994.minor
new file mode 100644
index 000000000..e69de29bb

From 727d10af931bed91d3a9216d97f8687869210d63 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 22 Mar 2023 09:40:58 -0400
Subject: [PATCH 083/362] hit the right build target

---
 .circleci/config.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d39e3de56..b64152a94 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -392,10 +392,7 @@ jobs:
                     --cores 8 \
                     --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
                     --argstr pythonVersion "<<parameters.pythonVersion>>" \
-                    nix/unittests.nix
-
-
-
+                    nix/tests.nix
 
   typechecks:
     docker:

From 0da059b64486642864c0dbd211ccdb98909d79c1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 10:10:18 -0400
Subject: [PATCH 084/362] Update the connection status during the initial
 choice of NURLs.

---
 src/allmydata/storage_client.py           | 50 ++++++++++++-----------
 src/allmydata/test/test_storage_client.py | 23 +++++++----
 2 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index c88613803..f71931c8b 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1024,7 +1024,7 @@ async def _pick_a_http_server(
         reactor,
         nurls: list[DecodedURL],
         request: Callable[[Any, DecodedURL], defer.Deferred[Any]]
-) -> Optional[DecodedURL]:
+) -> DecodedURL:
     """Pick the first server we successfully send a request to.
 
     Fires with ``None`` if no server was found, or with the ``DecodedURL`` of
@@ -1035,18 +1035,8 @@ async def _pick_a_http_server(
         for nurl in nurls
     ])
 
-    try:
-        _, nurl = await queries
-        return nurl
-    except Exception as e:
-        # Logging errors breaks a bunch of tests, and it's not a _bug_ to
-        # have a failed connection, it's often expected and transient. More
-        # of a warning, really?
-        log.msg(
-            "Failed to connect to a storage server advertised by NURL: {}".format(
-                e)
-        )
-        return None
+    _, nurl = await queries
+    return nurl
 
 
 @implementer(IServer)
@@ -1223,19 +1213,31 @@ class HTTPNativeStorageServer(service.MultiService):
             picking = _pick_a_http_server(reactor, self._nurls, request)
             self._connecting_deferred = picking
             try:
-                nurl = await picking
-            finally:
-                self._connecting_deferred = None
-
-            if nurl is None:
-                # We failed to find a server to connect to. Perhaps the next
-                # iteration of the loop will succeed.
-                return
-            else:
-                self._istorage_server = _HTTPStorageServer.from_http_client(
-                    StorageClient.from_nurl(nurl, reactor)
+                try:
+                    nurl = await picking
+                finally:
+                    self._connecting_deferred = None
+            except Exception as e:
+                # Logging errors breaks a bunch of tests, and it's not a _bug_ to
+                # have a failed connection, it's often expected and transient. More
+                # of a warning, really?
+                log.msg(
+                    "Failed to connect to a storage server advertised by NURL: {}".format(e)
                 )
 
+                # Update the connection status:
+                self._failed_to_connect(Failure(e))
+
+                # Since we failed to find a server to connect to, give up
+                # for now. Perhaps the next iteration of the loop will
+                # succeed.
+                return
+
+            # iF we've gotten this far, we've found a working NURL.
+            self._istorage_server = _HTTPStorageServer.from_http_client(
+                StorageClient.from_nurl(nurl, reactor)
+            )
+
         result = self._istorage_server.get_version()
 
         def remove_connecting_deferred(result):
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 91668e7ca..cf4a939e8 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -63,6 +63,8 @@ from foolscap.ipb import (
     IConnectionHintHandler,
 )
 
+from allmydata.util.deferredutil import MultiFailure
+
 from .no_network import LocalWrapper
 from .common import (
     EMPTY_CLIENT_CONFIG,
@@ -782,7 +784,7 @@ storage:
 class PickHTTPServerTests(unittest.SynchronousTestCase):
     """Tests for ``_pick_a_http_server``."""
 
-    def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Optional[DecodedURL]:
+    def pick_result(self, url_to_results: dict[DecodedURL, tuple[float, Union[Exception, Any]]]) -> Deferred[DecodedURL]:
         """
         Given mapping of URLs to (delay, result), return the URL of the
         first selected server, or None.
@@ -803,7 +805,7 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
         d = _pick_a_http_server(clock, list(url_to_results.keys()), request)
         for i in range(100):
             clock.advance(0.1)
-        return self.successResultOf(d)
+        return d
 
     def test_first_successful_connect_is_picked(self):
         """
@@ -817,16 +819,21 @@ class PickHTTPServerTests(unittest.SynchronousTestCase):
             earliest_url: (1, None),
             bad_url: (0.5, RuntimeError()),
         })
-        self.assertEqual(result, earliest_url)
+        self.assertEqual(self.successResultOf(result), earliest_url)
 
-    def test_failures_are_turned_into_none(self):
+    def test_failures_include_all_reasons(self):
         """
-        If the requests all fail, ``_pick_a_http_server`` returns ``None``.
+        If all the requests fail, ``_pick_a_http_server`` raises a
+        ``allmydata.util.deferredutil.MultiFailure``.
         """
         eventually_good_url = DecodedURL.from_text("http://good")
         bad_url = DecodedURL.from_text("http://bad")
+        exception1 = RuntimeError()
+        exception2 = ZeroDivisionError()
         result = self.pick_result({
-            eventually_good_url: (1, ZeroDivisionError()),
-            bad_url: (0.1, RuntimeError())
+            eventually_good_url: (1, exception1),
+            bad_url: (0.1, exception2),
         })
-        self.assertEqual(result, None)
+        exc = self.failureResultOf(result).value
+        self.assertIsInstance(exc, MultiFailure)
+        self.assertEqual({f.value for f in exc.failures}, {exception2, exception1})

From 6659350ff3279c9c4162f16f5a14f8aa4d10fee4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 10:18:15 -0400
Subject: [PATCH 085/362] Improve type annotations.

---
 src/allmydata/util/deferredutil.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 83de411ce..77451b132 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -14,6 +14,7 @@ from typing import (
     TypeVar,
     Optional,
 )
+from typing_extensions import Awaitable, ParamSpec
 
 from foolscap.api import eventually
 from eliot.twisted import (
@@ -226,7 +227,11 @@ def until(
             break
 
 
-def async_to_deferred(f):
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]:
     """
     Wrap an async function to return a Deferred instead.
 
@@ -234,8 +239,8 @@ def async_to_deferred(f):
     """
 
     @wraps(f)
-    def not_async(*args, **kwargs):
-        return defer.Deferred.fromCoroutine(f(*args, **kwargs))
+    def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]:
+        return defer.Deferred.fromCoroutine(f(*args, **kwargs))  # type: ignore
 
     return not_async
 

From f0e60a80afa1b970299a9ebf97da1f7aeb12d784 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 10:22:52 -0400
Subject: [PATCH 086/362] Remove unneeded import.

---
 src/allmydata/test/test_storage_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index cf4a939e8..0671526ae 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -8,7 +8,7 @@ from json import (
     loads,
 )
 import hashlib
-from typing import Union, Any, Optional
+from typing import Union, Any
 
 from hyperlink import DecodedURL
 from fixtures import (

From 7715c4c6d01fda3da79245a3438360281b5abab9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 10:23:10 -0400
Subject: [PATCH 087/362] News fragment.

---
 newsfragments/3978.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3978.minor

diff --git a/newsfragments/3978.minor b/newsfragments/3978.minor
new file mode 100644
index 000000000..e69de29bb

From 9baafea00ed8aab9703c6d5af53a2efbed3303b0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:06:58 -0400
Subject: [PATCH 088/362] Refactor: simplify code so there are fewer codepaths.

---
 src/allmydata/storage_client.py | 68 ++++++++++++---------------------
 1 file changed, 25 insertions(+), 43 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index f71931c8b..96f37e599 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1188,16 +1188,16 @@ class HTTPNativeStorageServer(service.MultiService):
 
     @async_to_deferred
     async def _connect(self):
-        if self._istorage_server is None:
+        async def get_istorage_server() -> _HTTPStorageServer:
+            if self._istorage_server is not None:
+                return self._istorage_server
+
             # We haven't selected a server yet, so let's do so.
 
             # TODO This is somewhat inefficient on startup: it takes two successful
             # version() calls before we are live talking to a server, it could only
             # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992
 
-            # TODO Another problem with this scheme is that while picking
-            # the HTTP server to talk to, we don't have connection status
-            # updates... https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3978
             def request(reactor, nurl: DecodedURL):
                 # Since we're just using this one off to check if the NURL
                 # works, no need for persistent pool or other fanciness.
@@ -1213,51 +1213,33 @@ class HTTPNativeStorageServer(service.MultiService):
             picking = _pick_a_http_server(reactor, self._nurls, request)
             self._connecting_deferred = picking
             try:
-                try:
-                    nurl = await picking
-                finally:
-                    self._connecting_deferred = None
-            except Exception as e:
-                # Logging errors breaks a bunch of tests, and it's not a _bug_ to
-                # have a failed connection, it's often expected and transient. More
-                # of a warning, really?
-                log.msg(
-                    "Failed to connect to a storage server advertised by NURL: {}".format(e)
-                )
+                nurl = await picking
+            finally:
+                self._connecting_deferred = None
 
-                # Update the connection status:
-                self._failed_to_connect(Failure(e))
-
-                # Since we failed to find a server to connect to, give up
-                # for now. Perhaps the next iteration of the loop will
-                # succeed.
-                return
-
-            # iF we've gotten this far, we've found a working NURL.
+            # If we've gotten this far, we've found a working NURL.
             self._istorage_server = _HTTPStorageServer.from_http_client(
                 StorageClient.from_nurl(nurl, reactor)
             )
+            return self._istorage_server
 
-        result = self._istorage_server.get_version()
-
-        def remove_connecting_deferred(result):
-            self._connecting_deferred = None
-            return result
-
-        # Set a short timeout since we're relying on this for server liveness.
-        self._connecting_deferred = result.addTimeout(5, self._reactor).addBoth(
-                remove_connecting_deferred).addCallbacks(
-            self._got_version,
-            self._failed_to_connect
-        )
-
-        # We don't want to do another iteration of the loop until this
-        # iteration has finished, so wait here:
         try:
-            if self._connecting_deferred is not None:
-                await self._connecting_deferred
-        except Exception as e:
-            log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
+            try:
+                storage_server = await get_istorage_server()
+
+                # Get the version from the remote server. Set a short timeout since
+                # we're relying on this for server liveness.
+                self._connecting_deferred = storage_server.get_version().addTimeout(
+                    5, self._reactor)
+                # We don't want to do another iteration of the loop until this
+                # iteration has finished, so wait here:
+                version = await self._connecting_deferred
+                self._got_version(version)
+            except Exception as e:
+                log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
+                self._failed_to_connect(Failure(e))
+        finally:
+            self._connecting_deferred = None
 
     def stopService(self):
         if self._connecting_deferred is not None:

From 33d30b5c80bad6a647e1d8e6d6268555e4a72826 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:20:31 -0400
Subject: [PATCH 089/362] Type annotations.

---
 src/allmydata/storage_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 96f37e599..de756e322 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1069,7 +1069,7 @@ class HTTPNativeStorageServer(service.MultiService):
             DecodedURL.from_text(u)
             for u in announcement[ANONYMOUS_STORAGE_NURLS]
         ]
-        self._istorage_server = None
+        self._istorage_server : Optional[_HTTPStorageServer] = None
 
         self._connection_status = connection_status.ConnectionStatus.unstarted()
         self._version = None
@@ -1456,7 +1456,7 @@ class _HTTPStorageServer(object):
     _http_client = attr.ib(type=StorageClient)
 
     @staticmethod
-    def from_http_client(http_client):  # type: (StorageClient) -> _HTTPStorageServer
+    def from_http_client(http_client: StorageClient) -> _HTTPStorageServer:
         """
         Create an ``IStorageServer`` from a HTTP ``StorageClient``.
         """

From 1f29d5a23a6c772c35588f01b1c2a853691a4f5c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:38:15 -0400
Subject: [PATCH 090/362] News fragment.

---
 newsfragments/3996.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3996.minor

diff --git a/newsfragments/3996.minor b/newsfragments/3996.minor
new file mode 100644
index 000000000..e69de29bb

From ce6b7aeb828e4cdd2f2056e18ae7872dd53d6787 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:38:22 -0400
Subject: [PATCH 091/362] More modern pylint and flake8 and friends.

---
 setup.py | 2 +-
 tox.ini  | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index 152c49f0e..6528b01ed 100644
--- a/setup.py
+++ b/setup.py
@@ -400,7 +400,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it
               # intentionally.
-              "pyflakes == 2.2.0",
+              "pyflakes == 3.0.1",
               "coverage ~= 5.0",
               "mock",
               "tox ~= 3.0",
diff --git a/tox.ini b/tox.ini
index 382ba973e..447745784 100644
--- a/tox.ini
+++ b/tox.ini
@@ -100,10 +100,9 @@ commands =
 [testenv:codechecks]
 basepython = python3
 deps =
-     # Newer versions of PyLint have buggy configuration
-     # (https://github.com/PyCQA/pylint/issues/4574), so stick to old version
-     # for now.
-     pylint < 2.5
+     # Make sure we get a version of PyLint that respects config, and isn't too
+     # old.
+     pylint < 2.18, >2.14
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME
 setenv =

From 56e3aaad03f1839f50fce1a526f1b969517cc538 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:41:25 -0400
Subject: [PATCH 092/362] Lint fix and cleanup.

---
 src/allmydata/immutable/upload.py | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index 1d70759ff..9d6842f44 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -2,22 +2,12 @@
 Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations
 
-from future.utils import PY2, native_str
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from future.utils import native_str
 from past.builtins import long, unicode
 from six import ensure_str
 
-try:
-    from typing import List
-except ImportError:
-    pass
-
 import os, time, weakref, itertools
 
 import attr
@@ -915,8 +905,8 @@ class _Accum(object):
     :ivar remaining: The number of bytes still expected.
     :ivar ciphertext: The bytes accumulated so far.
     """
-    remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int
-    ciphertext = attr.ib(default=attr.Factory(list))                # type: List[bytes]
+    remaining : int = attr.ib(validator=attr.validators.instance_of(int))
+    ciphertext : list[bytes] = attr.ib(default=attr.Factory(list))
 
     def extend(self,
                size,           # type: int

From eb1cb84455883660301f51c7783497963c58007e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:42:38 -0400
Subject: [PATCH 093/362] Lint fix and cleanup.

---
 src/allmydata/introducer/server.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index f0638439a..98136157d 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -2,24 +2,13 @@
 Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations
 
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from past.builtins import long
 from six import ensure_text
 
 import time, os.path, textwrap
-
-try:
-    from typing import Any, Dict, Union
-except ImportError:
-    pass
+from typing import Any, Union
 
 from zope.interface import implementer
 from twisted.application import service
@@ -161,11 +150,11 @@ class IntroducerService(service.MultiService, Referenceable):
     # v1 is the original protocol, added in 1.0 (but only advertised starting
     # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
     # TODO: reconcile bytes/str for keys
-    VERSION = {
+    VERSION : dict[Union[bytes, str], Any]= {
                 #"http://allmydata.org/tahoe/protocols/introducer/v1": { },
                 b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
                 b"application-version": allmydata.__full_version__.encode("utf-8"),
-                }  # type: Dict[Union[bytes, str], Any]
+                }
 
     def __init__(self):
         service.MultiService.__init__(self)

From 958c08d6f577fa97e3b5a146405b4329a14c6235 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:44:14 -0400
Subject: [PATCH 094/362] Lint fix and cleanup.

---
 src/allmydata/node.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 34abb307f..58ee33ef5 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -4,14 +4,8 @@ a node for Tahoe-LAFS.
 
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations
 
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 from six import ensure_str, ensure_text
 
 import json
@@ -23,11 +17,7 @@ import errno
 from base64 import b32decode, b32encode
 from errno import ENOENT, EPERM
 from warnings import warn
-
-try:
-    from typing import Union
-except ImportError:
-    pass
+from typing import Union
 
 import attr
 
@@ -281,8 +271,7 @@ def _error_about_old_config_files(basedir, generated_files):
         raise e
 
 
-def ensure_text_and_abspath_expanduser_unicode(basedir):
-    # type: (Union[bytes, str]) -> str
+def ensure_text_and_abspath_expanduser_unicode(basedir: Union[bytes, str]) -> str:
     return abspath_expanduser_unicode(ensure_text(basedir))
 
 

From 76ecdfb7bcd9e64bb191409c33a10fd5621a7102 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:44:59 -0400
Subject: [PATCH 095/362] Fix lint.

---
 src/allmydata/scripts/admin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py
index 579505399..02fd9a143 100644
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@@ -255,9 +255,9 @@ def do_admin(options):
     return f(so)
 
 
-subCommands = [
+subCommands : SubCommands = [
     ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"),
-    ]  # type: SubCommands
+    ]
 
 dispatch = {
     "admin": do_admin,

From e1839ff30d629129b2aed9f0462a3f1bae1df9de Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:45:56 -0400
Subject: [PATCH 096/362] Fix lints.

---
 src/allmydata/scripts/cli.py | 22 +++++-----------------
 1 file changed, 5 insertions(+), 17 deletions(-)

diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py
index 579b37906..6e1f28d11 100644
--- a/src/allmydata/scripts/cli.py
+++ b/src/allmydata/scripts/cli.py
@@ -1,22 +1,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 
 import os.path, re, fnmatch
 
-try:
-    from allmydata.scripts.types_ import SubCommands, Parameters
-except ImportError:
-    pass
+from allmydata.scripts.types_ import SubCommands, Parameters
 
 from twisted.python import usage
 from allmydata.scripts.common import get_aliases, get_default_nodedir, \
@@ -29,14 +17,14 @@ NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
 _default_nodedir = get_default_nodedir()
 
 class FileStoreOptions(BaseOptions):
-    optParameters = [
+    optParameters : Parameters = [
         ["node-url", "u", None,
          "Specify the URL of the Tahoe gateway node, such as "
          "'http://127.0.0.1:3456'. "
          "This overrides the URL found in the --node-directory ."],
         ["dir-cap", None, None,
          "Specify which dirnode URI should be used as the 'tahoe' alias."]
-        ]  # type: Parameters
+        ]
 
     def postOptions(self):
         self["quiet"] = self.parent["quiet"]
@@ -484,7 +472,7 @@ class DeepCheckOptions(FileStoreOptions):
     (which must be a directory), like 'tahoe check' but for multiple files.
     Optionally repair any problems found."""
 
-subCommands = [
+subCommands : SubCommands = [
     ("mkdir", None, MakeDirectoryOptions, "Create a new directory."),
     ("add-alias", None, AddAliasOptions, "Add a new alias cap."),
     ("create-alias", None, CreateAliasOptions, "Create a new alias cap."),
@@ -503,7 +491,7 @@ subCommands = [
     ("check", None, CheckOptions, "Check a single file or directory."),
     ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."),
     ("status", None, TahoeStatusCommand, "Various status information."),
-    ]  # type: SubCommands
+    ]
 
 def mkdir(options):
     from allmydata.scripts import tahoe_mkdir

From 0cd197d4d0b156124f73df7b9b16607c846208ee Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:46:46 -0400
Subject: [PATCH 097/362] Update another instance of List.

---
 src/allmydata/immutable/upload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index 9d6842f44..0421de4e0 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -910,7 +910,7 @@ class _Accum(object):
 
     def extend(self,
                size,           # type: int
-               ciphertext,     # type: List[bytes]
+               ciphertext,     # type: list[bytes]
     ):
         """
         Accumulate some more ciphertext.

From ae29ea2b23cda231a953e9b1a8c92016c3ac0f53 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 11:47:43 -0400
Subject: [PATCH 098/362] Fix lint, and some Python 3 cleanups.

---
 src/allmydata/scripts/common.py | 36 ++++++++-------------------------
 1 file changed, 8 insertions(+), 28 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index c9fc8e031..d6ca8556d 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -4,29 +4,13 @@
 Ported to Python 3.
 """
 
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-else:
-    from typing import Union
-
+from typing import Union, Optional
 
 import os, sys, textwrap
 import codecs
 from os.path import join
 import urllib.parse
 
-try:
-    from typing import Optional
-    from .types_ import Parameters
-except ImportError:
-    pass
-
 from yaml import (
     safe_dump,
 )
@@ -37,6 +21,8 @@ from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import quote_output, \
     quote_local_unicode_path, argv_to_abspath
 from allmydata.scripts.default_nodedir import _default_nodedir
+from .types_ import Parameters
+
 
 def get_default_nodedir():
     return _default_nodedir
@@ -59,7 +45,7 @@ class BaseOptions(usage.Options):
     def opt_version(self):
         raise usage.UsageError("--version not allowed on subcommands")
 
-    description = None  # type: Optional[str]
+    description : Optional[str] = None
     description_unwrapped = None  # type: Optional[str]
 
     def __str__(self):
@@ -80,10 +66,10 @@ class BaseOptions(usage.Options):
 class BasedirOptions(BaseOptions):
     default_nodedir = _default_nodedir
 
-    optParameters = [
+    optParameters : Parameters = [
         ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
          % quote_local_unicode_path(_default_nodedir)],
-    ]  # type: Parameters
+    ]
 
     def parseArgs(self, basedir=None):
         # This finds the node-directory option correctly even if we are in a subcommand.
@@ -283,9 +269,8 @@ def get_alias(aliases, path_unicode, default):
                                 quote_output(alias))
     return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
 
-def escape_path(path):
-    # type: (Union[str,bytes]) -> str
-    u"""
+def escape_path(path: Union[str, bytes]) -> str:
+    """
     Return path quoted to US-ASCII, valid URL characters.
 
     >>> path = u'/føö/bar/☃'
@@ -302,9 +287,4 @@ def escape_path(path):
         ]),
         "ascii"
     )
-    # Eventually (i.e. as part of Python 3 port) we want this to always return
-    # Unicode strings. However, to reduce diff sizes in the short term it'll
-    # return native string (i.e. bytes) on Python 2.
-    if PY2:
-        result = result.encode("ascii").__native__()
     return result

From 29a66e51583f549c6d080dc0dd25cf2a77b7039a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:01:12 -0400
Subject: [PATCH 099/362] Fix lint.

---
 src/allmydata/scripts/debug.py | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py
index 6201ce28f..b6eba842a 100644
--- a/src/allmydata/scripts/debug.py
+++ b/src/allmydata/scripts/debug.py
@@ -1,19 +1,8 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
 
-from future.utils import PY2, bchr
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-try:
-    from allmydata.scripts.types_ import SubCommands
-except ImportError:
-    pass
+from future.utils import bchr
 
 import struct, time, os, sys
 
@@ -31,6 +20,7 @@ from allmydata.mutable.common import NeedMoreDataError
 from allmydata.immutable.layout import ReadBucketProxy
 from allmydata.util import base32
 from allmydata.util.encodingutil import quote_output
+from allmydata.scripts.types_ import SubCommands
 
 class DumpOptions(BaseOptions):
     def getSynopsis(self):
@@ -1076,9 +1066,9 @@ def do_debug(options):
     return f(so)
 
 
-subCommands = [
+subCommands : SubCommands = [
     ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
-    ]  # type: SubCommands
+    ]
 
 dispatch = {
     "debug": do_debug,

From 0e6825709dbe28178e58d0d66820b139531a24b5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:03:04 -0400
Subject: [PATCH 100/362] Fix lints.

---
 src/allmydata/scripts/create_node.py | 48 ++++++++++------------------
 src/allmydata/scripts/runner.py      | 21 +++---------
 2 files changed, 21 insertions(+), 48 deletions(-)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 5d9da518b..7d15b95ec 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -1,25 +1,11 @@
-# Ported to Python 3
-
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import io
 import os
 
-try:
-    from allmydata.scripts.types_ import (
-        SubCommands,
-        Parameters,
-        Flags,
-    )
-except ImportError:
-    pass
+from allmydata.scripts.types_ import (
+    SubCommands,
+    Parameters,
+    Flags,
+)
 
 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
@@ -48,7 +34,7 @@ def write_tac(basedir, nodetype):
     fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac)
 
 
-WHERE_OPTS = [
+WHERE_OPTS : Parameters = [
     ("location", None, None,
      "Server location to advertise (e.g. tcp:example.org:12345)"),
     ("port", None, None,
@@ -57,29 +43,29 @@ WHERE_OPTS = [
      "Hostname to automatically set --location/--port when --listen=tcp"),
     ("listen", None, "tcp",
      "Comma-separated list of listener types (tcp,tor,i2p,none)."),
-] # type: Parameters
+]
 
-TOR_OPTS = [
+TOR_OPTS : Parameters = [
     ("tor-control-port", None, None,
      "Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"),
     ("tor-executable", None, None,
      "The 'tor' executable to run (default is to search $PATH)."),
-] # type: Parameters
+]
 
-TOR_FLAGS = [
+TOR_FLAGS : Flags = [
     ("tor-launch", None, "Launch a tor instead of connecting to a tor control port."),
-] # type: Flags
+]
 
-I2P_OPTS = [
+I2P_OPTS : Parameters = [
     ("i2p-sam-port", None, None,
      "I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"),
     ("i2p-executable", None, None,
      "(future) The 'i2prouter' executable to run (default is to search $PATH)."),
-] # type: Parameters
+]
 
-I2P_FLAGS = [
+I2P_FLAGS : Flags = [
     ("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."),
-] # type: Flags
+]
 
 def validate_where_options(o):
     if o['listen'] == "none":
@@ -508,11 +494,11 @@ def create_introducer(config):
     defer.returnValue(0)
 
 
-subCommands = [
+subCommands : SubCommands = [
     ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."),
     ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."),
     ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."),
-]  # type: SubCommands
+]
 
 dispatch = {
     "create-node": create_node,
diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index d9fbc1b0a..18387cea5 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -1,28 +1,15 @@
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import os, sys
-from six.moves import StringIO
+from io import StringIO
 from past.builtins import unicode
 import six
 
-try:
-    from allmydata.scripts.types_ import SubCommands
-except ImportError:
-    pass
-
 from twisted.python import usage
 from twisted.internet import defer, task, threads
 
 from allmydata.scripts.common import get_default_nodedir
 from allmydata.scripts import debug, create_node, cli, \
     admin, tahoe_run, tahoe_invite
+from allmydata.scripts.types_ import SubCommands
 from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode
 from allmydata.util.eliotutil import (
     opt_eliot_destination,
@@ -47,9 +34,9 @@ if _default_nodedir:
     NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
 
 
-process_control_commands = [
+process_control_commands : SubCommands = [
     ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"),
-]  # type: SubCommands
+]
 
 
 class Options(usage.Options):

From aea748a890e16b845b477f39eeb2d186fdd40ea9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:03:43 -0400
Subject: [PATCH 101/362] Fix lint.

---
 src/allmydata/scripts/tahoe_invite.py | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py
index b62d6a463..b44efdeb9 100644
--- a/src/allmydata/scripts/tahoe_invite.py
+++ b/src/allmydata/scripts/tahoe_invite.py
@@ -1,19 +1,6 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-try:
-    from allmydata.scripts.types_ import SubCommands
-except ImportError:
-    pass
 
 from twisted.python import usage
 from twisted.internet import defer, reactor
@@ -21,6 +8,7 @@ from twisted.internet import defer, reactor
 from allmydata.util.encodingutil import argv_to_abspath
 from allmydata.util import jsonbytes as json
 from allmydata.scripts.common import get_default_nodedir, get_introducer_furl
+from allmydata.scripts.types_ import SubCommands
 from allmydata.client import read_config
 
 
@@ -112,10 +100,10 @@ def invite(options):
     print("Completed successfully", file=out)
 
 
-subCommands = [
+subCommands : SubCommands = [
     ("invite", None, InviteOptions,
      "Invite a new node to this grid"),
-]  # type: SubCommands
+]
 
 dispatch = {
     "invite": invite,

From 494a977525f6db91c3a3f5be2a7b75dd91b0ce22 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:06:01 -0400
Subject: [PATCH 102/362] Fix lint.

---
 src/allmydata/storage/http_server.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 3ae16ae5c..7437b3ec7 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -4,7 +4,7 @@ HTTP server for storage.
 
 from __future__ import annotations
 
-from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast
+from typing import Any, Callable, Union, cast
 from functools import wraps
 from base64 import b64decode
 import binascii
@@ -67,8 +67,8 @@ class ClientSecretsException(Exception):
 
 
 def _extract_secrets(
-    header_values, required_secrets
-):  # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes]
+    header_values: list[str], required_secrets: set[Secrets]
+) -> dict[Secrets, bytes]:
     """
     Given list of values of ``X-Tahoe-Authorization`` headers, and required
     secrets, return dictionary mapping secrets to decoded values.
@@ -173,7 +173,7 @@ class UploadsInProgress(object):
     _uploads: dict[bytes, StorageIndexUploads] = Factory(dict)
 
     # Map BucketWriter to (storage index, share number)
-    _bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict)
+    _bucketwriters: dict[BucketWriter, tuple[bytes, int]] = Factory(dict)
 
     def add_write_bucket(
         self,
@@ -798,7 +798,9 @@ class HTTPServer(object):
         # The reason can be a string with explanation, so in theory it could be
         # longish?
         info = await self._read_encoded(
-            request, _SCHEMAS["advise_corrupt_share"], max_size=32768,
+            request,
+            _SCHEMAS["advise_corrupt_share"],
+            max_size=32768,
         )
         bucket.advise_corrupt_share(info["reason"].encode("utf-8"))
         return b""
@@ -973,7 +975,7 @@ def listen_tls(
     endpoint: IStreamServerEndpoint,
     private_key_path: FilePath,
     cert_path: FilePath,
-) -> Deferred[Tuple[DecodedURL, IListeningPort]]:
+) -> Deferred[tuple[DecodedURL, IListeningPort]]:
     """
     Start a HTTPS storage server on the given port, return the NURL and the
     listening port.

From 3212311bbe3dd7f17bd8b7a7d74ad70fa503a7a1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:06:49 -0400
Subject: [PATCH 103/362] Fix lint.

---
 src/allmydata/storage/lease_schema.py | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py
index 7e604388e..63d3d4ed8 100644
--- a/src/allmydata/storage/lease_schema.py
+++ b/src/allmydata/storage/lease_schema.py
@@ -2,19 +2,7 @@
 Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-try:
-    from typing import Union
-except ImportError:
-    pass
+from typing import Union
 
 import attr
 
@@ -95,8 +83,7 @@ class HashedLeaseSerializer(object):
             cls._hash_secret,
         )
 
-    def serialize(self, lease):
-        # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes
+    def serialize(self, lease: Union[LeaseInfo, HashedLeaseInfo]) -> bytes:
         if isinstance(lease, LeaseInfo):
             # v2 of the immutable schema stores lease secrets hashed.  If
             # we're given a LeaseInfo then it holds plaintext secrets.  Hash

From 8d84e8a19f66cc05421608e3f25378f96ddad68c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 12:08:04 -0400
Subject: [PATCH 104/362] Fix lint.

---
 src/allmydata/storage/server.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 2bf99d74c..6099636f8 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -2,8 +2,9 @@
 Ported to Python 3.
 """
 from __future__ import annotations
+
 from future.utils import bytes_to_native_str
-from typing import Dict, Tuple, Iterable
+from typing import Iterable, Any
 
 import os, re
 
@@ -823,7 +824,7 @@ class FoolscapStorageServer(Referenceable):  # type: ignore # warner/foolscap#78
         self._server = storage_server
 
         # Canaries and disconnect markers for BucketWriters created via Foolscap:
-        self._bucket_writer_disconnect_markers = {}  # type: Dict[BucketWriter,Tuple[IRemoteReference, object]]
+        self._bucket_writer_disconnect_markers : dict[BucketWriter, tuple[IRemoteReference, Any]] = {}
 
         self._server.register_bucket_writer_close_handler(self._bucket_writer_closed)
 

From 4b25a923567e53b74ee04bba78b0342b7e09fe4d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 13:49:44 -0400
Subject: [PATCH 105/362] Limit cryptography for now.

---
 setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 152c49f0e..b12b8f4a2 100644
--- a/setup.py
+++ b/setup.py
@@ -63,7 +63,11 @@ install_requires = [
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
     #   transitively depends on cryptography.  So it's anyone's guess what
     #   version of cryptography will *really* be installed.
-    "cryptography >= 2.6",
+
+    # * cryptography 40 broke constants we need; should really be using them
+    # * via pyOpenSSL; will be fixed in
+    # * https://github.com/pyca/pyopenssl/issues/1201
+    "cryptography >= 2.6, < 40",
 
     # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
     #   rekeying bug <https://twistedmatrix.com/trac/ticket/4395>

From 74e3e27bea3309b33dc153114aad151baf7a4dd2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:06:27 -0400
Subject: [PATCH 106/362] Fix lint.

---
 src/allmydata/test/cli/test_create.py | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py
index 609888fb3..1d1576082 100644
--- a/src/allmydata/test/cli/test_create.py
+++ b/src/allmydata/test/cli/test_create.py
@@ -1,21 +1,11 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
 import os
 
-try:
-    from typing import Any, List, Tuple
-except ImportError:
-    pass
+from typing import Any
 
 from twisted.trial import unittest
 from twisted.internet import defer, reactor
@@ -356,8 +346,7 @@ class Config(unittest.TestCase):
         self.assertIn("is not empty", err)
         self.assertIn("To avoid clobbering anything, I am going to quit now", err)
 
-def fake_config(testcase, module, result):
-    # type: (unittest.TestCase, Any, Any) -> List[Tuple]
+def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[tuple]:
     """
     Monkey-patch a fake configuration function into the given module.
 

From 0c92fe554ddc8ce8b4c1b4efed943ff71158efab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:07:22 -0400
Subject: [PATCH 107/362] Fix lint.

---
 src/allmydata/test/eliotutil.py | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py
index dd21f1e9d..bdc779f1d 100644
--- a/src/allmydata/test/eliotutil.py
+++ b/src/allmydata/test/eliotutil.py
@@ -3,18 +3,6 @@ Tools aimed at the interaction between tests and Eliot.
 
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-# Python 2 compatibility
-# Can't use `builtins.str` because it's not JSON encodable:
-# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
-from past.builtins import unicode as str
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
 
 from six import ensure_text
 
@@ -23,11 +11,7 @@ __all__ = [
     "EliotLoggedRunTest",
 ]
 
-try:
-    from typing import Callable
-except ImportError:
-    pass
-
+from typing import Callable
 from functools import (
     partial,
     wraps,
@@ -147,8 +131,8 @@ class EliotLoggedRunTest(object):
 
 
 def with_logging(
-        test_id,      # type: str
-        test_method,  # type: Callable
+        test_id: str,
+        test_method: Callable,
 ):
     """
     Decorate a test method with additional log-related behaviors.

From 1668b2fcf6c6c65250922853047059786096add6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:09:11 -0400
Subject: [PATCH 108/362] Fix lint.

---
 src/allmydata/test/no_network.py | 47 ++++++++++++--------------------
 1 file changed, 17 insertions(+), 30 deletions(-)

diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index 66748e4b1..2346d96c1 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -1,34 +1,23 @@
 """
-Ported to Python 3.
+This contains a test harness that creates a full Tahoe grid in a single
+process (actually in a single MultiService) which does not use the network.
+It does not use an Introducer, and there are no foolscap Tubs. Each storage
+server puts real shares on disk, but is accessed through loopback
+RemoteReferences instead of over serialized SSL. It is not as complete as
+the common.SystemTestMixin framework (which does use the network), but
+should be considerably faster: on my laptop, it takes 50-80ms to start up,
+whereas SystemTestMixin takes close to 2s.
+
+This should be useful for tests which want to examine and/or manipulate the
+uploaded shares, checker/verifier/repairer tests, etc. The clients have no
+Tubs, so it is not useful for tests that involve a Helper.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
 
-# This contains a test harness that creates a full Tahoe grid in a single
-# process (actually in a single MultiService) which does not use the network.
-# It does not use an Introducer, and there are no foolscap Tubs. Each storage
-# server puts real shares on disk, but is accessed through loopback
-# RemoteReferences instead of over serialized SSL. It is not as complete as
-# the common.SystemTestMixin framework (which does use the network), but
-# should be considerably faster: on my laptop, it takes 50-80ms to start up,
-# whereas SystemTestMixin takes close to 2s.
+from __future__ import annotations
 
-# This should be useful for tests which want to examine and/or manipulate the
-# uploaded shares, checker/verifier/repairer tests, etc. The clients have no
-# Tubs, so it is not useful for tests that involve a Helper.
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-from past.builtins import unicode
 from six import ensure_text
 
-try:
-    from typing import Dict, Callable
-except ImportError:
-    pass
+from typing import Callable
 
 import os
 from base64 import b32encode
@@ -251,7 +240,7 @@ def create_no_network_client(basedir):
     :return: a Deferred yielding an instance of _Client subclass which
         does no actual networking but has the same API.
     """
-    basedir = abspath_expanduser_unicode(unicode(basedir))
+    basedir = abspath_expanduser_unicode(str(basedir))
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
 
     from allmydata.client import read_config
@@ -577,8 +566,7 @@ class GridTestMixin(object):
                     pass
         return sorted(shares)
 
-    def copy_shares(self, uri):
-        # type: (bytes) -> Dict[bytes, bytes]
+    def copy_shares(self, uri: bytes) -> dict[bytes, bytes]:
         """
         Read all of the share files for the given capability from the storage area
         of the storage servers created by ``set_up_grid``.
@@ -630,8 +618,7 @@ class GridTestMixin(object):
                 with open(i_sharefile, "wb") as f:
                     f.write(corruptdata)
 
-    def corrupt_all_shares(self, uri, corruptor, debug=False):
-        # type: (bytes, Callable[[bytes, bool], bytes], bool) -> None
+    def corrupt_all_shares(self, uri: Callable, corruptor: Callable[[bytes, bool], bytes], debug: bool=False):
         """
         Apply ``corruptor`` to the contents of all share files associated with a
         given capability and replace the share file contents with its result.

From 9d45cd85c712c9ee857d79032e026b1b149bbf0f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:12:16 -0400
Subject: [PATCH 109/362] Fix lint.

---
 src/allmydata/test/test_download.py | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_download.py b/src/allmydata/test/test_download.py
index 85d89cde6..4d57fa828 100644
--- a/src/allmydata/test/test_download.py
+++ b/src/allmydata/test/test_download.py
@@ -1,23 +1,14 @@
 """
 Ported to Python 3.
 """
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
 
-from future.utils import PY2, bchr
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from future.utils import bchr
 
 # system-level upload+download roundtrip test, but using shares created from
 # a previous run. This asserts that the current code is capable of decoding
 # shares from a previous version.
 
-try:
-    from typing import Any
-except ImportError:
-    pass
+from typing import Any
 
 import six
 import os
@@ -1197,8 +1188,7 @@ class Corruption(_Base, unittest.TestCase):
 
         return d
 
-    def _corrupt_flip_all(self, ign, imm_uri, which):
-        # type: (Any, bytes, int) -> None
+    def _corrupt_flip_all(self, ign: Any, imm_uri: bytes, which: int) -> None:
         """
         Flip the least significant bit at a given byte position in all share files
         for the given capability.

From 0bdea026f0023b318389ed26b025bc3d5d1f5355 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:13:20 -0400
Subject: [PATCH 110/362] Fix lint.

---
 src/allmydata/test/test_helper.py | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py
index 933a2b591..b280f95df 100644
--- a/src/allmydata/test/test_helper.py
+++ b/src/allmydata/test/test_helper.py
@@ -1,14 +1,7 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
 import os
 from struct import (
@@ -17,13 +10,8 @@ from struct import (
 from functools import (
     partial,
 )
-import attr
 
-try:
-    from typing import List
-    from allmydata.introducer.client import IntroducerClient
-except ImportError:
-    pass
+import attr
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -35,6 +23,7 @@ from eliot.twisted import (
     inline_callbacks,
 )
 
+from allmydata.introducer.client import IntroducerClient
 from allmydata.crypto import aes
 from allmydata.storage.server import (
     si_b2a,
@@ -132,7 +121,7 @@ class FakeCHKCheckerAndUEBFetcher(object):
         ))
 
 class FakeClient(service.MultiService):
-    introducer_clients = []  # type: List[IntroducerClient]
+    introducer_clients : list[IntroducerClient] = []
     DEFAULT_ENCODING_PARAMETERS = {"k":25,
                                    "happy": 75,
                                    "n": 100,

From 0377f858c2aa4628a2cc86018fcdea01a3ccc00e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:14:23 -0400
Subject: [PATCH 111/362] Correct type.

---
 src/allmydata/test/no_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index 2346d96c1..ee1f48b17 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -618,7 +618,7 @@ class GridTestMixin(object):
                 with open(i_sharefile, "wb") as f:
                     f.write(corruptdata)
 
-    def corrupt_all_shares(self, uri: Callable, corruptor: Callable[[bytes, bool], bytes], debug: bool=False):
+    def corrupt_all_shares(self, uri: bytes, corruptor: Callable[[bytes, bool], bytes], debug: bool=False):
         """
         Apply ``corruptor`` to the contents of all share files associated with a
         given capability and replace the share file contents with its result.

From 0d92aecbf3b97237f611628404efb24d45a4196e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:14:59 -0400
Subject: [PATCH 112/362] Fix lint.

---
 src/allmydata/test/test_istorageserver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py
index 9e7e7b6e1..ded9ac1ac 100644
--- a/src/allmydata/test/test_istorageserver.py
+++ b/src/allmydata/test/test_istorageserver.py
@@ -8,9 +8,9 @@ reused across tests, so each test should be careful to generate unique storage
 indexes.
 """
 
-from future.utils import bchr
+from __future__ import annotations
 
-from typing import Set
+from future.utils import bchr
 
 from random import Random
 from unittest import SkipTest
@@ -1041,7 +1041,7 @@ class IStorageServerMutableAPIsTestsMixin(object):
 class _SharedMixin(SystemTestMixin):
     """Base class for Foolscap and HTTP mixins."""
 
-    SKIP_TESTS = set()  # type: Set[str]
+    SKIP_TESTS : set[str] = set()
 
     def _get_istorage_server(self):
         native_server = next(iter(self.clients[0].storage_broker.get_known_servers()))

From f5d9947368d799581e65609da2bd18dfb5352509 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:15:51 -0400
Subject: [PATCH 113/362] Fix lint.

---
 src/allmydata/uri.py | 19 ++-----------------
 1 file changed, 2 insertions(+), 17 deletions(-)

diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py
index 5641771d3..fccf05db9 100644
--- a/src/allmydata/uri.py
+++ b/src/allmydata/uri.py
@@ -6,26 +6,11 @@ Ported to Python 3.
 Methods ending in to_string() are actually to_bytes(), possibly should be fixed
 in follow-up port.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    # Don't import bytes or str, to prevent future's newbytes leaking and
-    # breaking code that only expects normal bytes.
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min  # noqa: F401
-    from past.builtins import unicode as str
 
 from past.builtins import unicode, long
 
 import re
-
-try:
-    from typing import Type
-except ImportError:
-    pass
+from typing import Type
 
 from zope.interface import implementer
 from twisted.python.components import registerAdapter
@@ -707,7 +692,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI):
 
     BASE_STRING=b'URI:DIR2-Verifier:'
     BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
-    INNER_URI_CLASS=SSKVerifierURI  # type: Type[IVerifierURI]
+    INNER_URI_CLASS : Type[IVerifierURI] = SSKVerifierURI
 
     def __init__(self, filenode_uri=None):
         if filenode_uri:

From 63549c71efee7c952ea0a9d8b3a80e4a53fa7236 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:18:46 -0400
Subject: [PATCH 114/362] Fix lints, remove some Python 2 junk.

---
 src/allmydata/util/base32.py | 60 +++++++++++-------------------------
 1 file changed, 18 insertions(+), 42 deletions(-)

diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py
index ab65beeac..19a3bbe26 100644
--- a/src/allmydata/util/base32.py
+++ b/src/allmydata/util/base32.py
@@ -3,30 +3,11 @@ Base32 encoding.
 
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
-if PY2:
-    def backwardscompat_bytes(b):
-        """
-        Replace Future bytes with native Python 2 bytes, so % works
-        consistently until other modules are ported.
-        """
-        return getattr(b, "__native__", lambda: b)()
-    import string
-    maketrans = string.maketrans
-else:
-    def backwardscompat_bytes(b):
-        return b
-    maketrans = bytes.maketrans
-    from typing import Optional
+def backwardscompat_bytes(b):
+    return b
+maketrans = bytes.maketrans
 
+from typing import Optional
 import base64
 
 from allmydata.util.assertutil import precondition
@@ -34,7 +15,7 @@ from allmydata.util.assertutil import precondition
 rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus...
 chars = rfc3548_alphabet
 
-vals = backwardscompat_bytes(bytes(range(32)))
+vals = bytes(range(32))
 c2vtranstable = maketrans(chars, vals)
 v2ctranstable = maketrans(vals, chars)
 identitytranstable = maketrans(b'', b'')
@@ -61,16 +42,16 @@ def get_trailing_chars_without_lsbs(N):
     d = {}
     return b''.join(_get_trailing_chars_without_lsbs(N, d=d))
 
-BASE32CHAR = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(0)+b']')
-BASE32CHAR_4bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(1)+b']')
-BASE32CHAR_3bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(2)+b']')
-BASE32CHAR_2bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(3)+b']')
-BASE32CHAR_1bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(4)+b']')
-BASE32STR_1byte = backwardscompat_bytes(BASE32CHAR+BASE32CHAR_3bits)
-BASE32STR_2bytes = backwardscompat_bytes(BASE32CHAR+b'{3}'+BASE32CHAR_1bits)
-BASE32STR_3bytes = backwardscompat_bytes(BASE32CHAR+b'{4}'+BASE32CHAR_4bits)
-BASE32STR_4bytes = backwardscompat_bytes(BASE32CHAR+b'{6}'+BASE32CHAR_2bits)
-BASE32STR_anybytes = backwardscompat_bytes(bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes))
+BASE32CHAR = b'['+get_trailing_chars_without_lsbs(0)+b']'
+BASE32CHAR_4bits = b'['+get_trailing_chars_without_lsbs(1)+b']'
+BASE32CHAR_3bits = b'['+get_trailing_chars_without_lsbs(2)+b']'
+BASE32CHAR_2bits = b'['+get_trailing_chars_without_lsbs(3)+b']'
+BASE32CHAR_1bits = b'['+get_trailing_chars_without_lsbs(4)+b']'
+BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits
+BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits
+BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits
+BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits
+BASE32STR_anybytes = bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes)
 
 def b2a(os):  # type: (bytes) -> bytes
     """
@@ -80,7 +61,7 @@ def b2a(os):  # type: (bytes) -> bytes
     """
     return base64.b32encode(os).rstrip(b"=").lower()
 
-def b2a_or_none(os):  # type: (Optional[bytes]) -> Optional[bytes]
+def b2a_or_none(os: Optional[bytes]) -> Optional[bytes]:
     if os is not None:
         return b2a(os)
     return None
@@ -100,8 +81,6 @@ NUM_OS_TO_NUM_QS=(0, 2, 4, 5, 7,)
 NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4)
 NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,)
 NUM_QS_TO_NUM_BITS=tuple([_x*8 for _x in NUM_QS_TO_NUM_OS])
-if PY2:
-    del _x
 
 # A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the
 # original data had 8K bits for a positive integer K.
@@ -135,8 +114,6 @@ def a2b(cs):  # type: (bytes) -> bytes
     """
     @param cs the base-32 encoded data (as bytes)
     """
-    # Workaround Future newbytes issues by converting to real bytes on Python 2:
-    cs = backwardscompat_bytes(cs)
     precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
     precondition(isinstance(cs, bytes), cs)
 
@@ -144,9 +121,8 @@ def a2b(cs):  # type: (bytes) -> bytes
     # Add padding back, to make Python's base64 module happy:
     while (len(cs) * 5) % 8 != 0:
         cs += b"="
-    # Let newbytes come through and still work on Python 2, where the base64
-    # module gets confused by them.
-    return base64.b32decode(backwardscompat_bytes(cs))
+
+    return base64.b32decode(cs)
 
 
 __all__ = ["b2a", "a2b", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"]

From 6ce53000f0382cf53b72006d1df48648a2e8f651 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:19:39 -0400
Subject: [PATCH 115/362] Fix lint.

---
 src/allmydata/util/deferredutil.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 83de411ce..70ce8dade 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -208,10 +208,9 @@ class WaitForDelayedCallsMixin(PollMixin):
 
 @inline_callbacks
 def until(
-        action,     # type: Callable[[], defer.Deferred[Any]]
-        condition,  # type: Callable[[], bool]
-):
-    # type: (...) -> defer.Deferred[None]
+        action: Callable[[], defer.Deferred[Any]],
+        condition: Callable[[], bool],
+) -> defer.Deferred[None]:
     """
     Run a Deferred-returning function until a condition is true.
 

From 06dc32a6c0e625c7188321aa6b6f5a2b2d2c7e89 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:20:11 -0400
Subject: [PATCH 116/362] Fix lint.

---
 src/allmydata/util/pollmixin.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py
index 582bafe86..b23277565 100644
--- a/src/allmydata/util/pollmixin.py
+++ b/src/allmydata/util/pollmixin.py
@@ -4,22 +4,10 @@ Polling utility that returns Deferred.
 Ported to Python 3.
 """
 
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
 import time
 
-try:
-    from typing import List
-except ImportError:
-    pass
-
 from twisted.internet import task
 
 class TimeoutError(Exception):
@@ -29,7 +17,7 @@ class PollComplete(Exception):
     pass
 
 class PollMixin(object):
-    _poll_should_ignore_these_errors = []  # type: List[Exception]
+    _poll_should_ignore_these_errors : list[Exception] = []
 
     def poll(self, check_f, pollinterval=0.01, timeout=1000):
         # Return a Deferred, then call check_f periodically until it returns

From ee75bcd26bb2336c275224988feffdc531c36d05 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:20:48 -0400
Subject: [PATCH 117/362] Fix lint.

---
 src/allmydata/web/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index 3d85b1c4d..bd1e3838e 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -117,7 +117,7 @@ def boolean_of_arg(arg):  # type: (bytes) -> bool
     return arg.lower() in (b"true", b"t", b"1", b"on")
 
 
-def parse_replace_arg(replace):  # type: (bytes) -> Union[bool,_OnlyFiles]
+def parse_replace_arg(replace: bytes) -> Union[bool,_OnlyFiles]:
     assert isinstance(replace, bytes)
     if replace.lower() == b"only-files":
         return ONLY_FILES

From 51c7ca8d2cee964c94c8bce689520eb25c8325ee Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:22:21 -0400
Subject: [PATCH 118/362] Workaround for incompatibility.

---
 setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6528b01ed..854a333f1 100644
--- a/setup.py
+++ b/setup.py
@@ -63,7 +63,11 @@ install_requires = [
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
     #   transitively depends on cryptography.  So it's anyone's guess what
     #   version of cryptography will *really* be installed.
-    "cryptography >= 2.6",
+
+    # * cryptography 40 broke constants we need; should really be using them
+    # * via pyOpenSSL; will be fixed in
+    # * https://github.com/pyca/pyopenssl/issues/1201
+    "cryptography >= 2.6, < 40",
 
     # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
     #   rekeying bug <https://twistedmatrix.com/trac/ticket/4395>

From 796fc5bdc532d3809fcda2a50175ebacd9eb0504 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 24 Mar 2023 15:27:51 -0400
Subject: [PATCH 119/362] Fix lint.

---
 misc/checkers/check_load.py | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/misc/checkers/check_load.py b/misc/checkers/check_load.py
index 21576ea3a..d509b89ae 100644
--- a/misc/checkers/check_load.py
+++ b/misc/checkers/check_load.py
@@ -33,20 +33,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
 
 
 """
+from __future__ import annotations
 
 import os, sys, httplib, binascii
 import urllib, json, random, time, urlparse
 
-try:
-    from typing import Dict
-except ImportError:
-    pass
-
-# Python 2 compatibility
-from future.utils import PY2
-if PY2:
-    from future.builtins import str  # noqa: F401
-
 if sys.argv[1] == "--stats":
     statsfiles = sys.argv[2:]
     # gather stats every 10 seconds, do a moving-window average of the last
@@ -54,9 +45,9 @@ if sys.argv[1] == "--stats":
     DELAY = 10
     MAXSAMPLES = 6
     totals = []
-    last_stats = {}  # type: Dict[str, float]
+    last_stats : dict[str, float] = {}
     while True:
-        stats = {}  # type: Dict[str, float]
+        stats : dict[str, float] = {}
         for sf in statsfiles:
             for line in open(sf, "r").readlines():
                 name, str_value = line.split(":")

From 226da2fb2afa5961f8580619002905e3aeec580d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 26 Mar 2023 11:49:17 -0400
Subject: [PATCH 120/362] Add missing pyyaml dependency

It worked without this because we got the pyyaml dependency transitively but
we should declare it directly since it is a direct dependency.
---
 nix/tahoe-lafs.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index 2e1c4aa39..bf3ea83d3 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -34,6 +34,7 @@ let
     magic-wormhole
     netifaces
     psutil
+    pyyaml
     pycddl
     pyrsistent
     pyutil

From 6bf1f0846a9858e50988cf235562b5f92c11ebd5 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 26 Mar 2023 12:56:26 -0400
Subject: [PATCH 121/362] additional news fragment

---
 newsfragments/3997.installation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3997.installation

diff --git a/newsfragments/3997.installation b/newsfragments/3997.installation
new file mode 100644
index 000000000..186be0fc2
--- /dev/null
+++ b/newsfragments/3997.installation
@@ -0,0 +1 @@
+Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version.

From 51f763ca9ec7b571fdc8079c6c3e498e78691de1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 26 Mar 2023 20:04:46 -0400
Subject: [PATCH 122/362] fix word-o

---
 .circleci/config.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b64152a94..a1a95e9df 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,7 +12,8 @@
 version: 2.1
 
 # Every job that pushes a Docker image from Docker Hub must authenticate to
-# it.  Define a couple yaml anchors that can be used to supply a the necessary credentials.
+# it.  Define a couple yaml anchors that can be used to supply the necessary
+# credentials.
 
 # First is a CircleCI job context which makes Docker Hub credentials available
 # in the environment.

From ca7d60097c2b228191f9854db16960ad4abc667e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 26 Mar 2023 20:05:35 -0400
Subject: [PATCH 123/362] update stale explanation about CACHIX_NAME

---
 .circleci/config.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index a1a95e9df..d46e255af 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -512,9 +512,9 @@ executors:
       - <<: *DOCKERHUB_AUTH
         image: "nixos/nix:2.10.3"
     environment:
-      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
-      # allows us to push to CACHIX_NAME.  We only need this set for
-      # `cachix use` in this step.
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
+      # to push to CACHIX_NAME.  CACHIX_NAME tells cachix which cache to push
+      # to.
       CACHIX_NAME: "tahoe-lafs-opensource"
 
 commands:

From 4211fd8525bfe5e45c323a9389df1ec5f54aab8d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Mar 2023 13:41:30 -0400
Subject: [PATCH 124/362] Revert to old code.

---
 src/allmydata/storage/http_client.py | 46 ++++++++++++++++------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index a61f94708..44ba64363 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -355,7 +355,6 @@ class StorageClient(object):
         )
         return headers
 
-    @inlineCallbacks
     def request(
         self,
         method,
@@ -378,26 +377,35 @@ class StorageClient(object):
 
         Default timeout is 60 seconds.
         """
-        with start_action(
-            action_type="allmydata:storage:http-client:request",
-            method=method,
-            url=str(url),
-        ) as ctx:
-            headers = self._get_headers(headers)
+        headers = self._get_headers(headers)
 
-            # Add secrets:
-            for secret, value in [
-                (Secrets.LEASE_RENEW, lease_renew_secret),
-                (Secrets.LEASE_CANCEL, lease_cancel_secret),
-                (Secrets.UPLOAD, upload_secret),
-                (Secrets.WRITE_ENABLER, write_enabler_secret),
-            ]:
-                if value is None:
-                    continue
-                headers.addRawHeader(
-                    "X-Tahoe-Authorization",
-                    b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
+        # Add secrets:
+        for secret, value in [
+            (Secrets.LEASE_RENEW, lease_renew_secret),
+            (Secrets.LEASE_CANCEL, lease_cancel_secret),
+            (Secrets.UPLOAD, upload_secret),
+            (Secrets.WRITE_ENABLER, write_enabler_secret),
+        ]:
+            if value is None:
+                continue
+            headers.addRawHeader(
+                "X-Tahoe-Authorization",
+                b"%s %s" % (secret.value.encode("ascii"), b64encode(value).strip()),
+            )
+
+        # Note we can accept CBOR:
+        headers.addRawHeader("Accept", CBOR_MIME_TYPE)
+
+        # If there's a request message, serialize it and set the Content-Type
+        # header:
+        if message_to_serialize is not None:
+            if "data" in kwargs:
+                raise TypeError(
+                    "Can't use both `message_to_serialize` and `data` "
+                    "as keyword arguments at the same time"
                 )
+            kwargs["data"] = dumps(message_to_serialize)
+            headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
         return self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs

From b65e8c72dffe637d3a78f733866c212d823503c3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:55:53 -0400
Subject: [PATCH 125/362] Skip the tor integration tests if any needed tor
 tools are missing

---
 integration/conftest.py  | 7 +++----
 newsfragments/4000.minor | 0
 2 files changed, 3 insertions(+), 4 deletions(-)
 create mode 100644 newsfragments/4000.minor

diff --git a/integration/conftest.py b/integration/conftest.py
index 33e7998c1..5970e5ba4 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -451,10 +451,9 @@ def chutney(reactor, temp_dir):
     chutney_dir = join(temp_dir, 'chutney')
     mkdir(chutney_dir)
 
-    # TODO:
-
-    # check for 'tor' binary explicitly and emit a "skip" if we can't
-    # find it
+    missing = [exe for exe in ["tor", "tor-gencert"] if not which(exe)]
+    if missing:
+        pytest.skip(f"Some command-line tools not found: {missing}")
 
     # XXX yuck! should add a setup.py to chutney so we can at least
     # "pip install <path to tarball>" and/or depend on chutney in "pip
diff --git a/newsfragments/4000.minor b/newsfragments/4000.minor
new file mode 100644
index 000000000..e69de29bb

From fbcef2d1ae7f0893e4e4dc55066baa0b01feff4e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:32:40 -0400
Subject: [PATCH 126/362] Safely customize the Tor introducer's configuration

Previously we clobbered the whole generated configuration and potentially
wiped out additional important fields.

Now we modify the configuration by just changing the fields we need to change.
---
 integration/conftest.py | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 33e7998c1..54632be26 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -43,7 +43,7 @@ from .util import (
     generate_ssh_key,
     block_with_timeout,
 )
-
+from allmydata.node import read_config
 
 # pytest customization hooks
 
@@ -275,13 +275,6 @@ def introducer_furl(introducer, temp_dir):
     include_result=False,
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_tor
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_tor')
     print("making introducer", intro_dir)
 
@@ -301,9 +294,11 @@ log_gatherer.furl = {log_furl}
         )
         pytest_twisted.blockon(done_proto.done)
 
-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    # adjust a few settings
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 1c11f9e7d4957fcb1312418feb65c6a56847586e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:35:14 -0400
Subject: [PATCH 127/362] Add a little more debug info to the integration test
 suite output

---
 integration/conftest.py | 3 +++
 integration/test_tor.py | 2 +-
 integration/util.py     | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 54632be26..280d98f72 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -321,7 +321,9 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             pass
     request.addfinalizer(cleanup)
 
+    print("Waiting for introducer to be ready...")
     pytest_twisted.blockon(protocol.magic_seen)
+    print("Introducer ready.")
     return transport
 
 
@@ -332,6 +334,7 @@ def tor_introducer_furl(tor_introducer, temp_dir):
         print("Don't see {} yet".format(furl_fname))
         sleep(.1)
     furl = open(furl_fname, 'r').read()
+    print(f"Found Tor introducer furl: {furl}")
     return furl
 
 
diff --git a/integration/test_tor.py b/integration/test_tor.py
index 901858347..8398cf9a4 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -93,7 +93,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
     if True:
-        print("creating", node_dir.path)
+        print(f"creating {node_dir.path} with introducer {introducer_furl}")
         node_dir.makedirs()
         proto = util._DumpOutputProtocol(None)
         reactor.spawnProcess(
diff --git a/integration/util.py b/integration/util.py
index 05fef8fed..08c07a059 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -607,7 +607,7 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             continue
 
         if len(js['servers']) < minimum_number_of_servers:
-            print("waiting because insufficient servers")
+            print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})")
             time.sleep(1)
             continue
         server_times = [

From 1c99817e1b0d098a422a2d0ccdc4cb6a5cb3d489 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 13:41:51 -0400
Subject: [PATCH 128/362] Safely customize the client node's configuration

This is similar to the fix to the `tor_introducer` fixture.
---
 integration/test_tor.py | 39 +++++++++++----------------------------
 1 file changed, 11 insertions(+), 28 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 8398cf9a4..b116fe319 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -25,6 +25,7 @@ from twisted.python.filepath import (
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.client import read_config
 
 # see "conftest.py" for the fixtures (e.g. "tor_network")
 
@@ -103,10 +104,14 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 sys.executable, '-b', '-m', 'allmydata.scripts.runner',
                 'create-node',
                 '--nickname', name,
+                '--webport', web_port,
                 '--introducer', introducer_furl,
                 '--hide-ip',
                 '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                 '--listen', 'tor',
+                '--shares-needed', '1',
+                '--shares-happy', '1',
+                '--shares-total', '2',
                 node_dir.path,
             )
         )
@@ -115,35 +120,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
-    with node_dir.child('tahoe.cfg').open('w') as f:
-        node_config = '''
-[node]
-nickname = %(name)s
-web.port = %(web_port)s
-web.static = public_html
-log_gatherer.furl = %(log_furl)s
 
-[tor]
-control.port = tcp:localhost:%(control_port)d
-onion.external_port = 3457
-onion.local_port = %(local_port)d
-onion = true
-onion.private_key_file = private/tor_onion.privkey
-
-[client]
-shares.needed = 1
-shares.happy = 1
-shares.total = 2
-
-''' % {
-    'name': name,
-    'web_port': web_port,
-    'log_furl': flog_gatherer,
-    'control_port': control_port,
-    'local_port': control_port + 1000,
-}
-        node_config = node_config.encode("utf-8")
-        f.write(node_config)
+    config = read_config(node_dir.path, "tub.port")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
+    config.set_config("tor", "onion", "true")
+    config.set_config("tor", "onion.external_port", "3457")
+    config.set_config("tor", "onion.local_port", str(control_port + 1000))
+    config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
     print("running")
     result = yield util._run_node(reactor, node_dir.path, request, None)

From 8613e36bae8068aea67ee662296ab367e19c268c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:06:16 -0400
Subject: [PATCH 129/362] Propagate parent environment to children in the
 integration tests

---
 integration/conftest.py                  |  5 +++-
 integration/test_i2p.py                  | 29 ++++++++----------------
 integration/test_servers_of_happiness.py | 12 +++-------
 integration/test_tor.py                  | 21 ++++++-----------
 4 files changed, 23 insertions(+), 44 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 33e7998c1..b8db4e580 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -146,7 +146,8 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
             '--location', 'tcp:localhost:3117',
             '--port', '3117',
             gather_dir,
-        )
+        ),
+        env=environ,
     )
     pytest_twisted.blockon(out_protocol.done)
 
@@ -159,6 +160,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
             join(gather_dir, 'gatherer.tac'),
         ),
         path=gather_dir,
+        env=environ,
     )
     pytest_twisted.blockon(twistd_protocol.magic_seen)
 
@@ -177,6 +179,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
             (
                 'flogtool', 'dump', join(temp_dir, 'flog_gather', flogs[0])
             ),
+            env=environ,
         )
         print("Waiting for flogtool to complete")
         try:
diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 2deb01fab..96619a93a 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -2,26 +2,11 @@
 Integration tests for I2P support.
 """
 
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import sys
 from os.path import join, exists
-from os import mkdir
+from os import mkdir, environ
 from time import sleep
-
-if PY2:
-    def which(path):
-        # This will result in skipping I2P tests on Python 2. Oh well.
-        return None
-else:
-    from shutil import which
+from shutil import which
 
 from eliot import log_call
 
@@ -62,6 +47,7 @@ def i2p_network(reactor, temp_dir, request):
             "--log=stdout",
             "--loglevel=info"
         ),
+        env=environ,
     )
 
     def cleanup():
@@ -170,7 +156,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'carol_i2p'),
             'put', gold_path,
-        )
+        ),
+        env=environ,
     )
     yield proto.done
     cap = proto.output.getvalue().strip().split()[-1]
@@ -184,7 +171,8 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'dave_i2p'),
             'get', cap,
-        )
+        ),
+        env=environ,
     )
     yield proto.done
 
@@ -211,7 +199,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
             '--hide-ip',
             '--listen', 'i2p',
             node_dir.path,
-        )
+        ),
+        env=environ,
     )
     yield proto.done
 
diff --git a/integration/test_servers_of_happiness.py b/integration/test_servers_of_happiness.py
index c63642066..8363edb35 100644
--- a/integration/test_servers_of_happiness.py
+++ b/integration/test_servers_of_happiness.py
@@ -1,17 +1,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import sys
 from os.path import join
+from os import environ
 
 from twisted.internet.error import ProcessTerminated
 
@@ -45,7 +38,8 @@ def test_upload_immutable(reactor, temp_dir, introducer_furl, flog_gatherer, sto
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', node_dir,
             'put', __file__,
-        ]
+        ],
+        env=environ,
     )
     try:
         yield proto.done
diff --git a/integration/test_tor.py b/integration/test_tor.py
index 901858347..f82dcd052 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -1,17 +1,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import sys
 from os.path import join
+from os import environ
 
 import pytest
 import pytest_twisted
@@ -35,9 +28,6 @@ from allmydata.test.common import (
 if sys.platform.startswith('win'):
     pytest.skip('Skipping Tor tests on Windows', allow_module_level=True)
 
-if PY2:
-    pytest.skip('Skipping Tor tests on Python 2 because dependencies are hard to come by', allow_module_level=True)
-
 @pytest_twisted.inlineCallbacks
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
@@ -65,7 +55,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'carol'),
             'put', gold_path,
-        )
+        ),
+        env=environ,
     )
     yield proto.done
     cap = proto.output.getvalue().strip().split()[-1]
@@ -79,7 +70,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
             '-d', join(temp_dir, 'dave'),
             'get', cap,
-        )
+        ),
+        env=environ,
     )
     yield proto.done
 
@@ -108,7 +100,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                 '--listen', 'tor',
                 node_dir.path,
-            )
+            ),
+            env=environ,
         )
         yield proto.done
 

From 92eeaef4bded2dae3d1e0d99b176bee6a5a3ebaf Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:07:31 -0400
Subject: [PATCH 130/362] news fragment

---
 newsfragments/4001.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4001.minor

diff --git a/newsfragments/4001.minor b/newsfragments/4001.minor
new file mode 100644
index 000000000..e69de29bb

From 50c4ad81136a21096a6ce540938c70afc299fadd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:07:53 -0400
Subject: [PATCH 131/362] news fragment

---
 newsfragments/3999.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3999.minor

diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor
new file mode 100644
index 000000000..e69de29bb

From fb8c10c55fd13afc4d408f8dce47cd79a5e403f4 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 11:24:32 -0400
Subject: [PATCH 132/362] Use an already-installed Chutney if there is one

---
 integration/conftest.py | 114 +++++++++++++++++++---------------------
 1 file changed, 53 insertions(+), 61 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 33e7998c1..d184e61c9 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -1,6 +1,9 @@
 """
 Ported to Python 3.
 """
+
+from __future__ import annotations
+
 import sys
 import shutil
 from time import sleep
@@ -19,6 +22,7 @@ from eliot import (
     log_call,
 )
 
+from twisted.python.filepath import FilePath
 from twisted.python.procutils import which
 from twisted.internet.defer import DeferredList
 from twisted.internet.error import (
@@ -104,7 +108,7 @@ def reactor():
 
 @pytest.fixture(scope='session')
 @log_call(action_type=u"integration:temp_dir", include_args=[])
-def temp_dir(request):
+def temp_dir(request) -> str:
     """
     Invoke like 'py.test --keep-tempdir ...' to avoid deleting the temp-dir
     """
@@ -446,7 +450,23 @@ def bob(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, reques
 @pytest.fixture(scope='session')
 @pytest.mark.skipif(sys.platform.startswith('win'),
                     'Tor tests are unstable on Windows')
-def chutney(reactor, temp_dir):
+def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
+    # Try to find Chutney already installed in the environment.
+    try:
+        import chutney
+    except ImportError:
+        # Nope, we'll get our own in a moment.
+        pass
+    else:
+        # We already have one, just use it.
+        return (
+            # from `checkout/lib/chutney/__init__.py` we want to get back to
+            # `checkout` because that's the parent of the directory with all
+            # of the network definitions.  So, great-grand-parent.
+            FilePath(chutney.__file__).parent().parent().parent().path,
+            # There's nothing to add to the environment.
+            {},
+        )
 
     chutney_dir = join(temp_dir, 'chutney')
     mkdir(chutney_dir)
@@ -489,83 +509,55 @@ def chutney(reactor, temp_dir):
     )
     pytest_twisted.blockon(proto.done)
 
-    return chutney_dir
+    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
 
 
 @pytest.fixture(scope='session')
 @pytest.mark.skipif(sys.platform.startswith('win'),
                     reason='Tor tests are unstable on Windows')
 def tor_network(reactor, temp_dir, chutney, request):
+    """
+    Build a basic Tor network.
 
-    # this is the actual "chutney" script at the root of a chutney checkout
-    chutney_dir = chutney
-    chut = join(chutney_dir, 'chutney')
+    :param chutney: The root directory of a Chutney checkout and a dict of
+        additional environment variables to set so a Python process can use
+        it.
+
+    :return: None
+    """
+    chutney_root, chutney_env = chutney
+    basic_network = join(chutney_root, 'networks', 'basic')
+
+    env = environ.copy()
+    env.update(chutney_env)
+    chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
+    def chutney(argv):
+        proto = _DumpOutputProtocol(None)
+        reactor.spawnProcess(
+            proto,
+            sys.executable,
+            chutney_argv + argv,
+            path=join(chutney_root),
+            env=env,
+        )
+        return proto.done
 
     # now, as per Chutney's README, we have to create the network
     # ./chutney configure networks/basic
     # ./chutney start networks/basic
-
-    env = environ.copy()
-    env.update({"PYTHONPATH": join(chutney_dir, "lib")})
-    proto = _DumpOutputProtocol(None)
-    reactor.spawnProcess(
-        proto,
-        sys.executable,
-        (
-            sys.executable, '-m', 'chutney.TorNet', 'configure',
-            join(chutney_dir, 'networks', 'basic'),
-        ),
-        path=join(chutney_dir),
-        env=env,
-    )
-    pytest_twisted.blockon(proto.done)
-
-    proto = _DumpOutputProtocol(None)
-    reactor.spawnProcess(
-        proto,
-        sys.executable,
-        (
-            sys.executable, '-m', 'chutney.TorNet', 'start',
-            join(chutney_dir, 'networks', 'basic'),
-        ),
-        path=join(chutney_dir),
-        env=env,
-    )
-    pytest_twisted.blockon(proto.done)
+    pytest_twisted.blockon(chutney(("configure", basic_network)))
+    pytest_twisted.blockon(chutney(("start", basic_network)))
 
     # print some useful stuff
-    proto = _CollectOutputProtocol()
-    reactor.spawnProcess(
-        proto,
-        sys.executable,
-        (
-            sys.executable, '-m', 'chutney.TorNet', 'status',
-            join(chutney_dir, 'networks', 'basic'),
-        ),
-        path=join(chutney_dir),
-        env=env,
-    )
     try:
-        pytest_twisted.blockon(proto.done)
-    except ProcessTerminated:
-        print("Chutney.TorNet status failed (continuing):")
-        print(proto.output.getvalue())
+        pytest_twisted.blockon(chutney(("status", basic_network)))
+    except ProcessTerminated as e:
+        print("Chutney.TorNet status failed (continuing)")
 
     def cleanup():
         print("Tearing down Chutney Tor network")
-        proto = _CollectOutputProtocol()
-        reactor.spawnProcess(
-            proto,
-            sys.executable,
-            (
-                sys.executable, '-m', 'chutney.TorNet', 'stop',
-                join(chutney_dir, 'networks', 'basic'),
-            ),
-            path=join(chutney_dir),
-            env=env,
-        )
         try:
-            block_with_timeout(proto.done, reactor)
+            block_with_timeout(chutney(("stop", basic_network)), reactor)
         except ProcessTerminated:
             # If this doesn't exit cleanly, that's fine, that shouldn't fail
             # the test suite.

From d3d94937be84c751475b9e4d3da4820fe4cd0cdf Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 11:24:40 -0400
Subject: [PATCH 133/362] Nothing uses the return value of this fixture

---
 integration/conftest.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index d184e61c9..a8db66ec4 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -564,5 +564,3 @@ def tor_network(reactor, temp_dir, chutney, request):
             pass
 
     request.addfinalizer(cleanup)
-
-    return chut

From 81193aaddcfe319f09283492030eb7e47ac792a1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:43:58 -0400
Subject: [PATCH 134/362] news fragment

---
 newsfragments/4002.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4002.minor

diff --git a/newsfragments/4002.minor b/newsfragments/4002.minor
new file mode 100644
index 000000000..e69de29bb

From 0995772b24168a47049c35ed35825fc69a660316 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Mar 2023 14:54:27 -0400
Subject: [PATCH 135/362] Explain why we ignore type check.

---
 src/allmydata/util/deferredutil.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 89dc9704c..58ca7dde0 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -239,6 +239,11 @@ def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]:
 
     @wraps(f)
     def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]:
+        # Twisted documents fromCoroutine as accepting either a Generator or a
+        # Coroutine. However, the standard for type annotations of async
+        # functions is to return an Awaitable:
+        # https://github.com/twisted/twisted/issues/11832
+        # So, we ignore the type warning.
         return defer.Deferred.fromCoroutine(f(*args, **kwargs))  # type: ignore
 
     return not_async

From 7838f25bf8b70c981b29d8357a1edd427c253f80 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Mar 2023 14:54:36 -0400
Subject: [PATCH 136/362] Clean up with simpler idiom.

---
 src/allmydata/storage_client.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index de756e322..ee555819c 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1224,20 +1224,19 @@ class HTTPNativeStorageServer(service.MultiService):
             return self._istorage_server
 
         try:
-            try:
-                storage_server = await get_istorage_server()
+            storage_server = await get_istorage_server()
 
-                # Get the version from the remote server. Set a short timeout since
-                # we're relying on this for server liveness.
-                self._connecting_deferred = storage_server.get_version().addTimeout(
-                    5, self._reactor)
-                # We don't want to do another iteration of the loop until this
-                # iteration has finished, so wait here:
-                version = await self._connecting_deferred
-                self._got_version(version)
-            except Exception as e:
-                log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
-                self._failed_to_connect(Failure(e))
+            # Get the version from the remote server. Set a short timeout since
+            # we're relying on this for server liveness.
+            self._connecting_deferred = storage_server.get_version().addTimeout(
+                5, self._reactor)
+            # We don't want to do another iteration of the loop until this
+            # iteration has finished, so wait here:
+            version = await self._connecting_deferred
+            self._got_version(version)
+        except Exception as e:
+            log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
+            self._failed_to_connect(Failure(e))
         finally:
             self._connecting_deferred = None
 

From 4232c7f142d114d9440ceca7ea8ebfa9a044664c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 27 Mar 2023 14:55:10 -0400
Subject: [PATCH 137/362] remove unused binding

---
 integration/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index a8db66ec4..9f27ad014 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -551,7 +551,7 @@ def tor_network(reactor, temp_dir, chutney, request):
     # print some useful stuff
     try:
         pytest_twisted.blockon(chutney(("status", basic_network)))
-    except ProcessTerminated as e:
+    except ProcessTerminated:
         print("Chutney.TorNet status failed (continuing)")
 
     def cleanup():

From bd7c61cc5cce30ee75c0803e71ce9c5d7cc1643a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 27 Mar 2023 16:58:15 -0400
Subject: [PATCH 138/362] Split up the state management logic from the server
 pinging logic.

---
 src/allmydata/storage_client.py | 65 ++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 21 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index ee555819c..a40e98b03 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1186,8 +1186,46 @@ class HTTPNativeStorageServer(service.MultiService):
     def try_to_connect(self):
         self._connect()
 
+    def _connect(self) -> defer.Deferred[object]:
+        """
+        Try to connect to a working storage server.
+
+        If called while a previous ``_connect()`` is already running, it will
+        just return the same ``Deferred``.
+
+        ``LoopingCall.stop()`` doesn't cancel ``Deferred``s, unfortunately:
+        https://github.com/twisted/twisted/issues/11814. Thus we want to store
+        the ``Deferred`` so we can cancel it when necessary.
+
+        We also want to return it so that loop iterations take it into account,
+        and a new iteration doesn't start while we're in the middle of the
+        previous one.
+        """
+        # Conceivably try_to_connect() was called on this before, in which case
+        # we already are in the middle of connecting. So in that case just
+        # return whatever is in progress:
+        if self._connecting_deferred is not None:
+            return self._connecting_deferred
+
+        def done(_):
+            self._connecting_deferred = None
+
+        connecting = self._pick_server_and_get_version()
+        # Set a short timeout since we're relying on this for server liveness.
+        connecting = connecting.addTimeout(5, self._reactor).addCallbacks(
+            self._got_version, self._failed_to_connect
+        ).addBoth(done)
+        self._connecting_deferred = connecting
+        return connecting
+
     @async_to_deferred
-    async def _connect(self):
+    async def _pick_server_and_get_version(self):
+        """
+        Minimal implementation of connection logic: pick a server, get its
+        version.  This doesn't deal with errors much, so as to minimize
+        statefulness.  It does change ``self._istorage_server``, so possibly
+        more refactoring would be useful to remove even that much statefulness.
+        """
         async def get_istorage_server() -> _HTTPStorageServer:
             if self._istorage_server is not None:
                 return self._istorage_server
@@ -1207,15 +1245,7 @@ class HTTPNativeStorageServer(service.MultiService):
                     StorageClient.from_nurl(nurl, reactor, pool)
                 ).get_version()
 
-            # LoopingCall.stop() doesn't cancel Deferreds, unfortunately:
-            # https://github.com/twisted/twisted/issues/11814 Thus we want
-            # store the Deferred so it gets cancelled.
-            picking = _pick_a_http_server(reactor, self._nurls, request)
-            self._connecting_deferred = picking
-            try:
-                nurl = await picking
-            finally:
-                self._connecting_deferred = None
+            nurl = await _pick_a_http_server(reactor, self._nurls, request)
 
             # If we've gotten this far, we've found a working NURL.
             self._istorage_server = _HTTPStorageServer.from_http_client(
@@ -1226,19 +1256,12 @@ class HTTPNativeStorageServer(service.MultiService):
         try:
             storage_server = await get_istorage_server()
 
-            # Get the version from the remote server. Set a short timeout since
-            # we're relying on this for server liveness.
-            self._connecting_deferred = storage_server.get_version().addTimeout(
-                5, self._reactor)
-            # We don't want to do another iteration of the loop until this
-            # iteration has finished, so wait here:
-            version = await self._connecting_deferred
-            self._got_version(version)
+            # Get the version from the remote server.
+            version = await storage_server.get_version()
+            return version
         except Exception as e:
             log.msg(f"Failed to connect to a HTTP storage server: {e}", level=log.CURIOUS)
-            self._failed_to_connect(Failure(e))
-        finally:
-            self._connecting_deferred = None
+            raise
 
     def stopService(self):
         if self._connecting_deferred is not None:

From 2f106aa02adc157ef730ce49012da7e7f3b05b54 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:35:31 -0400
Subject: [PATCH 139/362] use foolscap.reconnector.ReconnectionInfo where one
 is required

It's *right* there.  Just use it!
---
 src/allmydata/test/test_connection_status.py | 22 +++++++-------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 2bd8bf6ab..f6b36d5ba 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -1,21 +1,13 @@
 """
 Tests for allmydata.util.connection_status.
-
-Port to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import mock
 
 from twisted.trial import unittest
 
+from foolscap.reconnector import ReconnectionInfo
+
 from ..util import connection_status
 
 class Status(unittest.TestCase):
@@ -33,7 +25,7 @@ class Status(unittest.TestCase):
         ci.connectionHandlers = {"h1": "hand1"}
         ci.winningHint = "h1"
         ci.establishedAt = 120
-        ri = mock.Mock()
+        ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
         rc = mock.Mock
@@ -51,7 +43,7 @@ class Status(unittest.TestCase):
         ci.connectionHandlers = {"h1": "hand1"}
         ci.winningHint = "h1"
         ci.establishedAt = 120
-        ri = mock.Mock()
+        ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
         rc = mock.Mock
@@ -70,7 +62,7 @@ class Status(unittest.TestCase):
         ci.listenerStatus = ("listener1", "successful")
         ci.winningHint = None
         ci.establishedAt = 120
-        ri = mock.Mock()
+        ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
         rc = mock.Mock
@@ -87,7 +79,7 @@ class Status(unittest.TestCase):
         ci = mock.Mock()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
+        ri = ReconnectionInfo()
         ri.state = "connecting"
         ri.connectionInfo = ci
         rc = mock.Mock
@@ -104,7 +96,7 @@ class Status(unittest.TestCase):
         ci = mock.Mock()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
+        ri = ReconnectionInfo()
         ri.state = "waiting"
         ri.lastAttempt = 10
         ri.nextAttempt = 20

From e2c6cc49d5e97627f4979ff4329c8cf3360f010e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:37:22 -0400
Subject: [PATCH 140/362] use foolscap.info.ConnectionInfo where one is
 required

It's *right* there.  Just use it!
---
 src/allmydata/test/test_connection_status.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index f6b36d5ba..ba57f8aee 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -7,6 +7,7 @@ import mock
 from twisted.trial import unittest
 
 from foolscap.reconnector import ReconnectionInfo
+from foolscap.info import ConnectionInfo
 
 from ..util import connection_status
 
@@ -20,7 +21,7 @@ class Status(unittest.TestCase):
                                "h2": "st2"})
 
     def test_reconnector_connected(self):
-        ci = mock.Mock()
+        ci = ConnectionInfo()
         ci.connectorStatuses = {"h1": "st1"}
         ci.connectionHandlers = {"h1": "hand1"}
         ci.winningHint = "h1"
@@ -38,7 +39,7 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_received_time, 123)
 
     def test_reconnector_connected_others(self):
-        ci = mock.Mock()
+        ci = ConnectionInfo()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
         ci.winningHint = "h1"
@@ -56,7 +57,7 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_received_time, 123)
 
     def test_reconnector_connected_listener(self):
-        ci = mock.Mock()
+        ci = ConnectionInfo()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
         ci.listenerStatus = ("listener1", "successful")
@@ -76,7 +77,7 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_received_time, 123)
 
     def test_reconnector_connecting(self):
-        ci = mock.Mock()
+        ci = ConnectionInfo()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
         ri = ReconnectionInfo()
@@ -93,7 +94,7 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_received_time, 123)
 
     def test_reconnector_waiting(self):
-        ci = mock.Mock()
+        ci = ConnectionInfo()
         ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
         ci.connectionHandlers = {"h1": "hand1"}
         ri = ReconnectionInfo()

From 6b7ea29d887150a8e041fbb0fe835b75ca76d565 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:40:25 -0400
Subject: [PATCH 141/362] use foolscap.reconnector.Reconnector where one is
 required

Unfortunately we need to touch a private attribute directly to shove our
expected info into it.  This isn't so bad though.  Foolscap isn't moving much
and we're not touching anything complex, just setting a simple model
attribute.
---
 src/allmydata/test/test_connection_status.py | 23 ++++++++++----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index ba57f8aee..3456a61f0 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -6,11 +6,17 @@ import mock
 
 from twisted.trial import unittest
 
-from foolscap.reconnector import ReconnectionInfo
+from foolscap.reconnector import ReconnectionInfo, Reconnector
 from foolscap.info import ConnectionInfo
 
 from ..util import connection_status
 
+def reconnector(info: ReconnectionInfo) -> Reconnector:
+    rc = Reconnector(None, None, (), {})
+    rc._reconnectionInfo = info
+    return rc
+
+
 class Status(unittest.TestCase):
     def test_hint_statuses(self):
         ncs = connection_status._hint_statuses(["h2","h1"],
@@ -29,8 +35,7 @@ class Status(unittest.TestCase):
         ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
         self.assertEqual(cs.summary, "Connected to h1 via hand1")
@@ -47,8 +52,7 @@ class Status(unittest.TestCase):
         ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
         self.assertEqual(cs.summary, "Connected to h1 via hand1")
@@ -66,8 +70,7 @@ class Status(unittest.TestCase):
         ri = ReconnectionInfo()
         ri.state = "connected"
         ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
         self.assertEqual(cs.summary, "Connected via listener (listener1)")
@@ -83,8 +86,7 @@ class Status(unittest.TestCase):
         ri = ReconnectionInfo()
         ri.state = "connecting"
         ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, False)
         self.assertEqual(cs.summary, "Trying to connect")
@@ -102,8 +104,7 @@ class Status(unittest.TestCase):
         ri.lastAttempt = 10
         ri.nextAttempt = 20
         ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        rc = reconnector(ri)
         with mock.patch("time.time", return_value=12):
             cs = connection_status.from_foolscap_reconnector(rc, 5)
         self.assertEqual(cs.connected, False)

From 32cd54501d0f92de067c39b4178814e438bb49a4 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:52:31 -0400
Subject: [PATCH 142/362] Pass a time function instead of patching the global

---
 src/allmydata/test/test_connection_status.py | 3 +--
 src/allmydata/util/connection_status.py      | 5 +++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 3456a61f0..6e258294b 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -105,8 +105,7 @@ class Status(unittest.TestCase):
         ri.nextAttempt = 20
         ri.connectionInfo = ci
         rc = reconnector(ri)
-        with mock.patch("time.time", return_value=12):
-            cs = connection_status.from_foolscap_reconnector(rc, 5)
+        cs = connection_status.from_foolscap_reconnector(rc, 5, time=lambda: 12)
         self.assertEqual(cs.connected, False)
         self.assertEqual(cs.summary,
                          "Reconnecting in 8 seconds (last attempt 2s ago)")
diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index 0e8595e81..e9c2c1388 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -16,6 +16,7 @@ if PY2:
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus
+from foolscap.reconnector import Reconnector
 
 @implementer(IConnectionStatus)
 class ConnectionStatus(object):
@@ -50,7 +51,7 @@ def _hint_statuses(which, handlers, statuses):
         non_connected_statuses["%s%s" % (hint, handler_dsc)] = dsc
     return non_connected_statuses
 
-def from_foolscap_reconnector(rc, last_received):
+def from_foolscap_reconnector(rc: Reconnector, last_received: int, time=time.time) -> ConnectionStatus:
     ri = rc.getReconnectionInfo()
     # See foolscap/reconnector.py, ReconnectionInfo, for details about possible
     # states. The returned result is a native string, it seems, so convert to
@@ -80,7 +81,7 @@ def from_foolscap_reconnector(rc, last_received):
         # ci describes the current in-progress attempt
         summary = "Trying to connect"
     elif state == "waiting":
-        now = time.time()
+        now = time()
         elapsed = now - ri.lastAttempt
         delay = ri.nextAttempt - now
         summary = "Reconnecting in %d seconds (last attempt %ds ago)" % \

From 9a8430c90fcbb879e3e8e1b32a44c5f7aea8a5c0 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:52:44 -0400
Subject: [PATCH 143/362] Remove porting boilerplate

---
 src/allmydata/util/connection_status.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index e9c2c1388..d7cf18e1b 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -1,18 +1,7 @@
 """
 Parse connection status from Foolscap.
-
-Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus

From 8e63fe2fddc8ca7b20a8a819d56c839546bc30e3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:52:55 -0400
Subject: [PATCH 144/362] Remove the unused mock import

---
 src/allmydata/test/test_connection_status.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 6e258294b..6c2e170f3 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -2,8 +2,6 @@
 Tests for allmydata.util.connection_status.
 """
 
-import mock
-
 from twisted.trial import unittest
 
 from foolscap.reconnector import ReconnectionInfo, Reconnector

From 6d4278b465a4eb2194845884d363d373e84433f3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:53:21 -0400
Subject: [PATCH 145/362] Factor some repetition out of the tests

---
 src/allmydata/test/test_connection_status.py | 86 ++++++++++----------
 src/allmydata/util/connection_status.py      |  2 +-
 2 files changed, 42 insertions(+), 46 deletions(-)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index 6c2e170f3..f415a6ebf 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -2,21 +2,43 @@
 Tests for allmydata.util.connection_status.
 """
 
-from twisted.trial import unittest
+from typing import Optional
 
 from foolscap.reconnector import ReconnectionInfo, Reconnector
 from foolscap.info import ConnectionInfo
 
 from ..util import connection_status
+from .common import SyncTestCase
 
 def reconnector(info: ReconnectionInfo) -> Reconnector:
-    rc = Reconnector(None, None, (), {})
+    rc = Reconnector(None, None, (), {}) # type: ignore[no-untyped-call]
     rc._reconnectionInfo = info
     return rc
 
+def connection_info(
+        statuses: dict[str, str],
+        handlers: dict[str, str],
+        winningHint: Optional[str],
+        establishedAt: Optional[int],
+) -> ConnectionInfo:
+    ci = ConnectionInfo() # type: ignore[no-untyped-call]
+    ci.connectorStatuses = statuses
+    ci.connectionHandlers = handlers
+    ci.winningHint = winningHint
+    ci.establishedAt = establishedAt
+    return ci
 
-class Status(unittest.TestCase):
-    def test_hint_statuses(self):
+def reconnection_info(
+        state: str,
+        connection_info: ConnectionInfo,
+) -> ReconnectionInfo:
+    ri = ReconnectionInfo() # type: ignore[no-untyped-call]
+    ri.state = state
+    ri.connectionInfo = connection_info
+    return ri
+
+class Status(SyncTestCase):
+    def test_hint_statuses(self) -> None:
         ncs = connection_status._hint_statuses(["h2","h1"],
                                                {"h1": "hand1", "h4": "hand4"},
                                                {"h1": "st1", "h2": "st2",
@@ -24,15 +46,9 @@ class Status(unittest.TestCase):
         self.assertEqual(ncs, {"h1 via hand1": "st1",
                                "h2": "st2"})
 
-    def test_reconnector_connected(self):
-        ci = ConnectionInfo()
-        ci.connectorStatuses = {"h1": "st1"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = ReconnectionInfo()
-        ri.state = "connected"
-        ri.connectionInfo = ci
+    def test_reconnector_connected(self) -> None:
+        ci = connection_info({"h1": "st1"}, {"h1": "hand1"}, "h1", 120)
+        ri = reconnection_info("connected", ci)
         rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
@@ -41,15 +57,9 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_connection_time, 120)
         self.assertEqual(cs.last_received_time, 123)
 
-    def test_reconnector_connected_others(self):
-        ci = ConnectionInfo()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = ReconnectionInfo()
-        ri.state = "connected"
-        ri.connectionInfo = ci
+    def test_reconnector_connected_others(self) -> None:
+        ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, "h1", 120)
+        ri = reconnection_info("connected", ci)
         rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
@@ -58,16 +68,10 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_connection_time, 120)
         self.assertEqual(cs.last_received_time, 123)
 
-    def test_reconnector_connected_listener(self):
-        ci = ConnectionInfo()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
+    def test_reconnector_connected_listener(self) -> None:
+        ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, 120)
         ci.listenerStatus = ("listener1", "successful")
-        ci.winningHint = None
-        ci.establishedAt = 120
-        ri = ReconnectionInfo()
-        ri.state = "connected"
-        ri.connectionInfo = ci
+        ri = reconnection_info("connected", ci)
         rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, True)
@@ -77,13 +81,9 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_connection_time, 120)
         self.assertEqual(cs.last_received_time, 123)
 
-    def test_reconnector_connecting(self):
-        ci = ConnectionInfo()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = ReconnectionInfo()
-        ri.state = "connecting"
-        ri.connectionInfo = ci
+    def test_reconnector_connecting(self) -> None:
+        ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, None)
+        ri = reconnection_info("connecting", ci)
         rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 123)
         self.assertEqual(cs.connected, False)
@@ -93,15 +93,11 @@ class Status(unittest.TestCase):
         self.assertEqual(cs.last_connection_time, None)
         self.assertEqual(cs.last_received_time, 123)
 
-    def test_reconnector_waiting(self):
-        ci = ConnectionInfo()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = ReconnectionInfo()
-        ri.state = "waiting"
+    def test_reconnector_waiting(self) -> None:
+        ci = connection_info({"h1": "st1", "h2": "st2"}, {"h1": "hand1"}, None, None)
+        ri = reconnection_info("waiting", ci)
         ri.lastAttempt = 10
         ri.nextAttempt = 20
-        ri.connectionInfo = ci
         rc = reconnector(ri)
         cs = connection_status.from_foolscap_reconnector(rc, 5, time=lambda: 12)
         self.assertEqual(cs.connected, False)
diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index d7cf18e1b..751eee4fe 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -31,7 +31,7 @@ class ConnectionStatus(object):
             last_received_time=None,
         )
 
-def _hint_statuses(which, handlers, statuses):
+def _hint_statuses(which, handlers, statuses) -> dict[str, str]:
     non_connected_statuses = {}
     for hint in which:
         handler = handlers.get(hint)

From 2e6a40294b7c30bb9e000eaa52e9bc00097504a3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:53:37 -0400
Subject: [PATCH 146/362] Crank the type checking ratchet

---
 mypy.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy.ini b/mypy.ini
index 7acc0ddc5..482fd6dd8 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -9,7 +9,7 @@ no_implicit_optional = True
 warn_redundant_casts = True
 strict_equality = True
 
-[mypy-allmydata.test.cli.wormholetesting]
+[mypy-allmydata.test.cli.wormholetesting,allmydata.test.test_connection_status]
 disallow_any_generics = True
 disallow_subclassing_any = True
 disallow_untyped_calls = True

From a839ace32ae144457bf2f0c1be6e97903f84e7d3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 08:53:54 -0400
Subject: [PATCH 147/362] news fragment

---
 newsfragments/4003.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4003.minor

diff --git a/newsfragments/4003.minor b/newsfragments/4003.minor
new file mode 100644
index 000000000..e69de29bb

From 3ea9e97606b8605befc8e2b2f1a6342cf47c0336 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 09:01:03 -0400
Subject: [PATCH 148/362] Python 3.8 compatibility

---
 src/allmydata/test/test_connection_status.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py
index f415a6ebf..da41f5a47 100644
--- a/src/allmydata/test/test_connection_status.py
+++ b/src/allmydata/test/test_connection_status.py
@@ -2,6 +2,8 @@
 Tests for allmydata.util.connection_status.
 """
 
+from __future__ import annotations
+
 from typing import Optional
 
 from foolscap.reconnector import ReconnectionInfo, Reconnector

From 80d8e5b465bbc717fc76708f319973ce40fb2907 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 28 Mar 2023 11:11:44 -0400
Subject: [PATCH 149/362] The function should return a coroutine.

---
 src/allmydata/util/deferredutil.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 58ca7dde0..695915ceb 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -13,8 +13,9 @@ from typing import (
     Sequence,
     TypeVar,
     Optional,
+    Coroutine,
 )
-from typing_extensions import Awaitable, ParamSpec
+from typing_extensions import ParamSpec
 
 from foolscap.api import eventually
 from eliot.twisted import (
@@ -230,7 +231,7 @@ P = ParamSpec("P")
 R = TypeVar("R")
 
 
-def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]:
+def async_to_deferred(f: Callable[P, Coroutine[defer.Deferred[R], None, R]]) -> Callable[P, Deferred[R]]:
     """
     Wrap an async function to return a Deferred instead.
 
@@ -239,12 +240,7 @@ def async_to_deferred(f: Callable[P, Awaitable[R]]) -> Callable[P, Deferred[R]]:
 
     @wraps(f)
     def not_async(*args: P.args, **kwargs: P.kwargs) -> Deferred[R]:
-        # Twisted documents fromCoroutine as accepting either a Generator or a
-        # Coroutine. However, the standard for type annotations of async
-        # functions is to return an Awaitable:
-        # https://github.com/twisted/twisted/issues/11832
-        # So, we ignore the type warning.
-        return defer.Deferred.fromCoroutine(f(*args, **kwargs))  # type: ignore
+        return defer.Deferred.fromCoroutine(f(*args, **kwargs))
 
     return not_async
 

From e8c72e6753db8287ef1dcfa824080e1191746c2a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 28 Mar 2023 12:55:41 -0400
Subject: [PATCH 150/362] Not sure if per method logging is worth it, will
 start from assumption that HTTP logging is enough.

---
 src/allmydata/storage/http_client.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 44ba64363..fcfc5bff3 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -746,22 +746,18 @@ class StorageClientImmutables(object):
         """
         Return the set of shares for a given storage index.
         """
-        with start_action(
-            action_type="allmydata:storage:http-client:immutable:list-shares",
-            storage_index=storage_index,
-        ) as ctx:
-            url = self._client.relative_url(
-                "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
-            )
-            response = yield self._client.request(
-                "GET",
-                url,
-            )
-            if response.code == http.OK:
-                body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
-                returnValue(set(body))
-            else:
-                raise ClientException(response.code)
+        url = self._client.relative_url(
+            "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
+        )
+        response = yield self._client.request(
+            "GET",
+            url,
+        )
+        if response.code == http.OK:
+            body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            returnValue(set(body))
+        else:
+            raise ClientException(response.code)
 
     def advise_corrupt_share(
         self,

From d36adf33a41be87814da8ccdf9a10c21813d53db Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 28 Mar 2023 13:06:43 -0400
Subject: [PATCH 151/362] Refactor; failing tests for some reason.

---
 src/allmydata/storage/http_server.py | 42 +++++++++++++++-------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 4f970b5a7..517771c02 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -106,28 +106,31 @@ def _authorization_decorator(required_secrets):
     def decorator(f):
         @wraps(f)
         def route(self, request, *args, **kwargs):
-            if not timing_safe_compare(
-                request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
-                    "utf-8"
-                ),
-                swissnum_auth_header(self._swissnum),
-            ):
-                request.setResponseCode(http.UNAUTHORIZED)
-                return b""
-            authorization = request.requestHeaders.getRawHeaders(
-                "X-Tahoe-Authorization", []
-            )
-            try:
-                secrets = _extract_secrets(authorization, required_secrets)
-            except ClientSecretsException:
-                request.setResponseCode(http.BAD_REQUEST)
-                return b"Missing required secrets"
             with start_action(
-                action_type="allmydata:storage:http-server:request",
+                action_type="allmydata:storage:http-server:handle_request",
                 method=request.method,
                 path=request.path,
             ) as ctx:
                 try:
+                    # Check Authorization header:
+                    if not timing_safe_compare(
+                        request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
+                            "utf-8"
+                        ),
+                        swissnum_auth_header(self._swissnum),
+                    ):
+                        raise _HTTPError(http.UNAUTHORIZED)
+
+                    # Check secrets:
+                    authorization = request.requestHeaders.getRawHeaders(
+                        "X-Tahoe-Authorization", []
+                    )
+                    try:
+                        secrets = _extract_secrets(authorization, required_secrets)
+                    except ClientSecretsException:
+                        raise _HTTPError(http.BAD_REQUEST)
+
+                    # Run the business logic:
                     result = f(self, request, secrets, *args, **kwargs)
                 except _HTTPError as e:
                     # This isn't an error necessarily for logging purposes,
@@ -136,8 +139,9 @@ def _authorization_decorator(required_secrets):
                     ctx.add_success_fields(response_code=e.code)
                     ctx.finish()
                     raise
-                ctx.add_success_fields(response_code=request.code)
-                return result
+                else:
+                    ctx.add_success_fields(response_code=request.code)
+                    return result
 
         return route
 

From ecfa76ac3268d8f23fe374b8f1ae7507e01f0773 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 28 Mar 2023 13:22:08 -0400
Subject: [PATCH 152/362] Python 3.8 compatibility

---
 src/allmydata/util/connection_status.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py
index 751eee4fe..0ccdcd672 100644
--- a/src/allmydata/util/connection_status.py
+++ b/src/allmydata/util/connection_status.py
@@ -2,6 +2,8 @@
 Parse connection status from Foolscap.
 """
 
+from __future__ import annotations
+
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus

From b81fad2970ff3eeea87fa869318dcee1f6db6ce9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 10:37:49 -0400
Subject: [PATCH 153/362] Make sure tests have the same error testing
 infrastructure as the real thing.

---
 src/allmydata/storage/http_server.py    | 28 ++++++++++++++-----------
 src/allmydata/test/test_storage_http.py |  2 ++
 2 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 517771c02..5ccb43c60 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -489,6 +489,21 @@ def read_range(
     return d
 
 
+def _add_error_handling(app: Klein):
+    """Add exception handlers to a Klein app."""
+    @app.handle_errors(_HTTPError)
+    def _http_error(_, request, failure):
+        """Handle ``_HTTPError`` exceptions."""
+        request.setResponseCode(failure.value.code)
+        return b""
+
+    @app.handle_errors(CDDLValidationError)
+    def _cddl_validation_error(_, request, failure):
+        """Handle CDDL validation errors."""
+        request.setResponseCode(http.BAD_REQUEST)
+        return str(failure.value).encode("utf-8")
+
+
 class HTTPServer(object):
     """
     A HTTP interface to the storage server.
@@ -496,18 +511,7 @@ class HTTPServer(object):
 
     _app = Klein()
     _app.url_map.converters["storage_index"] = StorageIndexConverter
-
-    @_app.handle_errors(_HTTPError)
-    def _http_error(self, request, failure):
-        """Handle ``_HTTPError`` exceptions."""
-        request.setResponseCode(failure.value.code)
-        return b""
-
-    @_app.handle_errors(CDDLValidationError)
-    def _cddl_validation_error(self, request, failure):
-        """Handle CDDL validation errors."""
-        request.setResponseCode(http.BAD_REQUEST)
-        return str(failure.value).encode("utf-8")
+    _add_error_handling(_app)
 
     def __init__(
         self,
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index eb5bcd4db..19529cd0e 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -54,6 +54,7 @@ from ..storage.http_server import (
     ClientSecretsException,
     _authorized_route,
     StorageIndexConverter,
+    _add_error_handling
 )
 from ..storage.http_client import (
     StorageClient,
@@ -253,6 +254,7 @@ class TestApp(object):
 
     clock: IReactorTime
     _app = Klein()
+    _add_error_handling(_app)
     _swissnum = SWISSNUM_FOR_TEST  # Match what the test client is using
 
     @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"])

From 41939e2b286fedca55af7ad202739751c40d7f3d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:11:24 -0400
Subject: [PATCH 154/362] Add some type annotations.

---
 src/allmydata/storage/http_client.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index fcfc5bff3..6450050a9 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -341,7 +341,7 @@ class StorageClient(object):
         https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
         return cls(https_url, swissnum, treq_client, reactor)
 
-    def relative_url(self, path):
+    def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
         return self._base_url.click(path)
 
@@ -357,14 +357,14 @@ class StorageClient(object):
 
     def request(
         self,
-        method,
-        url,
-        lease_renew_secret=None,
-        lease_cancel_secret=None,
-        upload_secret=None,
-        write_enabler_secret=None,
-        headers=None,
-        message_to_serialize=None,
+        method: str,
+        url: DecodedURL,
+        lease_renew_secret: Optional[bytes]=None,
+        lease_cancel_secret: Optional[bytes]=None,
+        upload_secret: Optional[bytes]=None,
+        write_enabler_secret: Optional[bytes]=None,
+        headers: Optional[Headers]=None,
+        message_to_serialize: object=None,
         timeout: float = 60,
         **kwargs,
     ):

From 3b3ea5409c7c31b5158c2ec5093d7021a927d2d3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:26:08 -0400
Subject: [PATCH 155/362] Type says we should only pass in DecodedURL.

---
 src/allmydata/test/test_storage_http.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 19529cd0e..ea93ad360 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -54,7 +54,7 @@ from ..storage.http_server import (
     ClientSecretsException,
     _authorized_route,
     StorageIndexConverter,
-    _add_error_handling
+    _add_error_handling,
 )
 from ..storage.http_client import (
     StorageClient,
@@ -348,7 +348,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/upload_secret",
+                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
             )
         )
         self.assertEqual(response.code, 400)
@@ -356,7 +356,9 @@ class CustomHTTPServerTests(SyncTestCase):
         # With secret, we're good.
         response = result_of(
             self.client.request(
-                "GET", "http://127.0.0.1/upload_secret", upload_secret=b"MAGIC"
+                "GET",
+                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
+                upload_secret=b"MAGIC",
             )
         )
         self.assertEqual(response.code, 200)
@@ -380,7 +382,7 @@ class CustomHTTPServerTests(SyncTestCase):
             response = result_of(
                 self.client.request(
                     "GET",
-                    f"http://127.0.0.1/bytes/{length}",
+                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                 )
             )
 
@@ -401,7 +403,7 @@ class CustomHTTPServerTests(SyncTestCase):
             response = result_of(
                 self.client.request(
                     "GET",
-                    f"http://127.0.0.1/bytes/{length}",
+                    DecodedURL.from_text(f"http://127.0.0.1/bytes/{length}"),
                 )
             )
 
@@ -416,7 +418,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/slowly_never_finish_result",
+                DecodedURL.from_text("http://127.0.0.1/slowly_never_finish_result"),
             )
         )
 
@@ -444,7 +446,7 @@ class CustomHTTPServerTests(SyncTestCase):
         response = result_of(
             self.client.request(
                 "GET",
-                "http://127.0.0.1/die",
+                DecodedURL.from_text("http://127.0.0.1/die"),
             )
         )
 
@@ -461,6 +463,7 @@ class Reactor(Clock):
 
     Advancing the clock also runs any callbacks scheduled via callFromThread.
     """
+
     def __init__(self):
         Clock.__init__(self)
         self._queue = Queue()
@@ -501,7 +504,9 @@ class HttpTestFixture(Fixture):
         self.storage_server = StorageServer(
             self.tempdir.path, b"\x00" * 20, clock=self.clock
         )
-        self.http_server = HTTPServer(self.clock, self.storage_server, SWISSNUM_FOR_TEST)
+        self.http_server = HTTPServer(
+            self.clock, self.storage_server, SWISSNUM_FOR_TEST
+        )
         self.treq = StubTreq(self.http_server.get_resource())
         self.client = StorageClient(
             DecodedURL.from_text("http://127.0.0.1"),

From 57ec669e1e70dc0fd6be4b6ddfbe3fe0f32221de Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:29:57 -0400
Subject: [PATCH 156/362] Add logging for request().

---
 src/allmydata/storage/http_client.py | 58 ++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 6450050a9..cbd4634b4 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -19,7 +19,7 @@ from collections_extended import RangeMap
 from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
-from twisted.web.iweb import IPolicyForHTTPS
+from twisted.web.iweb import IPolicyForHTTPS, IResponse
 from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
@@ -355,19 +355,20 @@ class StorageClient(object):
         )
         return headers
 
-    def request(
+    @async_to_deferred
+    async def request(
         self,
         method: str,
-        url: DecodedURL,
-        lease_renew_secret: Optional[bytes]=None,
-        lease_cancel_secret: Optional[bytes]=None,
-        upload_secret: Optional[bytes]=None,
-        write_enabler_secret: Optional[bytes]=None,
-        headers: Optional[Headers]=None,
-        message_to_serialize: object=None,
+        url: str,
+        lease_renew_secret: Optional[bytes] = None,
+        lease_cancel_secret: Optional[bytes] = None,
+        upload_secret: Optional[bytes] = None,
+        write_enabler_secret: Optional[bytes] = None,
+        headers: Optional[Headers] = None,
+        message_to_serialize: object = None,
         timeout: float = 60,
         **kwargs,
-    ):
+    ) -> Deferred[IResponse]:
         """
         Like ``treq.request()``, but with optional secrets that get translated
         into corresponding HTTP headers.
@@ -377,6 +378,41 @@ class StorageClient(object):
 
         Default timeout is 60 seconds.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:request",
+            method=method,
+            url=url.to_text(),
+            timeout=timeout,
+        ) as ctx:
+            response = await self._request(
+                method,
+                url,
+                lease_renew_secret,
+                lease_cancel_secret,
+                upload_secret,
+                write_enabler_secret,
+                headers,
+                message_to_serialize,
+                timeout,
+                **kwargs,
+            )
+            ctx.add_success_fields(response_code=response.code)
+            return response
+
+    async def _request(
+        self,
+        method: str,
+        url: str,
+        lease_renew_secret: Optional[bytes] = None,
+        lease_cancel_secret: Optional[bytes] = None,
+        upload_secret: Optional[bytes] = None,
+        write_enabler_secret: Optional[bytes] = None,
+        headers: Optional[Headers] = None,
+        message_to_serialize: object = None,
+        timeout: float = 60,
+        **kwargs,
+    ) -> IResponse:
+        """The implementation of request()."""
         headers = self._get_headers(headers)
 
         # Add secrets:
@@ -407,7 +443,7 @@ class StorageClient(object):
             kwargs["data"] = dumps(message_to_serialize)
             headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
-        return self._treq.request(
+        return await self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 

From 5e3fa04a3a6e0ae2210fcdc49347e18c3e384ec2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:30:22 -0400
Subject: [PATCH 157/362] Reformat with black.

---
 src/allmydata/storage/http_client.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index cbd4634b4..cead33732 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -488,12 +488,14 @@ class StorageClientGeneral(object):
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
-        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update({
-            b'tolerates-immutable-read-overrun': True,
-            b'delete-mutable-shares-with-zero-length-writev': True,
-            b'fills-holes-with-zero-bytes': True,
-            b'prevents-read-past-end-of-share-data': True,
-        })
+        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update(
+            {
+                b"tolerates-immutable-read-overrun": True,
+                b"delete-mutable-shares-with-zero-length-writev": True,
+                b"fills-holes-with-zero-bytes": True,
+                b"prevents-read-past-end-of-share-data": True,
+            }
+        )
         returnValue(decoded_response)
 
     @inlineCallbacks

From e19aeb5aea2bd57bfb2de3bc6f7f87a9097723c6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 3 Apr 2023 11:40:48 -0400
Subject: [PATCH 158/362] Correct the annotation.

---
 src/allmydata/storage/http_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index cead33732..bd9e3fc39 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -359,7 +359,7 @@ class StorageClient(object):
     async def request(
         self,
         method: str,
-        url: str,
+        url: DecodedURL,
         lease_renew_secret: Optional[bytes] = None,
         lease_cancel_secret: Optional[bytes] = None,
         upload_secret: Optional[bytes] = None,
@@ -402,7 +402,7 @@ class StorageClient(object):
     async def _request(
         self,
         method: str,
-        url: str,
+        url: DecodedURL,
         lease_renew_secret: Optional[bytes] = None,
         lease_cancel_secret: Optional[bytes] = None,
         upload_secret: Optional[bytes] = None,

From 1de8e811b5d963695af4c02886e85ee1ede36619 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 4 Apr 2023 10:58:22 -0400
Subject: [PATCH 159/362] Tweaks.

---
 integration/test_tor.py | 6 ++++--
 integration/util.py     | 1 +
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index b116fe319..d418f786b 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -43,8 +43,8 @@ if PY2:
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2)
-    yield util.await_client_ready(dave, minimum_number_of_servers=2)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
@@ -125,6 +125,8 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
     config.set_config("tor", "onion", "true")
     config.set_config("tor", "onion.external_port", "3457")
+    config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
+    #config.set_config("tor", "launch", "True")
     config.set_config("tor", "onion.local_port", str(control_port + 1000))
     config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
diff --git a/integration/util.py b/integration/util.py
index 08c07a059..a11c02225 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -90,6 +90,7 @@ class _CollectOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
+        print("OUT: {!r}".format(data))
         self.output.write(data)
 
     def errReceived(self, data):

From efa51d41dcdbf430510927b4ad6d41a9835b267a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 4 Apr 2023 10:58:28 -0400
Subject: [PATCH 160/362] Newer chutney.

---
 integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 280d98f72..36e7eef0b 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -465,7 +465,7 @@ def chutney(reactor, temp_dir):
         'git',
         (
             'git', 'clone',
-            'https://git.torproject.org/chutney.git',
+            'https://gitlab.torproject.org/tpo/core/chutney.git',
             chutney_dir,
         ),
         env=environ,
@@ -481,7 +481,7 @@ def chutney(reactor, temp_dir):
         (
             'git', '-C', chutney_dir,
             'reset', '--hard',
-            'c825cba0bcd813c644c6ac069deeb7347d3200ee'
+            'c4f6789ad2558dcbfeb7d024c6481d8112bfb6c2'
         ),
         env=environ,
     )

From 2be9e949f0c22c1daba57c2951bb5ff8eac9654d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:02:34 -0400
Subject: [PATCH 161/362] add Ubuntu 22.04 unit test job to CircleCI

---
 .circleci/config.yml | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d46e255af..638b4fc3e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -39,6 +39,8 @@ dockerhub-auth-template: &DOCKERHUB_AUTH
         <<: *DOCKERHUB_CONTEXT
     - "build-image-ubuntu-20-04":
         <<: *DOCKERHUB_CONTEXT
+    - "build-image-ubuntu-22-04":
+        <<: *DOCKERHUB_CONTEXT
     - "build-image-fedora-35":
         <<: *DOCKERHUB_CONTEXT
     - "build-image-oraclelinux-8":
@@ -78,6 +80,9 @@ workflows:
       - "ubuntu-20-04":
           {}
 
+      - "ubuntu-22-04":
+          {}
+
       # Equivalent to RHEL 8; CentOS 8 is dead.
       - "oraclelinux-8":
           {}
@@ -333,6 +338,16 @@ jobs:
       <<: *UTF_8_ENVIRONMENT
       TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
 
+  ubuntu-22-04:
+    <<: *DEBIAN
+    docker:
+      - <<: *DOCKERHUB_AUTH
+        image: "tahoelafsci/ubuntu:22.04-py3.10"
+        user: "nobody"
+    environment:
+      <<: *UTF_8_ENVIRONMENT
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py310"
+
   oraclelinux-8: &RHEL_DERIV
     docker:
       - <<: *DOCKERHUB_AUTH
@@ -479,6 +494,15 @@ jobs:
       PYTHON_VERSION: "3.9"
 
 
+  build-image-ubuntu-22-04:
+    <<: *BUILD_IMAGE
+
+    environment:
+      DISTRO: "ubuntu"
+      TAG: "22.04"
+      PYTHON_VERSION: "3.10"
+
+
   build-image-oraclelinux-8:
     <<: *BUILD_IMAGE
 

From 8557c66b39ed6ec806160eec857ba017ee2506de Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:03:20 -0400
Subject: [PATCH 162/362] Remove the "ubuntu-latest" unit test job from GitHub
 Actions

---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e006d90ac..adcf6cc5d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,7 +46,6 @@ jobs:
       matrix:
         os:
           - windows-latest
-          - ubuntu-latest
         python-version:
           - "3.8"
           - "3.9"

From 7ae7db678eae92fa41450240f91b56335aaff9dd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:03:51 -0400
Subject: [PATCH 163/362] add CPython 3.8 and CPython 3.9 unit test jobs to
 CircleCI

---
 .circleci/config.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 638b4fc3e..77c29734d 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -93,6 +93,8 @@ workflows:
           matrix:
             parameters:
               pythonVersion:
+                - "python38"
+                - "python39"
                 - "python310"
 
       - "nixos":

From 4c542dfa9b5aa06b3514e95204f74c0860e07329 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 5 Apr 2023 09:37:16 -0400
Subject: [PATCH 164/362] news fragment

---
 newsfragments/4006.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4006.minor

diff --git a/newsfragments/4006.minor b/newsfragments/4006.minor
new file mode 100644
index 000000000..e69de29bb

From 812458699dc22a62f49419a2fd62bcf7510b08b2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 5 Apr 2023 11:38:28 -0400
Subject: [PATCH 165/362] The tcp listening port needs to match the onion local
 port, or you get connection refused when you try to connect to the hidden
 service.

---
 integration/test_tor.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index d418f786b..6f6f54c25 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -126,8 +126,6 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     config.set_config("tor", "onion", "true")
     config.set_config("tor", "onion.external_port", "3457")
     config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
-    #config.set_config("tor", "launch", "True")
-    config.set_config("tor", "onion.local_port", str(control_port + 1000))
     config.set_config("tor", "onion.private_key_file", "private/tor_onion.privkey")
 
     print("running")

From 13e9f88309c15f5f4bfe71c0abe8bff5a2c2326b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 7 Apr 2023 15:23:20 -0400
Subject: [PATCH 166/362] Add necessary config option to ensure it listens on
 Tor, and also give correct Tor control port.

---
 integration/conftest.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 621c0224c..f3cf9a9d8 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -294,7 +294,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             request,
             (
                 'create-introducer',
-                '--tor-control-port', 'tcp:localhost:8010',
+                '--tor-control-port', 'tcp:localhost:8007',
+                '--hide-ip',
                 '--listen=tor',
                 intro_dir,
             ),

From 7b9432482724297c6d637aee20c2a6f5d94339ff Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 7 Apr 2023 15:23:51 -0400
Subject: [PATCH 167/362] More debugging.

---
 integration/util.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/util.py b/integration/util.py
index a11c02225..ac3fe2833 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -140,6 +140,7 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
+        print("OUT", data)
         data = str(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
@@ -148,6 +149,7 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
+        print("ERR", data)
         data = str(data, sys.stderr.encoding)
         sys.stdout.write(data)
 

From 4d4649f5c24ff89f1b538a09740eaffefea80dd2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 10 Apr 2023 11:28:26 -0400
Subject: [PATCH 168/362] Apply suggestions from code review

Co-authored-by: Jean-Paul Calderone <exarkun@twistedmatrix.com>
---
 src/allmydata/storage/http_client.py | 2 +-
 src/allmydata/storage/http_server.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index bd9e3fc39..ea142ed85 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -368,7 +368,7 @@ class StorageClient(object):
         message_to_serialize: object = None,
         timeout: float = 60,
         **kwargs,
-    ) -> Deferred[IResponse]:
+    ) -> IResponse:
         """
         Like ``treq.request()``, but with optional secrets that get translated
         into corresponding HTTP headers.
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 5ccb43c60..8647274f8 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -107,7 +107,7 @@ def _authorization_decorator(required_secrets):
         @wraps(f)
         def route(self, request, *args, **kwargs):
             with start_action(
-                action_type="allmydata:storage:http-server:handle_request",
+                action_type="allmydata:storage:http-server:handle-request",
                 method=request.method,
                 path=request.path,
             ) as ctx:

From cebf62176ee7ec064936c111aa28cb65f69d649b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 10 Apr 2023 11:40:59 -0400
Subject: [PATCH 169/362] WIP add logging to decode_cbor.

---
 src/allmydata/storage/http_client.py | 46 +++++++++++++++-------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ea142ed85..131f23846 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -447,24 +447,28 @@ class StorageClient(object):
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 
-    def decode_cbor(self, response, schema: Schema):
+    async def decode_cbor(self, response, schema: Schema) -> object:
         """Given HTTP response, return decoded CBOR body."""
-
-        def got_content(f: BinaryIO):
-            data = f.read()
-            schema.validate_cbor(data)
-            return loads(data)
-
-        if response.code > 199 and response.code < 300:
-            content_type = get_content_type(response.headers)
-            if content_type == CBOR_MIME_TYPE:
-                return limited_content(response, self._clock).addCallback(got_content)
+        with start_action(action_type="allmydata:storage:http-client:decode-cbor"):
+            if response.code > 199 and response.code < 300:
+                content_type = get_content_type(response.headers)
+                if content_type == CBOR_MIME_TYPE:
+                    f = await limited_content(response, self._clock)
+                    data = f.read()
+                    schema.validate_cbor(data)
+                    return loads(data)
+                else:
+                    raise ClientException(
+                        -1,
+                        "Server didn't send CBOR, content type is {}".format(
+                            content_type
+                        ),
+                    )
             else:
-                raise ClientException(-1, "Server didn't send CBOR")
-        else:
-            return treq.content(response).addCallback(
-                lambda data: fail(ClientException(response.code, response.phrase, data))
-            )
+                data = (
+                    await limited_content(response, self._clock, max_length=10_000)
+                ).read()
+                raise ClientException(response.code, response.phrase, data)
 
 
 @define(hash=True)
@@ -475,14 +479,14 @@ class StorageClientGeneral(object):
 
     _client: StorageClient
 
-    @inlineCallbacks
-    def get_version(self):
+    @async_to_deferred
+    async def get_version(self):
         """
         Return the version metadata for the server.
         """
         url = self._client.relative_url("/storage/v1/version")
-        response = yield self._client.request("GET", url)
-        decoded_response = yield self._client.decode_cbor(
+        response = await self._client.request("GET", url)
+        decoded_response = await self._client.decode_cbor(
             response, _SCHEMAS["get_version"]
         )
         # Add some features we know are true because the HTTP API
@@ -496,7 +500,7 @@ class StorageClientGeneral(object):
                 b"prevents-read-past-end-of-share-data": True,
             }
         )
-        returnValue(decoded_response)
+        return decoded_response
 
     @inlineCallbacks
     def add_or_renew_lease(

From 2a7616e0bebe29865767141255e36e9058db77e6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 16:43:46 -0400
Subject: [PATCH 170/362] Get tests passing again.

---
 src/allmydata/storage/http_client.py    | 36 ++++++++++++-------------
 src/allmydata/test/test_storage_http.py |  3 ++-
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 131f23846..7fc68c902 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -651,8 +651,8 @@ class StorageClientImmutables(object):
 
     _client: StorageClient
 
-    @inlineCallbacks
-    def create(
+    @async_to_deferred
+    async def create(
         self,
         storage_index,
         share_numbers,
@@ -679,7 +679,7 @@ class StorageClientImmutables(object):
         )
         message = {"share-numbers": share_numbers, "allocated-size": allocated_size}
 
-        response = yield self._client.request(
+        response = await self._client.request(
             "POST",
             url,
             lease_renew_secret=lease_renew_secret,
@@ -687,14 +687,12 @@ class StorageClientImmutables(object):
             upload_secret=upload_secret,
             message_to_serialize=message,
         )
-        decoded_response = yield self._client.decode_cbor(
+        decoded_response = await self._client.decode_cbor(
             response, _SCHEMAS["allocate_buckets"]
         )
-        returnValue(
-            ImmutableCreateResult(
-                already_have=decoded_response["already-have"],
-                allocated=decoded_response["allocated"],
-            )
+        return ImmutableCreateResult(
+            already_have=decoded_response["already-have"],
+            allocated=decoded_response["allocated"],
         )
 
     @inlineCallbacks
@@ -720,8 +718,8 @@ class StorageClientImmutables(object):
                 response.code,
             )
 
-    @inlineCallbacks
-    def write_share_chunk(
+    @async_to_deferred
+    async def write_share_chunk(
         self, storage_index, share_number, upload_secret, offset, data
     ):  # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress]
         """
@@ -741,7 +739,7 @@ class StorageClientImmutables(object):
                 _encode_si(storage_index), share_number
             )
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "PATCH",
             url,
             upload_secret=upload_secret,
@@ -765,13 +763,13 @@ class StorageClientImmutables(object):
             raise ClientException(
                 response.code,
             )
-        body = yield self._client.decode_cbor(
+        body = await self._client.decode_cbor(
             response, _SCHEMAS["immutable_write_share_chunk"]
         )
         remaining = RangeMap()
         for chunk in body["required"]:
             remaining.set(True, chunk["begin"], chunk["end"])
-        returnValue(UploadProgress(finished=finished, required=remaining))
+        return UploadProgress(finished=finished, required=remaining)
 
     def read_share_chunk(
         self, storage_index, share_number, offset, length
@@ -783,21 +781,21 @@ class StorageClientImmutables(object):
             self._client, "immutable", storage_index, share_number, offset, length
         )
 
-    @inlineCallbacks
-    def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
+    @async_to_deferred
+    async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
         """
         Return the set of shares for a given storage index.
         """
         url = self._client.relative_url(
             "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "GET",
             url,
         )
         if response.code == http.OK:
-            body = yield self._client.decode_cbor(response, _SCHEMAS["list_shares"])
-            returnValue(set(body))
+            body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            return set(body)
         else:
             raise ClientException(response.code)
 
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index ea93ad360..eca2be1c1 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -34,7 +34,7 @@ from hyperlink import DecodedURL
 from collections_extended import RangeMap
 from twisted.internet.task import Clock, Cooperator
 from twisted.internet.interfaces import IReactorTime, IReactorFromThreads
-from twisted.internet.defer import CancelledError, Deferred
+from twisted.internet.defer import CancelledError, Deferred, ensureDeferred
 from twisted.web import http
 from twisted.web.http_headers import Headers
 from werkzeug import routing
@@ -520,6 +520,7 @@ class HttpTestFixture(Fixture):
         Like ``result_of``, but supports fake reactor and ``treq`` testing
         infrastructure necessary to support asynchronous HTTP server endpoints.
         """
+        d = ensureDeferred(d)
         result = []
         error = []
         d.addCallbacks(result.append, error.append)

From 3997eaaf9048af3daca9cc291875f17b2a403218 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:00:31 -0400
Subject: [PATCH 171/362] Fix type annotations.

---
 src/allmydata/storage/http_client.py | 49 ++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 7fc68c902..b1877cd5f 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server.
 from __future__ import annotations
 
 from eliot import start_action, register_exception_extractor
-from typing import Union, Optional, Sequence, Mapping, BinaryIO
+from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
@@ -486,13 +486,17 @@ class StorageClientGeneral(object):
         """
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
-        decoded_response = await self._client.decode_cbor(
-            response, _SCHEMAS["get_version"]
+        decoded_response = cast(
+            dict[bytes, object],
+            await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
         )
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
-        decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"].update(
+        cast(
+            dict[bytes, object],
+            decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
+        ).update(
             {
                 b"tolerates-immutable-read-overrun": True,
                 b"delete-mutable-shares-with-zero-length-writev": True,
@@ -687,8 +691,9 @@ class StorageClientImmutables(object):
             upload_secret=upload_secret,
             message_to_serialize=message,
         )
-        decoded_response = await self._client.decode_cbor(
-            response, _SCHEMAS["allocate_buckets"]
+        decoded_response = cast(
+            dict[str, set[int]],
+            await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]),
         )
         return ImmutableCreateResult(
             already_have=decoded_response["already-have"],
@@ -763,8 +768,11 @@ class StorageClientImmutables(object):
             raise ClientException(
                 response.code,
             )
-        body = await self._client.decode_cbor(
-            response, _SCHEMAS["immutable_write_share_chunk"]
+        body = cast(
+            dict[str, list[dict[str, int]]],
+            await self._client.decode_cbor(
+                response, _SCHEMAS["immutable_write_share_chunk"]
+            ),
         )
         remaining = RangeMap()
         for chunk in body["required"]:
@@ -794,7 +802,10 @@ class StorageClientImmutables(object):
             url,
         )
         if response.code == http.OK:
-            body = await self._client.decode_cbor(response, _SCHEMAS["list_shares"])
+            body = cast(
+                set[int],
+                await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
+            )
             return set(body)
         else:
             raise ClientException(response.code)
@@ -865,6 +876,12 @@ class ReadTestWriteResult:
     reads: Mapping[int, Sequence[bytes]]
 
 
+# Result type for mutable read/test/write HTTP response.
+MUTABLE_RTW = TypedDict(
+    "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]}
+)
+
+
 @frozen
 class StorageClientMutables:
     """
@@ -911,8 +928,11 @@ class StorageClientMutables:
             message_to_serialize=message,
         )
         if response.code == http.OK:
-            result = await self._client.decode_cbor(
-                response, _SCHEMAS["mutable_read_test_write"]
+            result = cast(
+                MUTABLE_RTW,
+                await self._client.decode_cbor(
+                    response, _SCHEMAS["mutable_read_test_write"]
+                ),
             )
             return ReadTestWriteResult(success=result["success"], reads=result["data"])
         else:
@@ -942,8 +962,11 @@ class StorageClientMutables:
         )
         response = await self._client.request("GET", url)
         if response.code == http.OK:
-            return await self._client.decode_cbor(
-                response, _SCHEMAS["mutable_list_shares"]
+            return cast(
+                set[int],
+                await self._client.decode_cbor(
+                    response, _SCHEMAS["mutable_list_shares"]
+                ),
             )
         else:
             raise ClientException(response.code)

From 8bda370b30ef187d321840ecea97c56a439e1280 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:00:47 -0400
Subject: [PATCH 172/362] News fragment.

---
 newsfragments/4005.misc | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4005.misc

diff --git a/newsfragments/4005.misc b/newsfragments/4005.misc
new file mode 100644
index 000000000..e69de29bb

From 840ed0bf47561c7c9720c91deab3d5a5c56a7b35 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:04:00 -0400
Subject: [PATCH 173/362] Unused imports.

---
 src/allmydata/storage/http_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index b1877cd5f..ef4005414 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -20,7 +20,7 @@ from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
 from twisted.web.iweb import IPolicyForHTTPS, IResponse
-from twisted.internet.defer import inlineCallbacks, returnValue, fail, Deferred, succeed
+from twisted.internet.defer import inlineCallbacks, Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
     IReactorTime,

From 33ab0ce0422ff6211bd614bb8429fbe2084b9e02 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 12 Apr 2023 17:10:33 -0400
Subject: [PATCH 174/362] Fix name.

---
 newsfragments/{4005.misc => 4005.minor} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename newsfragments/{4005.misc => 4005.minor} (100%)

diff --git a/newsfragments/4005.misc b/newsfragments/4005.minor
similarity index 100%
rename from newsfragments/4005.misc
rename to newsfragments/4005.minor

From 507d1f8394cedc672715fee7bec1b5ef75cb6037 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Wed, 12 Apr 2023 22:34:45 -0600
Subject: [PATCH 175/362] Fix some Chutney things (and a couple cleanups): wait
 for bootstrap, increase timeout

---
 integration/conftest.py | 44 ++++++++++++++++++++++-------------------
 integration/test_tor.py |  2 +-
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index f3cf9a9d8..7a4234de7 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -206,13 +206,6 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     include_result=False,
 )
 def introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer0
-web.port = 4560
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer')
     print("making introducer", intro_dir)
 
@@ -232,6 +225,10 @@ log_gatherer.furl = {log_furl}
         )
         pytest_twisted.blockon(done_proto.done)
 
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer-tor")
+    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
     # over-write the config file with our stuff
     with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
         f.write(config)
@@ -283,7 +280,8 @@ def introducer_furl(introducer, temp_dir):
 )
 def tor_introducer(reactor, temp_dir, flog_gatherer, request):
     intro_dir = join(temp_dir, 'introducer_tor')
-    print("making introducer", intro_dir)
+    print("making Tor introducer in {}".format(intro_dir))
+    print("(this can take tens of seconds to allocate Onion address)")
 
     if not exists(intro_dir):
         mkdir(intro_dir)
@@ -342,7 +340,7 @@ def tor_introducer_furl(tor_introducer, temp_dir):
         print("Don't see {} yet".format(furl_fname))
         sleep(.1)
     furl = open(furl_fname, 'r').read()
-    print(f"Found Tor introducer furl: {furl}")
+    print(f"Found Tor introducer furl: {furl} in {furl_fname}")
     return furl
 
 
@@ -510,7 +508,13 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
     )
     pytest_twisted.blockon(proto.done)
 
-    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
+    return (
+        chutney_dir,
+        {
+            "PYTHONPATH": join(chutney_dir, "lib"),
+            "CHUTNEY_START_TIME": "200",  # default is 60
+        }
+    )
 
 
 @pytest.fixture(scope='session')
@@ -544,17 +548,9 @@ def tor_network(reactor, temp_dir, chutney, request):
         return proto.done
 
     # now, as per Chutney's README, we have to create the network
-    # ./chutney configure networks/basic
-    # ./chutney start networks/basic
     pytest_twisted.blockon(chutney(("configure", basic_network)))
-    pytest_twisted.blockon(chutney(("start", basic_network)))
-
-    # print some useful stuff
-    try:
-        pytest_twisted.blockon(chutney(("status", basic_network)))
-    except ProcessTerminated:
-        print("Chutney.TorNet status failed (continuing)")
 
+    # ensure we will tear down the network right before we start it
     def cleanup():
         print("Tearing down Chutney Tor network")
         try:
@@ -563,5 +559,13 @@ def tor_network(reactor, temp_dir, chutney, request):
             # If this doesn't exit cleanly, that's fine, that shouldn't fail
             # the test suite.
             pass
-
     request.addfinalizer(cleanup)
+
+    pytest_twisted.blockon(chutney(("start", basic_network)))
+    pytest_twisted.blockon(chutney(("wait_for_bootstrap", basic_network)))
+
+    # print some useful stuff
+    try:
+        pytest_twisted.blockon(chutney(("status", basic_network)))
+    except ProcessTerminated:
+        print("Chutney.TorNet status failed (continuing)")
diff --git a/integration/test_tor.py b/integration/test_tor.py
index fb9d8c086..c3041f6d3 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -61,7 +61,7 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     )
     yield proto.done
     cap = proto.output.getvalue().strip().split()[-1]
-    print("TEH CAP!", cap)
+    print("capability: {}".format(cap))
 
     proto = util._CollectOutputProtocol(capture_stderr=False)
     reactor.spawnProcess(

From 9472841c39120b408bfb7efab3b90b3fcb048a53 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Wed, 12 Apr 2023 23:01:28 -0600
Subject: [PATCH 176/362] enable tor, i2p services

---
 src/allmydata/introducer/server.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index 98136157d..e0ff138cc 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -83,6 +83,8 @@ def create_introducer(basedir=u"."):
             i2p_provider,
             tor_provider,
         )
+        i2p_provider.setServiceParent(node)
+        tor_provider.setServiceParent(node)
         return defer.succeed(node)
     except Exception:
         return Failure()

From 175473df407157db53276e0721fec324242e4bd2 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 13 Apr 2023 00:37:32 -0600
Subject: [PATCH 177/362] longer timeouts, forget less

---
 integration/conftest.py            | 2 +-
 integration/test_tor.py            | 4 ++--
 src/allmydata/introducer/server.py | 4 ----
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 7a4234de7..e7e021016 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -512,7 +512,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         chutney_dir,
         {
             "PYTHONPATH": join(chutney_dir, "lib"),
-            "CHUTNEY_START_TIME": "200",  # default is 60
+            "CHUTNEY_START_TIME": "600",  # default is 60
         }
     )
 
diff --git a/integration/test_tor.py b/integration/test_tor.py
index c3041f6d3..10e326e46 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -33,8 +33,8 @@ if sys.platform.startswith('win'):
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=60)
-    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=60)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
+    yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
 
     # ensure both nodes are connected to "a grid" by uploading
     # something via carol, and retrieve it using dave.
diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index e0ff138cc..5dad89ae8 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -68,10 +68,6 @@ def create_introducer(basedir=u"."):
         default_connection_handlers, foolscap_connection_handlers = create_connection_handlers(config, i2p_provider, tor_provider)
         tub_options = create_tub_options(config)
 
-        # we don't remember these because the Introducer doesn't make
-        # outbound connections.
-        i2p_provider = None
-        tor_provider = None
         main_tub = create_main_tub(
             config, tub_options, default_connection_handlers,
             foolscap_connection_handlers, i2p_provider, tor_provider,

From cf0d3c09f8fd863fb7103077d9564cff9c405317 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:20:40 -0400
Subject: [PATCH 178/362] News file.

---
 newsfragments/4009.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4009.minor

diff --git a/newsfragments/4009.minor b/newsfragments/4009.minor
new file mode 100644
index 000000000..e69de29bb

From 64dbeeab8f55e17e40097195957789792b3a5fc6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:33:18 -0400
Subject: [PATCH 179/362] Add logging to get_version().

---
 src/allmydata/storage/http_client.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ef4005414..610fbaddc 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -480,10 +480,17 @@ class StorageClientGeneral(object):
     _client: StorageClient
 
     @async_to_deferred
-    async def get_version(self):
+    async def get_version(self) -> dict[bytes, object]:
         """
         Return the version metadata for the server.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:get-version",
+        ):
+            return await self._get_version()
+
+    async def _get_version(self) -> dict[bytes, object]:
+        """Implementation of get_version()."""
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
         decoded_response = cast(

From af845a40c6dd1fe626771c63dff0c8dd7a6d86c8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:38:33 -0400
Subject: [PATCH 180/362] Fix type annotations, removing Deferred in
 particular.

---
 src/allmydata/storage/http_client.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ef4005414..e21cfc5cc 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -658,13 +658,13 @@ class StorageClientImmutables(object):
     @async_to_deferred
     async def create(
         self,
-        storage_index,
-        share_numbers,
-        allocated_size,
-        upload_secret,
-        lease_renew_secret,
-        lease_cancel_secret,
-    ):  # type: (bytes, set[int], int, bytes, bytes, bytes) -> Deferred[ImmutableCreateResult]
+        storage_index: bytes,
+        share_numbers: set[int],
+        allocated_size: int,
+        upload_secret: bytes,
+        lease_renew_secret: bytes,
+        lease_cancel_secret: bytes,
+    ) -> ImmutableCreateResult:
         """
         Create a new storage index for an immutable.
 
@@ -725,8 +725,13 @@ class StorageClientImmutables(object):
 
     @async_to_deferred
     async def write_share_chunk(
-        self, storage_index, share_number, upload_secret, offset, data
-    ):  # type: (bytes, int, bytes, int, bytes) -> Deferred[UploadProgress]
+        self,
+        storage_index: bytes,
+        share_number: int,
+        upload_secret: bytes,
+        offset: int,
+        data: bytes,
+    ) -> UploadProgress:
         """
         Upload a chunk of data for a specific share.
 
@@ -790,7 +795,7 @@ class StorageClientImmutables(object):
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> Deferred[set[int]]:
+    async def list_shares(self, storage_index: bytes) -> set[int]:
         """
         Return the set of shares for a given storage index.
         """

From e9a9ac7110a88e8410a80d0481040fb44f614be2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:44:52 -0400
Subject: [PATCH 181/362] Rip out codecov for now.

---
 .circleci/config.yml             | 2 +-
 .circleci/populate-wheelhouse.sh | 2 +-
 .github/workflows/ci.yml         | 2 +-
 newsfragments/4010.minor         | 0
 4 files changed, 3 insertions(+), 3 deletions(-)
 create mode 100644 newsfragments/4010.minor

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 77c29734d..54b2706cd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -260,7 +260,7 @@ jobs:
           name: "Submit coverage results"
           command: |
             if [ -n "${UPLOAD_COVERAGE}" ]; then
-              /tmp/venv/bin/codecov
+              echo "TODO: Need a new coverage solution, see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4011"
             fi
 
     docker:
diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 857171979..374ca0adb 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -9,7 +9,7 @@ BASIC_DEPS="pip wheel"
 
 # Python packages we need to support the test infrastructure.  *Not* packages
 # Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0 codecov"
+TEST_DEPS="tox~=3.0"
 
 # Python packages we need to generate test reports for CI infrastructure.
 # *Not* packages Tahoe-LAFS itself (implement or test suite) need.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index adcf6cc5d..1bb7c9efb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -79,7 +79,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade codecov "tox<4" tox-gh-actions setuptools
+          pip install --upgrade "tox<4" tox-gh-actions setuptools
           pip list
 
       - name: Display tool versions
diff --git a/newsfragments/4010.minor b/newsfragments/4010.minor
new file mode 100644
index 000000000..e69de29bb

From e0ca48b707b1c6ca35c05083ebcda0547c23b920 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 09:54:36 -0400
Subject: [PATCH 182/362] Add logging to add_or_renew_lease().

---
 src/allmydata/storage/common.py      |  5 +++++
 src/allmydata/storage/http_client.py | 21 ++++++++++++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py
index 17a3f41b7..89e29b081 100644
--- a/src/allmydata/storage/common.py
+++ b/src/allmydata/storage/common.py
@@ -39,6 +39,11 @@ def si_b2a(storageindex):
 def si_a2b(ascii_storageindex):
     return base32.a2b(ascii_storageindex)
 
+def si_to_human_readable(storageindex: bytes) -> str:
+    """Create human-readable string of storage index."""
+    assert len(storageindex) == 16
+    return str(base32.b2a(storageindex), "ascii")
+
 def storage_index_to_dir(storageindex):
     """Convert storage index to directory path.
 
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 610fbaddc..6f3bb05e1 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -44,7 +44,7 @@ from .http_common import (
     CBOR_MIME_TYPE,
     get_spki_hash,
 )
-from .common import si_b2a
+from .common import si_b2a, si_to_human_readable
 from ..util.hashutil import timing_safe_compare
 from ..util.deferredutil import async_to_deferred
 
@@ -513,20 +513,31 @@ class StorageClientGeneral(object):
         )
         return decoded_response
 
-    @inlineCallbacks
-    def add_or_renew_lease(
+    @async_to_deferred
+    async def add_or_renew_lease(
         self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes
-    ) -> Deferred[None]:
+    ) -> None:
         """
         Add or renew a lease.
 
         If the renewal secret matches an existing lease, it is renewed.
         Otherwise a new lease is added.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:add-or-renew-lease",
+            storage_index=si_to_human_readable(storage_index),
+        ):
+            return await self._add_or_renew_lease(
+                storage_index, renew_secret, cancel_secret
+            )
+
+    async def _add_or_renew_lease(
+        self, storage_index: bytes, renew_secret: bytes, cancel_secret: bytes
+    ) -> None:
         url = self._client.relative_url(
             "/storage/v1/lease/{}".format(_encode_si(storage_index))
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "PUT",
             url,
             lease_renew_secret=renew_secret,

From 4c2f241361b1f337fd0f667c555bdce86f818268 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 10:28:29 -0400
Subject: [PATCH 183/362] Add logging for limited_content().

---
 src/allmydata/storage/http_client.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 6f3bb05e1..ce69fdb76 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -4,13 +4,14 @@ HTTP client that talks to the HTTP storage server.
 
 from __future__ import annotations
 
-from eliot import start_action, register_exception_extractor
 from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
 
 from attrs import define, asdict, frozen, field
+from eliot import start_action, register_exception_extractor
+from eliot.twisted import DeferredContext
 
 # TODO Make sure to import Python version?
 from cbor2 import loads, dumps
@@ -160,9 +161,18 @@ def limited_content(
     trickle of data continues to arrive, it will continue to run.
     """
     d = succeed(None)
+
+    # Sadly, addTimeout() won't work because we need access to the IDelayedCall
+    # in order to reset it on each data chunk received.
     timeout = clock.callLater(60, d.cancel)
     collector = _LengthLimitedCollector(max_length, timeout)
 
+    with start_action(
+        action_type="allmydata:storage:http-client:limited-content",
+        max_length=max_length,
+    ).context() as action:
+        d = DeferredContext(d)
+
     # Make really sure everything gets called in Deferred context, treq might
     # call collector directly...
     d.addCallback(lambda _: treq.collect(response, collector))
@@ -177,7 +187,8 @@ def limited_content(
             timeout.cancel()
         return f
 
-    return d.addCallbacks(done, failed)
+    result = d.addCallbacks(done, failed)
+    return result.addActionFinish()
 
 
 @define

From 28ff24b3a71e83ba04eab003e3264a1ce0706c3e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 10:40:35 -0400
Subject: [PATCH 184/362] Add logging to immutable creation.

---
 src/allmydata/storage/http_client.py | 29 ++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 6bca822b5..403a2a3c6 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -707,6 +707,35 @@ class StorageClientImmutables(object):
         Result fires when creating the storage index succeeded, if creating the
         storage index failed the result will fire with an exception.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:create",
+            storage_index=si_to_human_readable(storage_index),
+            share_numbers=share_numbers,
+            allocated_size=allocated_size,
+        ) as ctx:
+            result = await self._create(
+                storage_index,
+                share_numbers,
+                allocated_size,
+                upload_secret,
+                lease_renew_secret,
+                lease_cancel_secret,
+            )
+            ctx.add_success_fields(
+                already_have=result.already_have, allocated=result.allocated
+            )
+            return result
+
+    async def _create(
+        self,
+        storage_index: bytes,
+        share_numbers: set[int],
+        allocated_size: int,
+        upload_secret: bytes,
+        lease_renew_secret: bytes,
+        lease_cancel_secret: bytes,
+    ) -> ImmutableCreateResult:
+        """Implementation of create()."""
         url = self._client.relative_url(
             "/storage/v1/immutable/" + _encode_si(storage_index)
         )

From 464b47619028e9d194e660ad461467acaf8986b4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 13:11:17 -0400
Subject: [PATCH 185/362] Work on 3.8.

---
 src/allmydata/storage/http_client.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index e21cfc5cc..fc165bedd 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -5,7 +5,7 @@ HTTP client that talks to the HTTP storage server.
 from __future__ import annotations
 
 from eliot import start_action, register_exception_extractor
-from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict
+from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict, Set
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
@@ -487,14 +487,14 @@ class StorageClientGeneral(object):
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
         decoded_response = cast(
-            dict[bytes, object],
+            Mapping[bytes, object],
             await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
         )
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
         cast(
-            dict[bytes, object],
+            Mapping[bytes, object],
             decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
         ).update(
             {
@@ -692,7 +692,7 @@ class StorageClientImmutables(object):
             message_to_serialize=message,
         )
         decoded_response = cast(
-            dict[str, set[int]],
+            Mapping[str, Set[int]],
             await self._client.decode_cbor(response, _SCHEMAS["allocate_buckets"]),
         )
         return ImmutableCreateResult(
@@ -774,7 +774,7 @@ class StorageClientImmutables(object):
                 response.code,
             )
         body = cast(
-            dict[str, list[dict[str, int]]],
+            Mapping[str, Sequence[Mapping[str, int]]],
             await self._client.decode_cbor(
                 response, _SCHEMAS["immutable_write_share_chunk"]
             ),
@@ -795,7 +795,7 @@ class StorageClientImmutables(object):
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> set[int]:
+    async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         Return the set of shares for a given storage index.
         """
@@ -808,7 +808,7 @@ class StorageClientImmutables(object):
         )
         if response.code == http.OK:
             body = cast(
-                set[int],
+                Set[int],
                 await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
             )
             return set(body)
@@ -881,9 +881,10 @@ class ReadTestWriteResult:
     reads: Mapping[int, Sequence[bytes]]
 
 
-# Result type for mutable read/test/write HTTP response.
+# Result type for mutable read/test/write HTTP response. Can't just use
+# dict[int,list[bytes]] because on Python 3.8 that will error out.
 MUTABLE_RTW = TypedDict(
-    "MUTABLE_RTW", {"success": bool, "data": dict[int, list[bytes]]}
+    "MUTABLE_RTW", {"success": bool, "data": Mapping[int, Sequence[bytes]]}
 )
 
 
@@ -958,7 +959,7 @@ class StorageClientMutables:
         )
 
     @async_to_deferred
-    async def list_shares(self, storage_index: bytes) -> set[int]:
+    async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         List the share numbers for a given storage index.
         """
@@ -968,7 +969,7 @@ class StorageClientMutables:
         response = await self._client.request("GET", url)
         if response.code == http.OK:
             return cast(
-                set[int],
+                Set[int],
                 await self._client.decode_cbor(
                     response, _SCHEMAS["mutable_list_shares"]
                 ),

From aca35a553dd48b3175a7cfbaf8f694a87311f404 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 13:30:38 -0400
Subject: [PATCH 186/362] Add logging to more immutable methods.

---
 src/allmydata/storage/http_client.py | 41 +++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index d91104fe6..693448b24 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -759,17 +759,28 @@ class StorageClientImmutables(object):
             allocated=decoded_response["allocated"],
         )
 
-    @inlineCallbacks
-    def abort_upload(
+    @async_to_deferred
+    async def abort_upload(
         self, storage_index: bytes, share_number: int, upload_secret: bytes
-    ) -> Deferred[None]:
+    ) -> None:
         """Abort the upload."""
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:abort-upload",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+        ):
+            return await self._abort_upload(storage_index, share_number, upload_secret)
+
+    async def _abort_upload(
+        self, storage_index: bytes, share_number: int, upload_secret: bytes
+    ) -> None:
+        """Implementation of ``abort_upload()``."""
         url = self._client.relative_url(
             "/storage/v1/immutable/{}/{}/abort".format(
                 _encode_si(storage_index), share_number
             )
         )
-        response = yield self._client.request(
+        response = await self._client.request(
             "PUT",
             url,
             upload_secret=upload_secret,
@@ -803,6 +814,28 @@ class StorageClientImmutables(object):
         whether the _complete_ share (i.e. all chunks, not just this one) has
         been uploaded.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:write-share-chunk",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+            offset=offset,
+            data_len=len(data),
+        ) as ctx:
+            result = await self._write_share_chunk(
+                storage_index, share_number, upload_secret, offset, data
+            )
+            ctx.add_success_fields(finished=result.finished)
+            return result
+
+    async def _write_share_chunk(
+        self,
+        storage_index: bytes,
+        share_number: int,
+        upload_secret: bytes,
+        offset: int,
+        data: bytes,
+    ) -> UploadProgress:
+        """Implementation of ``write_share_chunk()``."""
         url = self._client.relative_url(
             "/storage/v1/immutable/{}/{}".format(
                 _encode_si(storage_index), share_number

From d8f176bb8f7bd02e13f98de60f94d059c7c3ee95 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 13 Apr 2023 13:49:19 -0400
Subject: [PATCH 187/362] Type check fixes.

---
 src/allmydata/storage/http_client.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 693448b24..8be2adbdc 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -5,7 +5,17 @@ HTTP client that talks to the HTTP storage server.
 from __future__ import annotations
 
 
-from typing import Union, Optional, Sequence, Mapping, BinaryIO, cast, TypedDict, Set
+from typing import (
+    Union,
+    Optional,
+    Sequence,
+    Mapping,
+    BinaryIO,
+    cast,
+    TypedDict,
+    Set,
+    Dict,
+)
 from base64 import b64encode
 from io import BytesIO
 from os import SEEK_END
@@ -506,14 +516,14 @@ class StorageClientGeneral(object):
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
         decoded_response = cast(
-            Mapping[bytes, object],
+            Dict[bytes, object],
             await self._client.decode_cbor(response, _SCHEMAS["get_version"]),
         )
         # Add some features we know are true because the HTTP API
         # specification requires them and because other parts of the storage
         # client implementation assumes they will be present.
         cast(
-            Mapping[bytes, object],
+            Dict[bytes, object],
             decoded_response[b"http://allmydata.org/tahoe/protocols/storage/v1"],
         ).update(
             {

From 250efe7d24cd43a676a6a7116da35f6ab226401a Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 13 Apr 2023 16:42:02 -0600
Subject: [PATCH 188/362] leftover

---
 integration/conftest.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index e7e021016..b54e18e26 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -229,9 +229,6 @@ def introducer(reactor, temp_dir, flog_gatherer, request):
     config.set_config("node", "nickname", "introducer-tor")
     config.set_config("node", "web.port", "4561")
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
-    # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 5dcbc00989c94e314a018a8a9d79027796e95ffe Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:18:55 -0400
Subject: [PATCH 189/362] News fragment.

---
 newsfragments/4012.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/4012.bugfix

diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
new file mode 100644
index 000000000..97dfe6aad
--- /dev/null
+++ b/newsfragments/4012.bugfix
@@ -0,0 +1 @@
+The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file

From d7ee1637dfabc3654ea6be735fbcd2094d39c1f3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:22:06 -0400
Subject: [PATCH 190/362] Set a timeout.

---
 src/allmydata/scripts/common_http.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 95099a2eb..7542a045f 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -62,9 +62,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port)
+        c = http_client.HTTPConnection(host, port, timeout=60)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port)
+        c = http_client.HTTPSConnection(host, port, timeout=60)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From 67702572a9d4ce03c6614207234ae909672d6df5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:22:14 -0400
Subject: [PATCH 191/362] Do a little modernization.

---
 src/allmydata/scripts/common_http.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 7542a045f..4da1345c9 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -1,18 +1,11 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 from io import BytesIO
-from six.moves import urllib, http_client
+from http import client as http_client
+import urllib
 import six
 import allmydata # for __full_version__
 

From 1823dd4c03b3d715ef896453542d0ac10e7f4aad Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:24:00 -0400
Subject: [PATCH 192/362] Switch to a slightly larger block size.

---
 src/allmydata/scripts/common_http.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 4da1345c9..4c0319d3c 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -55,9 +55,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=60)
+        c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=60)
+        c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)
@@ -78,7 +78,7 @@ def do_http(method, url, body=b""):
         return BadResponse(url, err)
 
     while True:
-        data = body.read(8192)
+        data = body.read(65536)
         if not data:
             break
         c.send(data)

From 2916984114bdb9ef85df5a34aa439f45c5a14cab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:29:25 -0400
Subject: [PATCH 193/362] More modernization.

---
 src/allmydata/scripts/common_http.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 4c0319d3c..7d627a7ad 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -1,12 +1,11 @@
 """
-Ported to Python 3.
+Blocking HTTP client APIs.
 """
 
 import os
 from io import BytesIO
 from http import client as http_client
 import urllib
-import six
 import allmydata # for __full_version__
 
 from allmydata.util.encodingutil import quote_output
@@ -44,7 +43,7 @@ class BadResponse(object):
 def do_http(method, url, body=b""):
     if isinstance(body, bytes):
         body = BytesIO(body)
-    elif isinstance(body, six.text_type):
+    elif isinstance(body, str):
         raise TypeError("do_http body must be a bytestring, not unicode")
     else:
         # We must give a Content-Length header to twisted.web, otherwise it
@@ -87,16 +86,14 @@ def do_http(method, url, body=b""):
 
 
 def format_http_success(resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s %s" % (resp.status, six.ensure_text(resp.reason)),
+        "%s %s" % (resp.status, resp.reason),
         quotemarks=False)
 
 def format_http_error(msg, resp):
-    # ensure_text() shouldn't be necessary when Python 2 is dropped.
     return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, six.ensure_text(resp.reason),
-                           six.ensure_text(resp.read())),
+        "%s: %s %s\n%s" % (msg, resp.status, resp.reason,
+                           resp.read()),
         quotemarks=False)
 
 def check_http_error(resp, stderr):

From e4e6831497de5b653b04439425f7ebb5bce1d4d3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 10:55:40 -0400
Subject: [PATCH 194/362] Add logging to the rest of the immutable API
 operations.

---
 src/allmydata/storage/http_client.py | 49 +++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 8be2adbdc..1f477d1aa 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -886,21 +886,41 @@ class StorageClientImmutables(object):
             remaining.set(True, chunk["begin"], chunk["end"])
         return UploadProgress(finished=finished, required=remaining)
 
-    def read_share_chunk(
-        self, storage_index, share_number, offset, length
-    ):  # type: (bytes, int, int, int) -> Deferred[bytes]
+    @async_to_deferred
+    async def read_share_chunk(
+        self, storage_index: bytes, share_number: int, offset: int, length: int
+    ) -> bytes:
         """
         Download a chunk of data from a share.
         """
-        return read_share_chunk(
-            self._client, "immutable", storage_index, share_number, offset, length
-        )
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:read-share-chunk",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+            offset=offset,
+            length=length,
+        ) as ctx:
+            result = await read_share_chunk(
+                self._client, "immutable", storage_index, share_number, offset, length
+            )
+            ctx.add_success_fields(data_len=len(result))
+            return result
 
     @async_to_deferred
     async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         Return the set of shares for a given storage index.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:list-shares",
+            storage_index=si_to_human_readable(storage_index),
+        ) as ctx:
+            result = await self._list_shares(storage_index)
+            ctx.add_success_fields(shares=result)
+            return result
+
+    async def _list_shares(self, storage_index: bytes) -> Set[int]:
+        """Implementation of ``list_shares()``."""
         url = self._client.relative_url(
             "/storage/v1/immutable/{}/shares".format(_encode_si(storage_index))
         )
@@ -917,16 +937,23 @@ class StorageClientImmutables(object):
         else:
             raise ClientException(response.code)
 
-    def advise_corrupt_share(
+    @async_to_deferred
+    async def advise_corrupt_share(
         self,
         storage_index: bytes,
         share_number: int,
         reason: str,
-    ):
+    ) -> None:
         """Indicate a share has been corrupted, with a human-readable message."""
-        return advise_corrupt_share(
-            self._client, "immutable", storage_index, share_number, reason
-        )
+        with start_action(
+            action_type="allmydata:storage:http-client:immutable:advise-corrupt-share",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+            reason=reason,
+        ):
+            await advise_corrupt_share(
+                self._client, "immutable", storage_index, share_number, reason
+            )
 
 
 @frozen

From 2e06990c5c89355dc280a62ff7ea6b3796d2fdd4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 11:04:53 -0400
Subject: [PATCH 195/362] Remove bad assertion.

---
 src/allmydata/storage/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py
index 89e29b081..f6d986f85 100644
--- a/src/allmydata/storage/common.py
+++ b/src/allmydata/storage/common.py
@@ -41,7 +41,6 @@ def si_a2b(ascii_storageindex):
 
 def si_to_human_readable(storageindex: bytes) -> str:
     """Create human-readable string of storage index."""
-    assert len(storageindex) == 16
     return str(base32.b2a(storageindex), "ascii")
 
 def storage_index_to_dir(storageindex):

From 3395ee8fc59762197a3f446fe946085ce26827b9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 11:05:03 -0400
Subject: [PATCH 196/362] Add logging for mutable operations.

---
 src/allmydata/storage/http_client.py | 70 ++++++++++++++++++++++++----
 1 file changed, 60 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 1f477d1aa..ec8cd4ade 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -1044,6 +1044,29 @@ class StorageClientMutables:
         Given a mapping between share numbers and test/write vectors, the tests
         are done and if they are valid the writes are done.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:mutable:read-test-write",
+            storage_index=si_to_human_readable(storage_index),
+        ):
+            return await self._read_test_write_chunks(
+                storage_index,
+                write_enabler_secret,
+                lease_renew_secret,
+                lease_cancel_secret,
+                testwrite_vectors,
+                read_vector,
+            )
+
+    async def _read_test_write_chunks(
+        self,
+        storage_index: bytes,
+        write_enabler_secret: bytes,
+        lease_renew_secret: bytes,
+        lease_cancel_secret: bytes,
+        testwrite_vectors: dict[int, TestWriteVectors],
+        read_vector: list[ReadVector],
+    ) -> ReadTestWriteResult:
+        """Implementation of ``read_test_write_chunks()``."""
         url = self._client.relative_url(
             "/storage/v1/mutable/{}/read-test-write".format(_encode_si(storage_index))
         )
@@ -1073,25 +1096,45 @@ class StorageClientMutables:
         else:
             raise ClientException(response.code, (await response.content()))
 
-    def read_share_chunk(
+    @async_to_deferred
+    async def read_share_chunk(
         self,
         storage_index: bytes,
         share_number: int,
         offset: int,
         length: int,
-    ) -> Deferred[bytes]:
+    ) -> bytes:
         """
         Download a chunk of data from a share.
         """
-        return read_share_chunk(
-            self._client, "mutable", storage_index, share_number, offset, length
-        )
+        with start_action(
+            action_type="allmydata:storage:http-client:mutable:read-share-chunk",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+            offset=offset,
+            length=length,
+        ) as ctx:
+            result = await read_share_chunk(
+                self._client, "mutable", storage_index, share_number, offset, length
+            )
+            ctx.add_success_fields(data_len=len(result))
+            return result
 
     @async_to_deferred
     async def list_shares(self, storage_index: bytes) -> Set[int]:
         """
         List the share numbers for a given storage index.
         """
+        with start_action(
+            action_type="allmydata:storage:http-client:mutable:list-shares",
+            storage_index=si_to_human_readable(storage_index),
+        ) as ctx:
+            result = await self._list_shares(storage_index)
+            ctx.add_success_fields(shares=result)
+            return result
+
+    async def _list_shares(self, storage_index: bytes) -> Set[int]:
+        """Implementation of ``list_shares()``."""
         url = self._client.relative_url(
             "/storage/v1/mutable/{}/shares".format(_encode_si(storage_index))
         )
@@ -1106,13 +1149,20 @@ class StorageClientMutables:
         else:
             raise ClientException(response.code)
 
-    def advise_corrupt_share(
+    @async_to_deferred
+    async def advise_corrupt_share(
         self,
         storage_index: bytes,
         share_number: int,
         reason: str,
-    ):
+    ) -> None:
         """Indicate a share has been corrupted, with a human-readable message."""
-        return advise_corrupt_share(
-            self._client, "mutable", storage_index, share_number, reason
-        )
+        with start_action(
+            action_type="allmydata:storage:http-client:mutable:advise-corrupt-share",
+            storage_index=si_to_human_readable(storage_index),
+            share_number=share_number,
+            reason=reason,
+        ):
+            await advise_corrupt_share(
+                self._client, "mutable", storage_index, share_number, reason
+            )

From 2d81ddc297b336607bc8eac691913e7e3ac2f178 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 14 Apr 2023 11:15:47 -0400
Subject: [PATCH 197/362] Don't call str() on bytes.

---
 src/allmydata/scripts/common_http.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 7d627a7ad..a2cae5a85 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -92,7 +92,7 @@ def format_http_success(resp):
 
 def format_http_error(msg, resp):
     return quote_output(
-        "%s: %s %s\n%s" % (msg, resp.status, resp.reason,
+        "%s: %s %s\n%r" % (msg, resp.status, resp.reason,
                            resp.read()),
         quotemarks=False)
 

From 76ce54ea53e4e802da612f1f2cbc53c88e9764da Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 13:23:28 -0600
Subject: [PATCH 198/362] remove debugging

---
 integration/util.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index ac3fe2833..887602906 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -90,11 +90,9 @@ class _CollectOutputProtocol(ProcessProtocol):
             self.done.errback(reason)
 
     def outReceived(self, data):
-        print("OUT: {!r}".format(data))
         self.output.write(data)
 
     def errReceived(self, data):
-        print("ERR: {!r}".format(data))
         if self.capture_stderr:
             self.output.write(data)
 

From abfca04af5a8c7e43fa18351b1131ebcf741efd8 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 13:24:22 -0600
Subject: [PATCH 199/362] turn off i2p tests for now

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 96619a93a..597623d9c 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -133,7 +133,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
+def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     # ensure both nodes are connected to "a grid" by uploading

From d3c39f8604fd924bbac424ce201533b4656416b3 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:27:19 -0600
Subject: [PATCH 200/362] fix i2p introducer, different ports

---
 integration/conftest.py |  2 +-
 integration/test_i2p.py | 13 ++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index b54e18e26..f65c84141 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -227,7 +227,7 @@ def introducer(reactor, temp_dir, flog_gatherer, request):
 
     config = read_config(intro_dir, "tub.port")
     config.set_config("node", "nickname", "introducer-tor")
-    config.set_config("node", "web.port", "4561")
+    config.set_config("node", "web.port", "4562")
     config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 597623d9c..df619c6eb 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -68,13 +68,6 @@ def i2p_network(reactor, temp_dir, request):
     include_result=False,
 )
 def i2p_introducer(reactor, temp_dir, flog_gatherer, request):
-    config = '''
-[node]
-nickname = introducer_i2p
-web.port = 4561
-log_gatherer.furl = {log_furl}
-'''.format(log_furl=flog_gatherer)
-
     intro_dir = join(temp_dir, 'introducer_i2p')
     print("making introducer", intro_dir)
 
@@ -94,8 +87,10 @@ log_gatherer.furl = {log_furl}
         pytest_twisted.blockon(done_proto.done)
 
     # over-write the config file with our stuff
-    with open(join(intro_dir, 'tahoe.cfg'), 'w') as f:
-        f.write(config)
+    config = read_config(intro_dir, "tub.port")
+    config.set_config("node", "nickname", "introducer_i2p")
+    config.set_config("node", "web.port", "4563")
+    config.set_config("node", "log_gatherer.furl", flog_gatherer)
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 34cee7ff73c05d978354894f734d710d3a5c1a2c Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:44:52 -0600
Subject: [PATCH 201/362] missing import

---
 integration/test_i2p.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index df619c6eb..10abb7e30 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -23,6 +23,8 @@ from twisted.internet.error import ProcessExitedAlready
 from allmydata.test.common import (
     write_introducer,
 )
+from allmydata.node import read_config
+
 
 if which("docker") is None:
     pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True)

From 3ccb7c4d1c1f26991a3a467fe4a559738eac0638 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 15:45:17 -0600
Subject: [PATCH 202/362] re-enable i2p tests

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 10abb7e30..2aa1a536f 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-def __test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
+def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     # ensure both nodes are connected to "a grid" by uploading

From 8b81bd7ebef79617d75ab1d5d5745d50a3e212b9 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 16:33:52 -0600
Subject: [PATCH 203/362] remove more debug

---
 integration/util.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 887602906..177983e2e 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -138,7 +138,6 @@ class _MagicTextProtocol(ProcessProtocol):
         self.exited.callback(None)
 
     def outReceived(self, data):
-        print("OUT", data)
         data = str(data, sys.stdout.encoding)
         sys.stdout.write(data)
         self._output.write(data)
@@ -147,7 +146,6 @@ class _MagicTextProtocol(ProcessProtocol):
             self.magic_seen.callback(self)
 
     def errReceived(self, data):
-        print("ERR", data)
         data = str(data, sys.stderr.encoding)
         sys.stdout.write(data)
 

From 8652bb71ad0ece21f9f75a1c290ccdb4abed6e92 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 17:05:57 -0600
Subject: [PATCH 204/362] skip i2p tests again?

---
 integration/test_i2p.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 2aa1a536f..42b848130 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,6 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
+@pytest.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)

From b5f6fa8933c03d5de2069de47d67230ae3d641f2 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Fri, 14 Apr 2023 19:07:27 -0600
Subject: [PATCH 205/362] skip properly

---
 integration/test_i2p.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 42b848130..a94648593 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -130,7 +130,7 @@ def i2p_introducer_furl(i2p_introducer, temp_dir):
 
 
 @pytest_twisted.inlineCallbacks
-@pytest.skip("I2P tests are not functioning at all, for unknown reasons")
+@pytest.mark.skip("I2P tests are not functioning at all, for unknown reasons")
 def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl):
     yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)
     yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl)

From bed2d33427c449da15403b758d7730f81cce9ac4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:01:26 -0400
Subject: [PATCH 206/362] Fix lint.

---
 src/allmydata/storage/http_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index ec8cd4ade..071b9bdb1 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -181,7 +181,7 @@ def limited_content(
     with start_action(
         action_type="allmydata:storage:http-client:limited-content",
         max_length=max_length,
-    ).context() as action:
+    ).context():
         d = DeferredContext(d)
 
     # Make really sure everything gets called in Deferred context, treq might

From cda97e4fa63deac934e3085e8bc0edb0a30b85da Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:06:50 -0400
Subject: [PATCH 207/362] Remove pylint, replacing with faster alternative.

---
 tox.ini | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/tox.ini b/tox.ini
index 447745784..982157bf1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -100,9 +100,7 @@ commands =
 [testenv:codechecks]
 basepython = python3
 deps =
-     # Make sure we get a version of PyLint that respects config, and isn't too
-     # old.
-     pylint < 2.18, >2.14
+     ruff
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME
 setenv =
@@ -114,9 +112,10 @@ commands =
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
-         # PyLint has other useful checks, might want to enable them:
-         # http://pylint.pycqa.org/en/latest/technical_reference/features.html
-         pylint --disable=all --enable=cell-var-from-loop {posargs:{env:DEFAULT_FILES}}
+         # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
+         #       cell-var-from-loop.
+         # ruff could probably replace flake8 and perhaps above tools as well...
+         ruff check --select=B023 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From aafbb00333312e6fc77dcb9558a8b5c9cad75b9e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:10:09 -0400
Subject: [PATCH 208/362] Use ruff for trailing whitespace.

---
 misc/coding_tools/find-trailing-spaces.py | 44 -----------------------
 newsfragments/4014.minor                  |  0
 tox.ini                                   |  4 +--
 3 files changed, 2 insertions(+), 46 deletions(-)
 delete mode 100644 misc/coding_tools/find-trailing-spaces.py
 create mode 100644 newsfragments/4014.minor

diff --git a/misc/coding_tools/find-trailing-spaces.py b/misc/coding_tools/find-trailing-spaces.py
deleted file mode 100644
index 19e7e3c28..000000000
--- a/misc/coding_tools/find-trailing-spaces.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import print_function
-
-import os, sys
-
-from twisted.python import usage
-
-class Options(usage.Options):
-    optFlags = [
-        ("recursive", "r", "Search for .py files recursively"),
-        ]
-    def parseArgs(self, *starting_points):
-        self.starting_points = starting_points
-
-found = [False]
-
-def check(fn):
-    f = open(fn, "r")
-    for i,line in enumerate(f.readlines()):
-        if line == "\n":
-            continue
-        if line[-1] == "\n":
-            line = line[:-1]
-        if line.rstrip() != line:
-            # the %s:%d:%d: lets emacs' compile-mode jump to those locations
-            print("%s:%d:%d: trailing whitespace" % (fn, i+1, len(line)+1))
-            found[0] = True
-    f.close()
-
-o = Options()
-o.parseOptions()
-if o['recursive']:
-    for starting_point in o.starting_points:
-        for root, dirs, files in os.walk(starting_point):
-            for fn in [f for f in files if f.endswith(".py")]:
-                fn = os.path.join(root, fn)
-                check(fn)
-else:
-    for fn in o.starting_points:
-        check(fn)
-if found[0]:
-    sys.exit(1)
-sys.exit(0)
diff --git a/newsfragments/4014.minor b/newsfragments/4014.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/tox.ini b/tox.ini
index 982157bf1..a191d6078 100644
--- a/tox.ini
+++ b/tox.ini
@@ -111,11 +111,11 @@ commands =
          flake8 {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
-         python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
          # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
          #       cell-var-from-loop.
+         # W291,W293: Trailing whitespace.
          # ruff could probably replace flake8 and perhaps above tools as well...
-         ruff check --select=B023 {posargs:{env:DEFAULT_FILES}}
+         ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From 7b33931df2982bd35e0cdb38d0a1e1d77d34ce47 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:21:20 -0400
Subject: [PATCH 209/362] Replace flake8 with ruff.

---
 .gitignore |  2 ++
 .ruff.toml | 12 ++++++++++++
 setup.cfg  |  3 +++
 setup.py   |  3 +--
 tox.ini    |  7 +------
 5 files changed, 19 insertions(+), 8 deletions(-)
 create mode 100644 .ruff.toml

diff --git a/.gitignore b/.gitignore
index 7c7fa2afd..0cf688c54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -53,3 +53,5 @@ zope.interface-*.egg
 # This is the plaintext of the private environment needed for some CircleCI
 # operations.  It's never supposed to be checked in.
 secret-env-plain
+
+.ruff_cache
\ No newline at end of file
diff --git a/.ruff.toml b/.ruff.toml
new file mode 100644
index 000000000..75ff62c2d
--- /dev/null
+++ b/.ruff.toml
@@ -0,0 +1,12 @@
+select = [
+    # Pyflakes checks
+    "F",
+    # Prohibit tabs:
+    "W191",
+    # No trailing whitespace:
+    "W291",
+    "W293",
+    # Make sure we bind closure variables in a loop (equivalent to pylint
+    # cell-var-from-loop):
+    "B023",
+]
\ No newline at end of file
diff --git a/setup.cfg b/setup.cfg
index f4539279e..9415b3ab4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,6 +6,9 @@ develop = update_version develop
 bdist_egg = update_version bdist_egg
 bdist_wheel = update_version bdist_wheel
 
+# This has been replaced by ruff (see .ruff.toml), which has same checks as
+# flake8 plus many more, and is also faster. However, we're keeping this config
+# in case people still use flake8 in IDEs, etc..
 [flake8]
 # Enforce all pyflakes constraints, and also prohibit tabs for indentation.
 # Reference:
diff --git a/setup.py b/setup.py
index 854a333f1..3358aa6c6 100644
--- a/setup.py
+++ b/setup.py
@@ -399,12 +399,11 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "gpg",
           ],
           "test": [
-              "flake8",
               # Pin a specific pyflakes so we don't have different folks
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it
               # intentionally.
-              "pyflakes == 3.0.1",
+              "ruff==0.0.261",
               "coverage ~= 5.0",
               "mock",
               "tox ~= 3.0",
diff --git a/tox.ini b/tox.ini
index a191d6078..2daf8dca3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -108,14 +108,9 @@ setenv =
 	 # entire codebase, including various pieces of supporting code.
 	 DEFAULT_FILES=src integration static misc setup.py
 commands =
-         flake8 {posargs:{env:DEFAULT_FILES}}
+         ruff check {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
-         # B023: Find loop variables that aren't bound in a loop, equivalent of pylint
-         #       cell-var-from-loop.
-         # W291,W293: Trailing whitespace.
-         # ruff could probably replace flake8 and perhaps above tools as well...
-         ruff check --select=B023,W291,W293 {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From 6517cd4a48338bc13d991299d361e8c5b65fed22 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:22:27 -0400
Subject: [PATCH 210/362] Fix lint found by ruff.

---
 misc/checkers/check_load.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/misc/checkers/check_load.py b/misc/checkers/check_load.py
index d509b89ae..01a9ed832 100644
--- a/misc/checkers/check_load.py
+++ b/misc/checkers/check_load.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 """
 this is a load-generating client program. It does all of its work through a
 given tahoe node (specified by URL), and performs random reads and writes

From c05afb19dfd7ffb5f053aa2a52972403ccb4fa43 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 10:33:31 -0400
Subject: [PATCH 211/362] Don't install code, it's not necessary.

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index 2daf8dca3..99487bc83 100644
--- a/tox.ini
+++ b/tox.ini
@@ -99,8 +99,10 @@ commands =
 
 [testenv:codechecks]
 basepython = python3
+skip_install = true
 deps =
      ruff
+     towncrier
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME
 setenv =

From ce93a7b869ceb29b49d985125e80a80fa19dad98 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 11:29:43 -0400
Subject: [PATCH 212/362] News fragment.

---
 newsfragments/4015.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4015.minor

diff --git a/newsfragments/4015.minor b/newsfragments/4015.minor
new file mode 100644
index 000000000..e69de29bb

From 5da5a82a8cc36399c7dbc228109afee2a17ff21e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 12:02:04 -0400
Subject: [PATCH 213/362] Get rid of default mutable arguments.

---
 .ruff.toml                                |  2 ++
 src/allmydata/client.py                   |  6 +++---
 src/allmydata/dirnode.py                  |  4 +++-
 src/allmydata/hashtree.py                 |  7 +++++--
 src/allmydata/immutable/upload.py         |  4 +++-
 src/allmydata/interfaces.py               |  6 +++---
 src/allmydata/node.py                     | 12 ++++++++----
 src/allmydata/nodemaker.py                |  5 +++--
 src/allmydata/test/cli/wormholetesting.py |  3 ++-
 src/allmydata/test/no_network.py          |  4 +++-
 src/allmydata/test/test_system.py         | 21 +++++++++------------
 src/allmydata/test/web/test_web.py        | 12 +++++++++---
 src/allmydata/util/dbutil.py              |  4 +++-
 13 files changed, 56 insertions(+), 34 deletions(-)

diff --git a/.ruff.toml b/.ruff.toml
index 75ff62c2d..516255d2a 100644
--- a/.ruff.toml
+++ b/.ruff.toml
@@ -9,4 +9,6 @@ select = [
     # Make sure we bind closure variables in a loop (equivalent to pylint
     # cell-var-from-loop):
     "B023",
+    # Don't use mutable default arguments:
+    "B006",
 ]
\ No newline at end of file
diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 8a10fe9e7..1d959cb98 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -7,7 +7,7 @@ import os
 import stat
 import time
 import weakref
-from typing import Optional
+from typing import Optional, Iterable
 from base64 import urlsafe_b64encode
 from functools import partial
 # On Python 2 this will be the backported package:
@@ -189,7 +189,7 @@ class Terminator(service.Service):
         return service.Service.stopService(self)
 
 
-def read_config(basedir, portnumfile, generated_files=[]):
+def read_config(basedir, portnumfile, generated_files: Iterable=()):
     """
     Read and validate configuration for a client-style Node. See
     :method:`allmydata.node.read_config` for parameter meanings (the
@@ -1103,7 +1103,7 @@ class _Client(node.Node, pollmixin.PollMixin):
         # may get an opaque node if there were any problems.
         return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
 
-    def create_dirnode(self, initial_children={}, version=None):
+    def create_dirnode(self, initial_children=None, version=None):
         d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
         return d
 
diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py
index fdf373b45..ccd045b05 100644
--- a/src/allmydata/dirnode.py
+++ b/src/allmydata/dirnode.py
@@ -678,8 +678,10 @@ class DirectoryNode(object):
         return d
 
     # XXX: Too many arguments? Worthwhile to break into mutable/immutable?
-    def create_subdirectory(self, namex, initial_children={}, overwrite=True,
+    def create_subdirectory(self, namex, initial_children=None, overwrite=True,
                             mutable=True, mutable_version=None, metadata=None):
+        if initial_children is None:
+            initial_children = {}
         name = normalize(namex)
         if self.is_readonly():
             return defer.fail(NotWriteableError())
diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py
index 17467459b..57bdbd9a1 100644
--- a/src/allmydata/hashtree.py
+++ b/src/allmydata/hashtree.py
@@ -332,7 +332,7 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
             name += " (leaf [%d] of %d)" % (leafnum, numleaves)
         return name
 
-    def set_hashes(self, hashes={}, leaves={}):
+    def set_hashes(self, hashes=None, leaves=None):
         """Add a bunch of hashes to the tree.
 
         I will validate these to the best of my ability. If I already have a
@@ -382,7 +382,10 @@ class IncompleteHashTree(CompleteBinaryTreeMixin, list):
         corrupted or one of the received hashes was corrupted. If it raises
         NotEnoughHashesError, then the otherhashes dictionary was incomplete.
         """
-
+        if hashes is None:
+            hashes = {}
+        if leaves is None:
+            leaves = {}
         assert isinstance(hashes, dict)
         for h in hashes.values():
             assert isinstance(h, bytes)
diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index 0421de4e0..a331cc5db 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -1391,7 +1391,9 @@ class CHKUploader(object):
     def get_upload_status(self):
         return self._upload_status
 
-def read_this_many_bytes(uploadable, size, prepend_data=[]):
+def read_this_many_bytes(uploadable, size, prepend_data=None):
+    if prepend_data is None:
+        prepend_data = []
     if size == 0:
         return defer.succeed([])
     d = uploadable.read(size)
diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 467d0d450..201ab082e 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -1447,7 +1447,7 @@ class IDirectoryNode(IFilesystemNode):
         is a file, or if must_be_file is True and the child is a directory,
         I raise ChildOfWrongTypeError."""
 
-    def create_subdirectory(name, initial_children={}, overwrite=True,
+    def create_subdirectory(name, initial_children=None, overwrite=True,
                             mutable=True, mutable_version=None, metadata=None):
         """I create and attach a directory at the given name. The new
         directory can be empty, or it can be populated with children
@@ -2586,7 +2586,7 @@ class IClient(Interface):
         @return: a Deferred that fires with an IMutableFileNode instance.
         """
 
-    def create_dirnode(initial_children={}):
+    def create_dirnode(initial_children=None):
         """Create a new unattached dirnode, possibly with initial children.
 
         @param initial_children: dict with keys that are unicode child names,
@@ -2641,7 +2641,7 @@ class INodeMaker(Interface):
         for use by unit tests, to create mutable files that are smaller than
         usual."""
 
-    def create_new_mutable_directory(initial_children={}):
+    def create_new_mutable_directory(initial_children=None):
         """I create a new mutable directory, and return a Deferred that will
         fire with the IDirectoryNode instance when it is ready. If
         initial_children= is provided (a dict mapping unicode child name to
diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 58ee33ef5..6c3082b50 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -17,7 +17,7 @@ import errno
 from base64 import b32decode, b32encode
 from errno import ENOENT, EPERM
 from warnings import warn
-from typing import Union
+from typing import Union, Iterable
 
 import attr
 
@@ -172,7 +172,7 @@ def create_node_dir(basedir, readme_text):
             f.write(readme_text)
 
 
-def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
+def read_config(basedir, portnumfile, generated_files: Iterable = (), _valid_config=None):
     """
     Read and validate configuration.
 
@@ -741,7 +741,7 @@ def create_connection_handlers(config, i2p_provider, tor_provider):
 
 
 def create_tub(tub_options, default_connection_handlers, foolscap_connection_handlers,
-               handler_overrides={}, force_foolscap=False, **kwargs):
+               handler_overrides=None, force_foolscap=False, **kwargs):
     """
     Create a Tub with the right options and handlers. It will be
     ephemeral unless the caller provides certFile= in kwargs
@@ -755,6 +755,8 @@ def create_tub(tub_options, default_connection_handlers, foolscap_connection_han
     :param bool force_foolscap: If True, only allow Foolscap, not just HTTPS
         storage protocol.
     """
+    if handler_overrides is None:
+        handler_overrides = {}
     # We listen simultaneously for both Foolscap and HTTPS on the same port,
     # so we have to create a special Foolscap Tub for that to work:
     if force_foolscap:
@@ -922,7 +924,7 @@ def tub_listen_on(i2p_provider, tor_provider, tub, tubport, location):
 def create_main_tub(config, tub_options,
                     default_connection_handlers, foolscap_connection_handlers,
                     i2p_provider, tor_provider,
-                    handler_overrides={}, cert_filename="node.pem"):
+                    handler_overrides=None, cert_filename="node.pem"):
     """
     Creates a 'main' Foolscap Tub, typically for use as the top-level
     access point for a running Node.
@@ -943,6 +945,8 @@ def create_main_tub(config, tub_options,
     :param tor_provider: None, or a _Provider instance if txtorcon +
         Tor are installed.
     """
+    if handler_overrides is None:
+        handler_overrides = {}
     portlocation = _tub_portlocation(
         config,
         iputil.get_local_addresses_sync,
diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py
index 1b7ea5f45..39663bda9 100644
--- a/src/allmydata/nodemaker.py
+++ b/src/allmydata/nodemaker.py
@@ -135,8 +135,9 @@ class NodeMaker(object):
         d.addCallback(lambda res: n)
         return d
 
-    def create_new_mutable_directory(self, initial_children={}, version=None):
-        # initial_children must have metadata (i.e. {} instead of None)
+    def create_new_mutable_directory(self, initial_children=None, version=None):
+        if initial_children is None:
+            initial_children = {}
         for (name, (node, metadata)) in initial_children.items():
             precondition(isinstance(metadata, dict),
                          "create_new_mutable_directory requires metadata to be a dict, not None", metadata)
diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index d1a3bfd07..647798bc8 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -70,7 +70,8 @@ class MemoryWormholeServer(object):
         appid: str,
         relay_url: str,
         reactor: Any,
-        versions: Any={},
+        # Unfortunately we need a mutable default to match the real API
+        versions: Any={},  # noqa: B006
         delegate: Optional[Any]=None,
         journal: Optional[Any]=None,
         tor: Optional[Any]=None,
diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index ee1f48b17..e3b57fb95 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -476,7 +476,7 @@ class GridTestMixin(object):
         ])
 
     def set_up_grid(self, num_clients=1, num_servers=10,
-                    client_config_hooks={}, oneshare=False):
+                    client_config_hooks=None, oneshare=False):
         """
         Create a Tahoe-LAFS storage grid.
 
@@ -489,6 +489,8 @@ class GridTestMixin(object):
 
         :return: ``None``
         """
+        if client_config_hooks is None:
+            client_config_hooks = {}
         # self.basedir must be set
         port_assigner = SameProcessStreamEndpointAssigner()
         port_assigner.setUp()
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index d11a6e866..b3287bf3b 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -1,20 +1,13 @@
 """
 Ported to Python 3.
 """
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    # Don't import bytes since it causes issues on (so far unported) modules on Python 2.
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str  # noqa: F401
+from __future__ import annotations
 
 from past.builtins import chr as byteschr, long
 from six import ensure_text
 
 import os, re, sys, time, json
+from typing import Optional
 
 from bs4 import BeautifulSoup
 
@@ -56,10 +49,12 @@ from .common_util import run_cli_unicode
 
 
 class RunBinTahoeMixin(object):
-    def run_bintahoe(self, args, stdin=None, python_options=[], env=None):
+    def run_bintahoe(self, args, stdin=None, python_options:Optional[list[str]]=None, env=None):
         # test_runner.run_bintahoe has better unicode support but doesn't
         # support env yet and is also synchronous.  If we could get rid of
         # this in favor of that, though, it would probably be an improvement.
+        if python_options is None:
+            python_options = []
         command = sys.executable
         argv = python_options + ["-b", "-m", "allmydata.scripts.runner"] + args
 
@@ -1088,7 +1083,9 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             headers["content-type"] = "multipart/form-data; boundary=%s" % str(sepbase, "ascii")
         return self.POST2(urlpath, body, headers, use_helper)
 
-    def POST2(self, urlpath, body=b"", headers={}, use_helper=False):
+    def POST2(self, urlpath, body=b"", headers=None, use_helper=False):
+        if headers is None:
+            headers = {}
         if use_helper:
             url = self.helper_webish_url + urlpath
         else:
@@ -1409,7 +1406,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             rc,out,err = yield run_cli(verb, *args, nodeargs=nodeargs, **kwargs)
             defer.returnValue((out,err))
 
-        def _check_ls(out_and_err, expected_children, unexpected_children=[]):
+        def _check_ls(out_and_err, expected_children, unexpected_children=()):
             (out, err) = out_and_err
             self.failUnlessEqual(err, "")
             for s in expected_children:
diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 4c828817a..08dce0ac0 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -565,7 +565,9 @@ class WebMixin(TimezoneMixin):
         returnValue(data)
 
     @inlineCallbacks
-    def HEAD(self, urlpath, return_response=False, headers={}):
+    def HEAD(self, urlpath, return_response=False, headers=None):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         response = yield treq.request("head", url, persistent=False,
                                       headers=headers)
@@ -573,7 +575,9 @@ class WebMixin(TimezoneMixin):
             raise Error(response.code, response="")
         returnValue( ("", response.code, response.headers) )
 
-    def PUT(self, urlpath, data, headers={}):
+    def PUT(self, urlpath, data, headers=None):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         return do_http("put", url, data=data, headers=headers)
 
@@ -618,7 +622,9 @@ class WebMixin(TimezoneMixin):
         body, headers = self.build_form(**fields)
         return self.POST2(urlpath, body, headers)
 
-    def POST2(self, urlpath, body="", headers={}, followRedirect=False):
+    def POST2(self, urlpath, body="", headers=None, followRedirect=False):
+        if headers is None:
+            headers = {}
         url = self.webish_url + urlpath
         if isinstance(body, str):
             body = body.encode("utf-8")
diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py
index 916382972..45e59cf00 100644
--- a/src/allmydata/util/dbutil.py
+++ b/src/allmydata/util/dbutil.py
@@ -25,7 +25,7 @@ class DBError(Exception):
 
 
 def get_db(dbfile, stderr=sys.stderr,
-           create_version=(None, None), updaters={}, just_create=False, dbname="db",
+           create_version=(None, None), updaters=None, just_create=False, dbname="db",
            ):
     """Open or create the given db file. The parent directory must exist.
     create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table.
@@ -33,6 +33,8 @@ def get_db(dbfile, stderr=sys.stderr,
     to get from ver=1 to ver=2. Returns a (sqlite3,db) tuple, or raises
     DBError.
     """
+    if updaters is None:
+        updaters = {}
     must_create = not os.path.exists(dbfile)
     try:
         db = sqlite3.connect(dbfile)

From 2a4dcb7a27cd0fd0592a9002e13c644a07ea8420 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 17 Apr 2023 13:08:26 -0400
Subject: [PATCH 214/362] More checks that are probably useful (doesn't trigger
 anything at the moment).

---
 .ruff.toml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.ruff.toml b/.ruff.toml
index 516255d2a..2dd6b59b5 100644
--- a/.ruff.toml
+++ b/.ruff.toml
@@ -9,6 +9,10 @@ select = [
     # Make sure we bind closure variables in a loop (equivalent to pylint
     # cell-var-from-loop):
     "B023",
+    # Don't silence exceptions in finally by accident:
+    "B012",
     # Don't use mutable default arguments:
     "B006",
+    # Errors from PyLint:
+    "PLE",
 ]
\ No newline at end of file

From 1371ffe9dc7e6a6b0346daad1603f6414bbd1fc7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:14:26 -0400
Subject: [PATCH 215/362] Just have ruff in one place.

---
 setup.py | 5 -----
 tox.ini  | 4 +++-
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/setup.py b/setup.py
index 3358aa6c6..2418c6dbe 100644
--- a/setup.py
+++ b/setup.py
@@ -399,11 +399,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "gpg",
           ],
           "test": [
-              # Pin a specific pyflakes so we don't have different folks
-              # disagreeing on what is or is not a lint issue.  We can bump
-              # this version from time to time, but we will do it
-              # intentionally.
-              "ruff==0.0.261",
               "coverage ~= 5.0",
               "mock",
               "tox ~= 3.0",
diff --git a/tox.ini b/tox.ini
index 99487bc83..5f18b6b95 100644
--- a/tox.ini
+++ b/tox.ini
@@ -101,7 +101,9 @@ commands =
 basepython = python3
 skip_install = true
 deps =
-     ruff
+     # Pin a specific version so we get consistent outcomes; update this
+     # occasionally:
+     ruff == 0.0.263
      towncrier
 # On macOS, git inside of towncrier needs $HOME.
 passenv = HOME

From ebed5100b9cf2a208875eb2977da23364559881d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:16:12 -0400
Subject: [PATCH 216/362] Switch to longer timeout so it's unlikely to impact
 users.

---
 newsfragments/4012.bugfix            | 2 +-
 src/allmydata/scripts/common_http.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
index 97dfe6aad..24d5bb49a 100644
--- a/newsfragments/4012.bugfix
+++ b/newsfragments/4012.bugfix
@@ -1 +1 @@
-The command-line tools now have a 60-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
+The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index a2cae5a85..46676b3f5 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -54,9 +54,9 @@ def do_http(method, url, body=b""):
         assert body.read
     scheme, host, port, path = parse_url(url)
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=60, blocksize=65536)
+        c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=60, blocksize=65536)
+        c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From 558e3bf79785fef5a8c3525f323590b9c7c15e36 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 08:46:57 -0400
Subject: [PATCH 217/362] Fix unnecessary conversion.

---
 src/allmydata/storage/http_client.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index fc165bedd..f786b8f30 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -807,11 +807,10 @@ class StorageClientImmutables(object):
             url,
         )
         if response.code == http.OK:
-            body = cast(
+            return cast(
                 Set[int],
                 await self._client.decode_cbor(response, _SCHEMAS["list_shares"]),
             )
-            return set(body)
         else:
             raise ClientException(response.code)
 

From 3d2e4d0798b874c493dc6b86487b527e36aa3324 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:26:58 -0400
Subject: [PATCH 218/362] note about port selection

---
 integration/conftest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/integration/conftest.py b/integration/conftest.py
index f65c84141..f670c6486 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -289,6 +289,8 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
             request,
             (
                 'create-introducer',
+                # The control port should agree with the configuration of the
+                # Tor network we bootstrap with chutney.
                 '--tor-control-port', 'tcp:localhost:8007',
                 '--hide-ip',
                 '--listen=tor',

From c595eea33e78eac579b3f3b63163a0354348a0d6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:27:51 -0400
Subject: [PATCH 219/362] always set the "start time" timeout

in both the "we installed it ourselves" and the "we found an existing
installation" cases.
---
 integration/conftest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index f670c6486..eaf740190 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -511,7 +511,6 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
         chutney_dir,
         {
             "PYTHONPATH": join(chutney_dir, "lib"),
-            "CHUTNEY_START_TIME": "600",  # default is 60
         }
     )
 
@@ -534,6 +533,10 @@ def tor_network(reactor, temp_dir, chutney, request):
 
     env = environ.copy()
     env.update(chutney_env)
+    env.update({
+        # default is 60, probably too short for reliable automated use.
+        "CHUTNEY_START_TIME": "600",
+    })
     chutney_argv = (sys.executable, '-m', 'chutney.TorNet')
     def chutney(argv):
         proto = _DumpOutputProtocol(None)

From ba387453cf95ff04a322d8ba4531d81a5f31d7b2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:30:53 -0400
Subject: [PATCH 220/362] it's a bug fix!

it's user-facing!
---
 newsfragments/3999.bugfix | 1 +
 newsfragments/3999.minor  | 0
 2 files changed, 1 insertion(+)
 create mode 100644 newsfragments/3999.bugfix
 delete mode 100644 newsfragments/3999.minor

diff --git a/newsfragments/3999.bugfix b/newsfragments/3999.bugfix
new file mode 100644
index 000000000..a8a8396f4
--- /dev/null
+++ b/newsfragments/3999.bugfix
@@ -0,0 +1 @@
+A bug where Introducer nodes configured to listen on Tor or I2P would not actually do so has been fixed.
\ No newline at end of file
diff --git a/newsfragments/3999.minor b/newsfragments/3999.minor
deleted file mode 100644
index e69de29bb..000000000

From 825bcf3f3b8ca5a924d8c3075db653f9e5bf3c99 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:31:04 -0400
Subject: [PATCH 221/362] revert reformatting

---
 integration/conftest.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index eaf740190..b0d8da90f 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -507,12 +507,7 @@ def chutney(reactor, temp_dir: str) -> tuple[str, dict[str, str]]:
     )
     pytest_twisted.blockon(proto.done)
 
-    return (
-        chutney_dir,
-        {
-            "PYTHONPATH": join(chutney_dir, "lib"),
-        }
-    )
+    return (chutney_dir, {"PYTHONPATH": join(chutney_dir, "lib")})
 
 
 @pytest.fixture(scope='session')

From fbb5f4c359800e606cc3d29d39d80896292e4c40 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 25 Apr 2023 09:31:10 -0400
Subject: [PATCH 222/362] slightly clarified comment

---
 integration/conftest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index b0d8da90f..cb590ef6f 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -547,7 +547,7 @@ def tor_network(reactor, temp_dir, chutney, request):
     # now, as per Chutney's README, we have to create the network
     pytest_twisted.blockon(chutney(("configure", basic_network)))
 
-    # ensure we will tear down the network right before we start it
+    # before we start the network, ensure we will tear down at the end
     def cleanup():
         print("Tearing down Chutney Tor network")
         try:

From f9a1eedaeadb818315bef8158902a47c863bbc65 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 25 Apr 2023 12:31:37 -0400
Subject: [PATCH 223/362] Make timeout optional, enable it only for integration
 tests.

---
 integration/conftest.py              |  6 ++++++
 newsfragments/4012.bugfix            |  1 -
 newsfragments/4012.minor             |  0
 src/allmydata/scripts/common_http.py | 11 +++++++++--
 4 files changed, 15 insertions(+), 3 deletions(-)
 delete mode 100644 newsfragments/4012.bugfix
 create mode 100644 newsfragments/4012.minor

diff --git a/integration/conftest.py b/integration/conftest.py
index 879649588..d76b2a9c7 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -4,6 +4,7 @@ Ported to Python 3.
 
 from __future__ import annotations
 
+import os
 import sys
 import shutil
 from time import sleep
@@ -49,6 +50,11 @@ from .util import (
 )
 
 
+# No reason for HTTP requests to take longer than two minutes in the
+# integration tests. See allmydata/scripts/common_http.py for usage.
+os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
+
+
 # pytest customization hooks
 
 def pytest_addoption(parser):
diff --git a/newsfragments/4012.bugfix b/newsfragments/4012.bugfix
deleted file mode 100644
index 24d5bb49a..000000000
--- a/newsfragments/4012.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-The command-line tools now have a 300-second timeout on individual network reads/writes/connects; previously they could block forever in some situations.
\ No newline at end of file
diff --git a/newsfragments/4012.minor b/newsfragments/4012.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/scripts/common_http.py b/src/allmydata/scripts/common_http.py
index 46676b3f5..f138b9c07 100644
--- a/src/allmydata/scripts/common_http.py
+++ b/src/allmydata/scripts/common_http.py
@@ -53,10 +53,17 @@ def do_http(method, url, body=b""):
         assert body.seek
         assert body.read
     scheme, host, port, path = parse_url(url)
+
+    # For testing purposes, allow setting a timeout on HTTP requests. If this
+    # ever become a user-facing feature, this should probably be a CLI option?
+    timeout = os.environ.get("__TAHOE_CLI_HTTP_TIMEOUT", None)
+    if timeout is not None:
+        timeout = float(timeout)
+
     if scheme == "http":
-        c = http_client.HTTPConnection(host, port, timeout=300, blocksize=65536)
+        c = http_client.HTTPConnection(host, port, timeout=timeout, blocksize=65536)
     elif scheme == "https":
-        c = http_client.HTTPSConnection(host, port, timeout=300, blocksize=65536)
+        c = http_client.HTTPSConnection(host, port, timeout=timeout, blocksize=65536)
     else:
         raise ValueError("unknown scheme '%s', need http or https" % scheme)
     c.putrequest(method, path)

From c0e49064ce64eb2860dba6c2957a86485c1a1e41 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 09:50:02 -0400
Subject: [PATCH 224/362] Attempt to get more information about client unready
 state

---
 integration/util.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 177983e2e..39e5dfa6d 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -604,19 +604,27 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             print("waiting because '{}'".format(e))
             time.sleep(1)
             continue
+        servers = js['servers']
 
-        if len(js['servers']) < minimum_number_of_servers:
-            print(f"waiting because {js['servers']} is fewer than required ({minimum_number_of_servers})")
+        if len(servers) < minimum_number_of_servers:
+            print(f"waiting because {servers} is fewer than required ({minimum_number_of_servers})")
             time.sleep(1)
             continue
+
+        print(
+            f"Now: {time.ctime()}\n"
+            f"Server last-received-data: {[time.ctime(s['last_received_data']) for s in servers]}"
+        )
+
         server_times = [
             server['last_received_data']
-            for server in js['servers']
+            for server in servers
         ]
         # if any times are null/None that server has never been
         # contacted (so it's down still, probably)
-        if any(t is None for t in server_times):
-            print("waiting because at least one server not contacted")
+        never_received_data = server_times.count(None)
+        if never_received_data > 0:
+            print(f"waiting because {never_received_data} server(s) not contacted")
             time.sleep(1)
             continue
 

From 8f1d1cc1a0db48a1aa219b9a8814ef4201206098 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:23:06 -0400
Subject: [PATCH 225/362] Include node name in the logging output from
 subprocesses.

---
 integration/conftest.py |  6 +++---
 integration/test_i2p.py |  4 ++--
 integration/util.py     | 11 ++++++-----
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index d76b2a9c7..69d1934b0 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -161,7 +161,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request):
     )
     pytest_twisted.blockon(out_protocol.done)
 
-    twistd_protocol = _MagicTextProtocol("Gatherer waiting at")
+    twistd_protocol = _MagicTextProtocol("Gatherer waiting at", "gatherer")
     twistd_process = reactor.spawnProcess(
         twistd_protocol,
         which('twistd')[0],
@@ -244,7 +244,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
@@ -320,7 +320,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = _MagicTextProtocol('introducer running')
+    protocol = _MagicTextProtocol('introducer running', "tor_introducer")
     transport = _tahoe_runner_optional_coverage(
         protocol,
         reactor,
diff --git a/integration/test_i2p.py b/integration/test_i2p.py
index 96619a93a..4d4dbe620 100644
--- a/integration/test_i2p.py
+++ b/integration/test_i2p.py
@@ -35,7 +35,7 @@ if sys.platform.startswith('win'):
 @pytest.fixture
 def i2p_network(reactor, temp_dir, request):
     """Fixture to start up local i2pd."""
-    proto = util._MagicTextProtocol("ephemeral keys")
+    proto = util._MagicTextProtocol("ephemeral keys", "i2pd")
     reactor.spawnProcess(
         proto,
         which("docker"),
@@ -99,7 +99,7 @@ log_gatherer.furl = {log_furl}
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.
-    protocol = util._MagicTextProtocol('introducer running')
+    protocol = util._MagicTextProtocol('introducer running', "introducer")
     transport = util._tahoe_runner_optional_coverage(
         protocol,
         reactor,
diff --git a/integration/util.py b/integration/util.py
index 05fef8fed..b1692a7a3 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -12,7 +12,7 @@ import sys
 import time
 import json
 from os import mkdir, environ
-from os.path import exists, join
+from os.path import exists, join, basename
 from io import StringIO, BytesIO
 from subprocess import check_output
 
@@ -129,8 +129,9 @@ class _MagicTextProtocol(ProcessProtocol):
     and then .callback()s on self.done and .errback's if the process exits
     """
 
-    def __init__(self, magic_text):
+    def __init__(self, magic_text: str, name: str) -> None:
         self.magic_seen = Deferred()
+        self.name = f"{name}: "
         self.exited = Deferred()
         self._magic_text = magic_text
         self._output = StringIO()
@@ -140,7 +141,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def outReceived(self, data):
         data = str(data, sys.stdout.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
@@ -148,7 +149,7 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         data = str(data, sys.stderr.encoding)
-        sys.stdout.write(data)
+        sys.stdout.write(self.name + data)
 
 
 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:
@@ -282,7 +283,7 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
     """
     if magic_text is None:
         magic_text = "client running"
-    protocol = _MagicTextProtocol(magic_text)
+    protocol = _MagicTextProtocol(magic_text, basename(node_dir))
 
     # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old
     # "start" command.

From 86a513282f67556e1791faa7ab09ac4fc3f5b6b7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:36:39 -0400
Subject: [PATCH 226/362] Include Foolscap logging in node output in
 integration tests.

---
 integration/conftest.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/integration/conftest.py b/integration/conftest.py
index 69d1934b0..cdc65f9e8 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -54,6 +54,10 @@ from .util import (
 # integration tests. See allmydata/scripts/common_http.py for usage.
 os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
 
+# Make Foolscap logging go into Twisted logging, so that integration test logs
+# include extra information
+# (https://github.com/warner/foolscap/blob/latest-release/doc/logging.rst):
+os.environ["FLOGTOTWISTED"] = "1"
 
 # pytest customization hooks
 

From 9faf742b411e5dddd11e098eee7421b7154e3b25 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 10:36:59 -0400
Subject: [PATCH 227/362] News file.

---
 newsfragments/4018.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4018.minor

diff --git a/newsfragments/4018.minor b/newsfragments/4018.minor
new file mode 100644
index 000000000..e69de29bb

From 3d0c872f4c3751d48c5cf1f2392ec431b11235f8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:44:10 -0400
Subject: [PATCH 228/362] restrict CI jobs to the wheelhouse

---
 .circleci/setup-virtualenv.sh | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index feccbbf23..7087c5120 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -26,12 +26,7 @@ shift || :
 
 # Tell pip where it can find any existing wheels.
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
-
-# It is tempting to also set PIP_NO_INDEX=1 but (a) that will cause problems
-# between the time dependencies change and the images are re-built and (b) the
-# upcoming-deprecations job wants to install some dependencies from github and
-# it's awkward to get that done any earlier than the tox run.  So, we don't
-# set it.
+export PIP_NO_INDEX="1"
 
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \

From f9269158baf5103e014bbd439944690f3138c1af Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:46:58 -0400
Subject: [PATCH 229/362] news fragment

---
 newsfragments/4019.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4019.minor

diff --git a/newsfragments/4019.minor b/newsfragments/4019.minor
new file mode 100644
index 000000000..e69de29bb

From 4d5b9f2d0c88e411b0cb032a46b8b681d984703c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 10:48:46 -0400
Subject: [PATCH 230/362] match the version in the docker image

it is maybe wrong that we pin a specific version here and also only include a
specific version (probably some interpretation of "the most recent release")
in the docker image...
---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 5f18b6b95..6e56496d4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,7 +36,7 @@ deps =
      # happening at the time.  The versions selected here are just the current
      # versions at the time.  Bumping them to keep up with future releases is
      # fine as long as those releases are known to actually work.
-     pip==22.0.3
+     pip==22.3.1
      setuptools==60.9.1
      wheel==0.37.1
      subunitreporter==22.2.0

From 58ccecff5414e7ceded6aaac3666e289aa54dd5b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:17:19 -0400
Subject: [PATCH 231/362] Take a step towards unifying dependency pins used by
 tox env and Docker image building

---
 .circleci/populate-wheelhouse.sh | 21 +++--------------
 setup.py                         | 39 +++++++++++++++++++++++++++++---
 tox.ini                          | 19 +---------------
 3 files changed, 40 insertions(+), 39 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 374ca0adb..f103a6af8 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -3,18 +3,6 @@
 # https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/
 set -euxo pipefail
 
-# Basic Python packages that you just need to have around to do anything,
-# practically speaking.
-BASIC_DEPS="pip wheel"
-
-# Python packages we need to support the test infrastructure.  *Not* packages
-# Tahoe-LAFS itself (implementation or test suite) need.
-TEST_DEPS="tox~=3.0"
-
-# Python packages we need to generate test reports for CI infrastructure.
-# *Not* packages Tahoe-LAFS itself (implement or test suite) need.
-REPORTING_DEPS="python-subunit junitxml subunitreporter"
-
 # The filesystem location of the wheelhouse which we'll populate with wheels
 # for all of our dependencies.
 WHEELHOUSE_PATH="$1"
@@ -41,15 +29,12 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
-    "${PROJECT_ROOT}"[test] \
-    ${BASIC_DEPS} \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
+    "${PROJECT_ROOT}"[testenv] \
+    "${PROJECT_ROOT}"[test]
 
 # Not strictly wheelhouse population but ... Note we omit basic deps here.
 # They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
 # have to ask.
 "${PIP}" \
     install \
-    ${TEST_DEPS} \
-    ${REPORTING_DEPS}
+    "${PROJECT_ROOT}"[testenv]
diff --git a/setup.py b/setup.py
index 2418c6dbe..6e16381e6 100644
--- a/setup.py
+++ b/setup.py
@@ -398,10 +398,44 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "dulwich",
               "gpg",
           ],
+
+          # Here are the dependencies required to set up a reproducible test
+          # environment.  This could be for CI or local development.  These
+          # are *not* library dependencies of the test suite itself.  They are
+          # the tools we use to run the test suite at all.
+          "testenv": [
+              # Pin all of these versions for the same reason you ever want to
+              # pin anything: to prevent new releases with regressions from
+              # introducing spurious failures into CI runs for whatever
+              # development work is happening at the time.  The versions
+              # selected here are just the current versions at the time.
+              # Bumping them to keep up with future releases is fine as long
+              # as those releases are known to actually work.
+
+              # XXX For the moment, unpinned so we use whatever is in the
+              # image.  The images vary in what versions they have. :/
+              "pip", # ==22.0.3",
+              "wheel", # ==0.37.1"
+              "setuptools", # ==60.9.1",
+              "tox", # ~=3.0",
+              "subunitreporter", # ==22.2.0",
+              "python-subunit", # ==1.4.2",
+              "junitxml", # ==0.7",
+              "coverage", # ~= 5.0",
+
+              # As an exception, we don't pin certifi because it contains CA
+              # certificates which necessarily change over time.  Pinning this
+              # is guaranteed to cause things to break eventually as old
+              # certificates expire and as new ones are used in the wild that
+              # aren't present in whatever version we pin.  Hopefully there
+              # won't be functionality regressions in new releases of this
+              # package that cause us the kind of suffering we're trying to
+              # avoid with the above pins.
+              "certifi",
+          ],
+
           "test": [
-              "coverage ~= 5.0",
               "mock",
-              "tox ~= 3.0",
               "pytest",
               "pytest-twisted",
               "hypothesis >= 3.6.1",
@@ -410,7 +444,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "fixtures",
               "beautifulsoup4",
               "html5lib",
-              "junitxml",
               # Pin old version until
               # https://github.com/paramiko/paramiko/issues/1961 is fixed.
               "paramiko < 2.9",
diff --git a/tox.ini b/tox.ini
index 6e56496d4..3b7a96503 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,24 +30,7 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
 # available to those systems.  Installing it ahead of time (with pip) avoids
 # this problem.
 deps =
-     # Pin all of these versions for the same reason you ever want to pin
-     # anything: to prevent new releases with regressions from introducing
-     # spurious failures into CI runs for whatever development work is
-     # happening at the time.  The versions selected here are just the current
-     # versions at the time.  Bumping them to keep up with future releases is
-     # fine as long as those releases are known to actually work.
-     pip==22.3.1
-     setuptools==60.9.1
-     wheel==0.37.1
-     subunitreporter==22.2.0
-     # As an exception, we don't pin certifi because it contains CA
-     # certificates which necessarily change over time.  Pinning this is
-     # guaranteed to cause things to break eventually as old certificates
-     # expire and as new ones are used in the wild that aren't present in
-     # whatever version we pin.  Hopefully there won't be functionality
-     # regressions in new releases of this package that cause us the kind of
-     # suffering we're trying to avoid with the above pins.
-     certifi
+     .[testenv]
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.

From 66d3de059432a3e1d12b14b50b66ac2ea263929d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:31:26 -0400
Subject: [PATCH 232/362] narrowly pin these dependencies

This will break because these are not the versions on all Docker CI images but
we need to pin them to rebuild those images with the correct versions.

Rebuilding the images might break CI for all other branches.  But!  It's
broken already, so it's not like it's any worse.
---
 setup.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/setup.py b/setup.py
index 6e16381e6..127d17328 100644
--- a/setup.py
+++ b/setup.py
@@ -411,17 +411,14 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # selected here are just the current versions at the time.
               # Bumping them to keep up with future releases is fine as long
               # as those releases are known to actually work.
-
-              # XXX For the moment, unpinned so we use whatever is in the
-              # image.  The images vary in what versions they have. :/
-              "pip", # ==22.0.3",
-              "wheel", # ==0.37.1"
-              "setuptools", # ==60.9.1",
-              "tox", # ~=3.0",
-              "subunitreporter", # ==22.2.0",
-              "python-subunit", # ==1.4.2",
-              "junitxml", # ==0.7",
-              "coverage", # ~= 5.0",
+              "pip==22.0.3",
+              "wheel==0.37.1"
+              "setuptools==60.9.1",
+              "tox~=3.0",
+              "subunitreporter==22.2.0",
+              "python-subunit==1.4.2",
+              "junitxml==0.7",
+              "coverage ~= 5.0",
 
               # As an exception, we don't pin certifi because it contains CA
               # certificates which necessarily change over time.  Pinning this

From 29961a08b2c4097ee527043216a41b17a7da048d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 27 Apr 2023 11:40:49 -0400
Subject: [PATCH 233/362] typo in the requirements list...

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 127d17328..b1d54b43c 100644
--- a/setup.py
+++ b/setup.py
@@ -412,7 +412,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # Bumping them to keep up with future releases is fine as long
               # as those releases are known to actually work.
               "pip==22.0.3",
-              "wheel==0.37.1"
+              "wheel==0.37.1",
               "setuptools==60.9.1",
               "tox~=3.0",
               "subunitreporter==22.2.0",

From 0f200e422e3278091843f6384fa91cfe5bf1101c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 27 Apr 2023 15:48:49 -0400
Subject: [PATCH 234/362] Give it more time.

---
 integration/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index b1692a7a3..c58fc2e93 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
 
 
 @run_in_thread
-def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
+def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
     `TahoeProcess` instance usually from a fixture e.g. `alice`) to be

From f6e4e862a9d1bb8cc16d27b791e76aec09a298e6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:50:50 -0400
Subject: [PATCH 235/362] Require that the actual test run step do this part

Keep this script to wheelhouse population.  We might be giving up a tiny bit
of performance here but let's make it work at all before we make it fast.
---
 .circleci/populate-wheelhouse.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index f103a6af8..239c8367b 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -31,10 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
     "${PROJECT_ROOT}"[test]
-
-# Not strictly wheelhouse population but ... Note we omit basic deps here.
-# They're in the wheelhouse if Tahoe-LAFS wants to drag them in but it will
-# have to ask.
-"${PIP}" \
-    install \
-    "${PROJECT_ROOT}"[testenv]

From 70caa22370b9646096e83cb18057287a3946698f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:51:45 -0400
Subject: [PATCH 236/362] have to do certifi in tox.ini

by the time setup.py is being processed it is too late for certifi to help
---
 setup.py | 10 ----------
 tox.ini  | 22 +++++++++++++++-------
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/setup.py b/setup.py
index b1d54b43c..599eb5898 100644
--- a/setup.py
+++ b/setup.py
@@ -419,16 +419,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "python-subunit==1.4.2",
               "junitxml==0.7",
               "coverage ~= 5.0",
-
-              # As an exception, we don't pin certifi because it contains CA
-              # certificates which necessarily change over time.  Pinning this
-              # is guaranteed to cause things to break eventually as old
-              # certificates expire and as new ones are used in the wild that
-              # aren't present in whatever version we pin.  Hopefully there
-              # won't be functionality regressions in new releases of this
-              # package that cause us the kind of suffering we're trying to
-              # avoid with the above pins.
-              "certifi",
           ],
 
           "test": [
diff --git a/tox.ini b/tox.ini
index 3b7a96503..609a78b13 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,14 +23,22 @@ minversion = 2.4
 
 [testenv]
 passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH
-# Get "certifi" to avoid bug #2913. Basically if a `setup_requires=...` causes
-# a package to be installed (with setuptools) then it'll fail on certain
-# platforms (travis's OX-X 10.12, Slackware 14.2) because PyPI's TLS
-# requirements (TLS >= 1.2) are incompatible with the old TLS clients
-# available to those systems.  Installing it ahead of time (with pip) avoids
-# this problem.
 deps =
-     .[testenv]
+    # We pull in certify *here* to avoid bug #2913. Basically if a
+    # `setup_requires=...` causes a package to be installed (with setuptools)
+    # then it'll fail on certain platforms (travis's OX-X 10.12, Slackware
+    # 14.2) because PyPI's TLS requirements (TLS >= 1.2) are incompatible with
+    # the old TLS clients available to those systems.  Installing it ahead of
+    # time (with pip) avoids this problem.
+    #
+    # We don't pin an exact version of it because it contains CA certificates
+    # which necessarily change over time.  Pinning this is guaranteed to cause
+    # things to break eventually as old certificates expire and as new ones
+    # are used in the wild that aren't present in whatever version we pin.
+    # Hopefully there won't be functionality regressions in new releases of
+    # this package that cause us the kind of suffering we're trying to avoid
+    # with the above pins.
+    certifi
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.

From 17706f582ee54532b7a117b3b97ffce3e0108b7a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:52:05 -0400
Subject: [PATCH 237/362] use tox testenv `extras` to request testenv too

---
 tox.ini | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 609a78b13..2edb15a0b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -43,9 +43,14 @@ deps =
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.
 usedevelop = False
-# We use extras=test to get things like "mock" that are required for our unit
-# tests.
-extras = test
+
+extras =
+       # Get general testing environment dependencies so we can run the tests
+       # how we like.
+       testenv
+
+       # And get all of the test suite's actual direct Python dependencies.
+       test
 
 setenv =
        # Define TEST_SUITE in the environment as an aid to constructing the

From f48eb81d9d741857b6fe6cbabab0e04942238036 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 07:57:51 -0400
Subject: [PATCH 238/362] restrict werkzeug more, at least for the moment

---
 setup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 599eb5898..d823029e2 100644
--- a/setup.py
+++ b/setup.py
@@ -141,8 +141,10 @@ install_requires = [
 
     # HTTP server and client
     "klein",
+
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    "werkzeug != 2.2.0",
+    # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
+    "werkzeug != 2.2.0, != 2.3.0, != 2.3.1",
     "treq",
     "cbor2",
 

From 44cd746ce480c3a1641ebfe55350d15698625323 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 28 Apr 2023 11:43:26 -0400
Subject: [PATCH 239/362] Limit klein version for now.

---
 setup.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 2418c6dbe..751c7ebc3 100644
--- a/setup.py
+++ b/setup.py
@@ -141,8 +141,10 @@ install_requires = [
 
     # HTTP server and client
     "klein",
-    # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    "werkzeug != 2.2.0",
+    # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465 and 2.3 is
+    # incompatible with klein 21.8 and earlier; see
+    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4020 for the latter.
+    "werkzeug != 2.2.0,<2.3",
     "treq",
     "cbor2",
 

From c15dd6c9f0fb9d43c9a17db523d6f726f67ee593 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 28 Apr 2023 11:43:48 -0400
Subject: [PATCH 240/362] This wasn't the issue.

---
 integration/util.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/util.py b/integration/util.py
index c58fc2e93..b1692a7a3 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -582,7 +582,7 @@ def web_post(tahoe, uri_fragment, **kwargs):
 
 
 @run_in_thread
-def await_client_ready(tahoe, timeout=30, liveness=60*2, minimum_number_of_servers=1):
+def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_servers=1):
     """
     Uses the status API to wait for a client-type node (in `tahoe`, a
     `TahoeProcess` instance usually from a fixture e.g. `alice`) to be

From f0b98aead562495f7f2019dd886b4e61c5fe68f9 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:32:42 -0400
Subject: [PATCH 241/362] You don't need tox *inside* your test environment.

You need tox to *manage* your test environment (this is the premise, at least).
---
 .circleci/setup-virtualenv.sh | 4 ++++
 setup.py                      | 1 -
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 7087c5120..7fc6dc528 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -28,6 +28,10 @@ shift || :
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 export PIP_NO_INDEX="1"
 
+# Get tox inside the bootstrap virtualenv since we use tox to manage the rest
+# of the environment.
+"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0"
+
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \
      -c "${PROJECT_ROOT}"/tox.ini \
diff --git a/setup.py b/setup.py
index d823029e2..bac93a4bb 100644
--- a/setup.py
+++ b/setup.py
@@ -416,7 +416,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "pip==22.0.3",
               "wheel==0.37.1",
               "setuptools==60.9.1",
-              "tox~=3.0",
               "subunitreporter==22.2.0",
               "python-subunit==1.4.2",
               "junitxml==0.7",

From d67016d1b99ed1750c01af0caa4c137a768f31ce Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:39:49 -0400
Subject: [PATCH 242/362] Get the right version of tox in the wheelhouse

---
 .circleci/populate-wheelhouse.sh | 3 ++-
 .circleci/setup-virtualenv.sh    | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 239c8367b..f7ce361a8 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -30,4 +30,5 @@ LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
-    "${PROJECT_ROOT}"[test]
+    "${PROJECT_ROOT}"[test] \
+    "tox~=3.0"
diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 7fc6dc528..3f0074da3 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -30,7 +30,7 @@ export PIP_NO_INDEX="1"
 
 # Get tox inside the bootstrap virtualenv since we use tox to manage the rest
 # of the environment.
-"${BOOTSTRAP_VENV}"/bin/pip install "tox~=3.0"
+"${BOOTSTRAP_VENV}"/bin/pip install tox
 
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \

From a088b1d8125404b53f76cd7408d29df08cb9ab2a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:49:14 -0400
Subject: [PATCH 243/362] don't bother to make a wheel of tox, just install it

---
 .circleci/populate-wheelhouse.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index f7ce361a8..14d421652 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -30,5 +30,8 @@ LANG="en_US.UTF-8" "${PIP}" \
     wheel \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
-    "${PROJECT_ROOT}"[test] \
-    "tox~=3.0"
+    "${PROJECT_ROOT}"[test]
+
+# Put tox right into the bootstrap environment because everyone is going to
+# need to use it.
+"${PIP}" install "tox~=3.0"

From 29c0ca59748507f95035de4aff1faaa65995102b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 13:51:22 -0400
Subject: [PATCH 244/362] put the tox installation near other software
 installation

---
 .circleci/create-virtualenv.sh   | 4 ++++
 .circleci/populate-wheelhouse.sh | 4 ----
 .circleci/setup-virtualenv.sh    | 4 ----
 3 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/.circleci/create-virtualenv.sh b/.circleci/create-virtualenv.sh
index 810ce5ae2..7327d0859 100755
--- a/.circleci/create-virtualenv.sh
+++ b/.circleci/create-virtualenv.sh
@@ -47,3 +47,7 @@ export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 # above, it may still not be able to get us a compatible version unless we
 # explicitly ask for one.
 "${PIP}" install --upgrade setuptools==44.0.0 wheel
+
+# Just about every user of this image wants to use tox from the bootstrap
+# virtualenv so go ahead and install it now.
+"${PIP}" install "tox~=3.0"
diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh
index 14d421652..239c8367b 100755
--- a/.circleci/populate-wheelhouse.sh
+++ b/.circleci/populate-wheelhouse.sh
@@ -31,7 +31,3 @@ LANG="en_US.UTF-8" "${PIP}" \
     --wheel-dir "${WHEELHOUSE_PATH}" \
     "${PROJECT_ROOT}"[testenv] \
     "${PROJECT_ROOT}"[test]
-
-# Put tox right into the bootstrap environment because everyone is going to
-# need to use it.
-"${PIP}" install "tox~=3.0"
diff --git a/.circleci/setup-virtualenv.sh b/.circleci/setup-virtualenv.sh
index 3f0074da3..7087c5120 100755
--- a/.circleci/setup-virtualenv.sh
+++ b/.circleci/setup-virtualenv.sh
@@ -28,10 +28,6 @@ shift || :
 export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}"
 export PIP_NO_INDEX="1"
 
-# Get tox inside the bootstrap virtualenv since we use tox to manage the rest
-# of the environment.
-"${BOOTSTRAP_VENV}"/bin/pip install tox
-
 # Get everything else installed in it, too.
 "${BOOTSTRAP_VENV}"/bin/tox \
      -c "${PROJECT_ROOT}"/tox.ini \

From 04ef5a02b2a27e7dc224b11f51615369d99f7e74 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 14:12:01 -0400
Subject: [PATCH 245/362] eh ... these things moved into the tox-managed venv

not intentional but not sure what a _good_ fix is, so try this.
---
 .circleci/run-tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index 6d7a881fe..b1e45af9b 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -93,5 +93,5 @@ if [ -n "${ARTIFACTS}" ]; then
 
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+    "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

From fa034781b46f2d3ae5351a9c57c7d55825fdebfe Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 28 Apr 2023 14:29:21 -0400
Subject: [PATCH 246/362] Perhaps this is the correct way to locate the
 tox-managed venv

---
 .circleci/run-tests.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh
index b1e45af9b..d897cc729 100755
--- a/.circleci/run-tests.sh
+++ b/.circleci/run-tests.sh
@@ -79,9 +79,10 @@ else
     alternative="false"
 fi
 
+WORKDIR=/tmp/tahoe-lafs.tox
 ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \
     -c ${PROJECT_ROOT}/tox.ini \
-    --workdir /tmp/tahoe-lafs.tox \
+    --workdir "${WORKDIR}" \
     -e "${TAHOE_LAFS_TOX_ENVIRONMENT}" \
     ${TAHOE_LAFS_TOX_ARGS} || "${alternative}"
 
@@ -93,5 +94,6 @@ if [ -n "${ARTIFACTS}" ]; then
 
     # Create a junitxml results area.
     mkdir -p "$(dirname "${JUNITXML}")"
-    "${BOOTSTRAP_VENV}"/.tox/"${TAHOE_LAFS_TOX_ENVIRONMENT}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
+
+    "${WORKDIR}/${TAHOE_LAFS_TOX_ENVIRONMENT}/bin/subunit2junitxml" < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}"
 fi

From 3c660aff5d23849cdc6cdd788a455a14f2bb7881 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 1 May 2023 09:19:01 -0400
Subject: [PATCH 247/362] a comment about the other test extra

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index bac93a4bb..49004b1ff 100644
--- a/setup.py
+++ b/setup.py
@@ -422,6 +422,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "coverage ~= 5.0",
           ],
 
+          # Here are the library dependencies of the test suite.
           "test": [
               "mock",
               "pytest",

From 0af84c9ac1fdf07cb644d381b693dfcafcdf594e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 1 May 2023 09:28:46 -0400
Subject: [PATCH 248/362] news fragment

---
 newsfragments/4020.installation | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/4020.installation

diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation
new file mode 100644
index 000000000..8badf4b3c
--- /dev/null
+++ b/newsfragments/4020.installation
@@ -0,0 +1 @@
+werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein.

From 5f196050753b70c9224f1a10427d13649553b66a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:41:51 -0400
Subject: [PATCH 249/362] During testing, ensure we're not getting text/html
 unexpectedly.

---
 src/allmydata/storage/http_client.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index f786b8f30..7314adf38 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -443,11 +443,20 @@ class StorageClient(object):
             kwargs["data"] = dumps(message_to_serialize)
             headers.addRawHeader("Content-Type", CBOR_MIME_TYPE)
 
-        return await self._treq.request(
+        response = await self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 
-    async def decode_cbor(self, response, schema: Schema) -> object:
+        if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
+            if response.code != 404:
+                # We're doing API queries, HTML is never correct except in 404, but
+                # it's the default for Twisted's web server so make sure nothing
+                # unexpected happened.
+                assert get_content_type(response.headers) != "text/html"
+
+        return response
+
+    async def decode_cbor(self, response: IResponse, schema: Schema) -> object:
         """Given HTTP response, return decoded CBOR body."""
         with start_action(action_type="allmydata:storage:http-client:decode-cbor"):
             if response.code > 199 and response.code < 300:

From fbd6dbda47fec79f016ff5ba4609f1b03203a248 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:42:02 -0400
Subject: [PATCH 250/362] text/html is a bad default content type.

---
 src/allmydata/storage/http_server.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 8647274f8..e0040d377 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -106,6 +106,9 @@ def _authorization_decorator(required_secrets):
     def decorator(f):
         @wraps(f)
         def route(self, request, *args, **kwargs):
+            # Don't set text/html content type by default:
+            request.defaultContentType = None
+
             with start_action(
                 action_type="allmydata:storage:http-server:handle-request",
                 method=request.method,

From 2292d64fcddc5d585551a0310a6b3076eb68caf3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:49:09 -0400
Subject: [PATCH 251/362] Set a better content type for data downloads.

---
 src/allmydata/storage/http_client.py | 6 ++++++
 src/allmydata/storage/http_server.py | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 7314adf38..64962e7b6 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -596,6 +596,12 @@ def read_share_chunk(
     if response.code == http.NO_CONTENT:
         return b""
 
+    content_type = get_content_type(response.headers)
+    if content_type != "application/octet-stream":
+        raise ValueError(
+            f"Content-type was wrong: {content_type}, should be application/octet-stream"
+        )
+
     if response.code == http.PARTIAL_CONTENT:
         content_range = parse_content_range_header(
             response.headers.getRawHeaders("content-range")[0] or ""
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index e0040d377..0791c3389 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -778,6 +778,7 @@ class HTTPServer(object):
     )
     def read_share_chunk(self, request, authorization, storage_index, share_number):
         """Read a chunk for an already uploaded immutable."""
+        request.setHeader("content-type", "application/octet-stream")
         try:
             bucket = self._storage_server.get_buckets(storage_index)[share_number]
         except KeyError:
@@ -883,7 +884,8 @@ class HTTPServer(object):
     )
     def read_mutable_chunk(self, request, authorization, storage_index, share_number):
         """Read a chunk from a mutable."""
-
+        request.setHeader("content-type", "application/octet-stream")
+        
         try:
             share_length = self._storage_server.get_mutable_share_length(
                 storage_index, share_number

From 5632e82e1338541d3c8f574070f402fabdc3523c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:49:29 -0400
Subject: [PATCH 252/362] News fragment.

---
 newsfragments/4016.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4016.minor

diff --git a/newsfragments/4016.minor b/newsfragments/4016.minor
new file mode 100644
index 000000000..e69de29bb

From 8c8e24a3b9c7655c197d24ece14e559511727610 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:50:05 -0400
Subject: [PATCH 253/362] Black reformat.

---
 src/allmydata/storage/http_server.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 0791c3389..7d7398b1e 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -117,9 +117,9 @@ def _authorization_decorator(required_secrets):
                 try:
                     # Check Authorization header:
                     if not timing_safe_compare(
-                        request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
-                            "utf-8"
-                        ),
+                        request.requestHeaders.getRawHeaders("Authorization", [""])[
+                            0
+                        ].encode("utf-8"),
                         swissnum_auth_header(self._swissnum),
                     ):
                         raise _HTTPError(http.UNAUTHORIZED)
@@ -494,6 +494,7 @@ def read_range(
 
 def _add_error_handling(app: Klein):
     """Add exception handlers to a Klein app."""
+
     @app.handle_errors(_HTTPError)
     def _http_error(_, request, failure):
         """Handle ``_HTTPError`` exceptions."""
@@ -885,7 +886,7 @@ class HTTPServer(object):
     def read_mutable_chunk(self, request, authorization, storage_index, share_number):
         """Read a chunk from a mutable."""
         request.setHeader("content-type", "application/octet-stream")
-        
+
         try:
             share_length = self._storage_server.get_mutable_share_length(
                 storage_index, share_number

From b21b15f3954ae328fbaafd7d53839b62c4329d4b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:56:59 -0400
Subject: [PATCH 254/362] Blocking newer werkzeug is a temporary measure.

---
 newsfragments/4020.installation | 1 -
 newsfragments/4020.minor        | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 newsfragments/4020.installation
 create mode 100644 newsfragments/4020.minor

diff --git a/newsfragments/4020.installation b/newsfragments/4020.installation
deleted file mode 100644
index 8badf4b3c..000000000
--- a/newsfragments/4020.installation
+++ /dev/null
@@ -1 +0,0 @@
-werkzeug 2.3.0 and werkzeug 2.3.1 are now blacklisted by the package metadata due to incompatibilities with klein.
diff --git a/newsfragments/4020.minor b/newsfragments/4020.minor
new file mode 100644
index 000000000..8b1378917
--- /dev/null
+++ b/newsfragments/4020.minor
@@ -0,0 +1 @@
+

From 4ca056b51c3dcc9f13f424bf133bd9dc34de8d93 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 1 May 2023 11:57:35 -0400
Subject: [PATCH 255/362] Be more general, 2.3.2 just came out for example.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 49004b1ff..4f28b4438 100644
--- a/setup.py
+++ b/setup.py
@@ -144,7 +144,7 @@ install_requires = [
 
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
     # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
-    "werkzeug != 2.2.0, != 2.3.0, != 2.3.1",
+    "werkzeug != 2.2.0, < 2.3",
     "treq",
     "cbor2",
 

From 5c2f18dfec743c104410ef514120a8b97ccc1364 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:03:14 -0400
Subject: [PATCH 256/362] Set a higher timeout.

---
 src/allmydata/test/test_system.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index d11a6e866..72e91f9b4 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -1749,6 +1749,10 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
         return d
 
+    # In CI this test can be very slow, so give it a longer timeout:
+    test_filesystem.timeout = 360
+
+
     def test_filesystem_with_cli_in_subprocess(self):
         # We do this in a separate test so that test_filesystem doesn't skip if we can't run bin/tahoe.
 

From 8fa89bd98585bd64b2510a6ec50ab44a5090bd4b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:05:40 -0400
Subject: [PATCH 257/362] Run a little faster.

---
 src/allmydata/test/test_system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 72e91f9b4..a6bed7f87 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
     def test_filesystem(self):
         self.data = LARGE_DATA
-        d = self.set_up_nodes()
+        d = self.set_up_nodes(4)
         def _new_happy_semantics(ign):
             for c in self.clients:
                 c.encoding_params['happy'] = 1

From d4f2038fd1ae943944c9a90e3e154371bc76d57d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:11:23 -0400
Subject: [PATCH 258/362] Rearrange nodes so it's possible to create even
 fewer.

---
 src/allmydata/test/common_system.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py
index 3491d413d..fa8d943e5 100644
--- a/src/allmydata/test/common_system.py
+++ b/src/allmydata/test/common_system.py
@@ -819,8 +819,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
             helper_furl = f.read()
 
         self.helper_furl = helper_furl
-        if self.numclients >= 4:
-            with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f:
+        if self.numclients >= 2:
+            with open(os.path.join(basedirs[1], 'tahoe.cfg'), 'a+') as f:
                 f.write(
                     "[client]\n"
                     "helper.furl = {}\n".format(helper_furl)
@@ -836,9 +836,9 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         log.msg("CONNECTED")
         # now find out where the web port was
         self.webish_url = self.clients[0].getServiceNamed("webish").getURL()
-        if self.numclients >=4:
+        if self.numclients >=2:
             # and the helper-using webport
-            self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL()
+            self.helper_webish_url = self.clients[1].getServiceNamed("webish").getURL()
 
     def _generate_config(self, which, basedir, force_foolscap=False):
         config = {}
@@ -854,10 +854,10 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
             ("node", "tub.location"): allclients,
 
             # client 0 runs a webserver and a helper
-            # client 3 runs a webserver but no helper
-            ("node", "web.port"): {0, 3},
+            # client 1 runs a webserver but no helper
+            ("node", "web.port"): {0, 1},
             ("node", "timeout.keepalive"): {0},
-            ("node", "timeout.disconnect"): {3},
+            ("node", "timeout.disconnect"): {1},
 
             ("helper", "enabled"): {0},
         }

From 9f78fd5c7f2d3340a3c64f79483373292570b732 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:11:31 -0400
Subject: [PATCH 259/362] Use even fewer nodes.

---
 src/allmydata/test/test_system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index a6bed7f87..c2fe1339f 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -787,7 +787,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
     def test_filesystem(self):
         self.data = LARGE_DATA
-        d = self.set_up_nodes(4)
+        d = self.set_up_nodes(2)
         def _new_happy_semantics(ign):
             for c in self.clients:
                 c.encoding_params['happy'] = 1

From 1ca30e1d2fc00c61b56ad6603984dd4d705bd549 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:11:44 -0400
Subject: [PATCH 260/362] News entry.

---
 newsfragments/4022.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4022.minor

diff --git a/newsfragments/4022.minor b/newsfragments/4022.minor
new file mode 100644
index 000000000..e69de29bb

From 22715abc854b68eaa38fd664d1ab207c88582894 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:17:55 -0400
Subject: [PATCH 261/362] This is fine.

---
 src/allmydata/test/test_system.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index c2fe1339f..58384e4d8 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -1750,7 +1750,7 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
         return d
 
     # In CI this test can be very slow, so give it a longer timeout:
-    test_filesystem.timeout = 360
+    test_filesystem.timeout = 360  # type: ignore[attr-defined]
 
 
     def test_filesystem_with_cli_in_subprocess(self):

From 63b082759dabe44dfbed45effd10e2cae26d037b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 2 May 2023 12:59:00 -0400
Subject: [PATCH 262/362] Use a modern coverage.py.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index d01efdf83..9837b3bab 100644
--- a/setup.py
+++ b/setup.py
@@ -418,7 +418,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "subunitreporter==22.2.0",
               "python-subunit==1.4.2",
               "junitxml==0.7",
-              "coverage ~= 5.0",
+              "coverage==7.2.5",
           ],
 
           # Here are the library dependencies of the test suite.

From 5a5031f02046c77a66761b5988177081631a6afc Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 3 May 2023 16:46:57 -0400
Subject: [PATCH 263/362] Try with newer Python, 3.11 might make it faster.

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1bb7c9efb..fe911e34d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -169,12 +169,12 @@ jobs:
             python-version: "3.9"
             force-foolscap: false
           - os: windows-latest
-            python-version: "3.9"
+            python-version: "3.11"
             force-foolscap: false
           # 22.04 has some issue with Tor at the moment:
           # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
           - os: ubuntu-20.04
-            python-version: "3.11"
+            python-version: "3.10"
             force-foolscap: false
     steps:
 

From dca19525b9d9f2cd6c6f63fdaba6945bb5f4759a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 3 May 2023 16:58:47 -0400
Subject: [PATCH 264/362] =?UTF-8?q?=F0=9F=AA=84?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 7360b891b..bf04e4424 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -50,9 +50,9 @@ from .util import (
 )
 from allmydata.node import read_config
 
-# No reason for HTTP requests to take longer than two minutes in the
+# No reason for HTTP requests to take longer than four minutes in the
 # integration tests. See allmydata/scripts/common_http.py for usage.
-os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "120"
+os.environ["__TAHOE_CLI_HTTP_TIMEOUT"] = "240"
 
 # Make Foolscap logging go into Twisted logging, so that integration test logs
 # include extra information

From 83a6a7de2835148baaa6aaa94a449c77389887a3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 3 May 2023 17:20:29 -0400
Subject: [PATCH 265/362] Newer klein and werkzeug.

---
 setup.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/setup.py b/setup.py
index 9837b3bab..0453fa63f 100644
--- a/setup.py
+++ b/setup.py
@@ -140,10 +140,10 @@ install_requires = [
     "collections-extended >= 2.0.2",
 
     # HTTP server and client
-    "klein",
+    # Latest version is necessary to work with latest werkzeug:
+    "klein >= 23.5.0",
     # 2.2.0 has a bug: https://github.com/pallets/werkzeug/issues/2465
-    # 2.3.x has an incompatibility with Klein: https://github.com/twisted/klein/pull/575
-    "werkzeug != 2.2.0, < 2.3",
+    "werkzeug != 2.2.0",
     "treq",
     "cbor2",
 

From c70930b47921facf49d5cc371bd575bd715f7669 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 3 May 2023 17:21:07 -0400
Subject: [PATCH 266/362] News fragment.

---
 newsfragments/4024.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4024.minor

diff --git a/newsfragments/4024.minor b/newsfragments/4024.minor
new file mode 100644
index 000000000..e69de29bb

From 19690c9c7bd319bf9a2d1931ef4c8b37e9cc3803 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 13:05:06 -0400
Subject: [PATCH 267/362] Don't mix blocking and async APIs!

---
 integration/test_web.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/integration/test_web.py b/integration/test_web.py
index b3c4a8e5f..c7d2275ae 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -14,6 +14,8 @@ from __future__ import annotations
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote
 
+from twisted.internet.defer import maybeDeferred
+
 import allmydata.uri
 from allmydata.util import jsonbytes as json
 
@@ -24,7 +26,7 @@ import requests
 import html5lib
 from bs4 import BeautifulSoup
 
-from pytest_twisted import ensureDeferred
+import pytest_twisted
 
 @run_in_thread
 def test_index(alice):
@@ -185,7 +187,7 @@ def test_deep_stats(alice):
         time.sleep(.5)
 
 
-@util.run_in_thread
+@run_in_thread
 def test_status(alice):
     """
     confirm we get something sensible from /status and the various sub-types
@@ -251,7 +253,7 @@ def test_status(alice):
     assert found_download, "Failed to find the file we downloaded in the status-page"
 
 
-@ensureDeferred
+@run_in_thread
 async def test_directory_deep_check(reactor, request, alice):
     """
     use deep-check and confirm the result pages work
@@ -262,7 +264,8 @@ async def test_directory_deep_check(reactor, request, alice):
     required = 2
     total = 4
 
-    await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
+    result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
+    pytest_twisted.blockon(maybeDeferred(result))
 
     # create a directory
     resp = requests.post(

From f54a2d3d76a0508c713c42c420105f98e384a5df Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 13:05:46 -0400
Subject: [PATCH 268/362] News file.

---
 newsfragments/4023.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4023.minor

diff --git a/newsfragments/4023.minor b/newsfragments/4023.minor
new file mode 100644
index 000000000..e69de29bb

From 3d6b3b3b74f2fc4878629e9838471f3d12a1eb6b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 13:10:23 -0400
Subject: [PATCH 269/362] Use modern Docker image.

---
 .readthedocs.yaml        | 5 +++++
 newsfragments/4026.minor | 0
 2 files changed, 5 insertions(+)
 create mode 100644 newsfragments/4026.minor

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
index 65b390f26..665b53178 100644
--- a/.readthedocs.yaml
+++ b/.readthedocs.yaml
@@ -1,5 +1,10 @@
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
 python:
    install:
    - requirements: docs/requirements.txt
diff --git a/newsfragments/4026.minor b/newsfragments/4026.minor
new file mode 100644
index 000000000..e69de29bb

From bee295e4119448958f1ea2b07666345328eaaee9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 13:33:54 -0400
Subject: [PATCH 270/362] Actually run the test.

---
 integration/test_web.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/integration/test_web.py b/integration/test_web.py
index c7d2275ae..1d9498264 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -14,7 +14,7 @@ from __future__ import annotations
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote
 
-from twisted.internet.defer import maybeDeferred
+from twisted.internet.defer import ensureDeferred
 
 import allmydata.uri
 from allmydata.util import jsonbytes as json
@@ -254,7 +254,7 @@ def test_status(alice):
 
 
 @run_in_thread
-async def test_directory_deep_check(reactor, request, alice):
+def test_directory_deep_check(reactor, request, alice):
     """
     use deep-check and confirm the result pages work
     """
@@ -265,7 +265,7 @@ async def test_directory_deep_check(reactor, request, alice):
     total = 4
 
     result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
-    pytest_twisted.blockon(maybeDeferred(result))
+    pytest_twisted.blockon(ensureDeferred(result))
 
     # create a directory
     resp = requests.post(

From 3b52457d1c42e4de5bffa7486738a837d81ceb56 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 13:54:04 -0400
Subject: [PATCH 271/362] Try a different way.

---
 integration/test_web.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/integration/test_web.py b/integration/test_web.py
index 1d9498264..fd29504f8 100644
--- a/integration/test_web.py
+++ b/integration/test_web.py
@@ -14,7 +14,7 @@ from __future__ import annotations
 import time
 from urllib.parse import unquote as url_unquote, quote as url_quote
 
-from twisted.internet.defer import ensureDeferred
+from twisted.internet.threads import deferToThread
 
 import allmydata.uri
 from allmydata.util import jsonbytes as json
@@ -253,8 +253,8 @@ def test_status(alice):
     assert found_download, "Failed to find the file we downloaded in the status-page"
 
 
-@run_in_thread
-def test_directory_deep_check(reactor, request, alice):
+@pytest_twisted.ensureDeferred
+async def test_directory_deep_check(reactor, request, alice):
     """
     use deep-check and confirm the result pages work
     """
@@ -264,9 +264,11 @@ def test_directory_deep_check(reactor, request, alice):
     required = 2
     total = 4
 
-    result = util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
-    pytest_twisted.blockon(ensureDeferred(result))
+    await util.reconfigure(reactor, request, alice, (happy, required, total), convergence=None)
+    await deferToThread(_test_directory_deep_check_blocking, alice)
 
+
+def _test_directory_deep_check_blocking(alice):
     # create a directory
     resp = requests.post(
         util.node_url(alice.node_dir, u"uri"),

From 049502e8c282eb507d46cdfcf24a945aceaa7e33 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 16:15:30 -0400
Subject: [PATCH 272/362] Don't mix blocking and async code.

---
 integration/test_get_put.py | 33 +++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/integration/test_get_put.py b/integration/test_get_put.py
index f121d6284..0aca6954a 100644
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@@ -6,8 +6,10 @@ and stdout.
 from subprocess import Popen, PIPE, check_output, check_call
 
 import pytest
-from pytest_twisted import ensureDeferred
+from pytest_twisted import blockon
 from twisted.internet import reactor
+from twisted.internet.threads import blockingCallFromThread
+from twisted.internet.defer import Deferred
 
 from .util import run_in_thread, cli, reconfigure
 
@@ -86,8 +88,8 @@ def test_large_file(alice, get_put_alias, tmp_path):
     assert outfile.read_bytes() == tempfile.read_bytes()
 
 
-@ensureDeferred
-async def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
+@run_in_thread
+def test_upload_download_immutable_different_default_max_segment_size(alice, get_put_alias, tmpdir, request):
     """
     Tahoe-LAFS used to have a default max segment size of 128KB, and is now
     1MB.  Test that an upload created when 128KB was the default can be
@@ -100,22 +102,25 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
     with tempfile.open("wb") as f:
         f.write(large_data)
 
-    async def set_segment_size(segment_size):
-        await reconfigure(
+    def set_segment_size(segment_size):
+        return blockingCallFromThread(
             reactor,
-            request,
-            alice,
-            (1, 1, 1),
-            None,
-            max_segment_size=segment_size
-    )
+            lambda: Deferred.fromCoroutine(reconfigure(
+                reactor,
+                request,
+                alice,
+                (1, 1, 1),
+                None,
+                max_segment_size=segment_size
+            ))
+        )
 
     # 1. Upload file 1 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
     cli(alice, "put", str(tempfile), "getput:seg1024kb")
 
     # 2. Download file 1 with default segment size set to 128KB
-    await set_segment_size(128 * 1024)
+    set_segment_size(128 * 1024)
     assert large_data == check_output(
         ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg1024kb", "-"]
     )
@@ -124,7 +129,7 @@ async def test_upload_download_immutable_different_default_max_segment_size(alic
     cli(alice, "put", str(tempfile), "getput:seg128kb")
 
     # 4. Download file 2 with default segment size set to 1MB
-    await set_segment_size(1024 * 1024)
+    set_segment_size(1024 * 1024)
     assert large_data == check_output(
         ["tahoe", "--node-directory", alice.node_dir, "get", "getput:seg128kb", "-"]
     )

From ba638f9ff602b3f7dcf8290f58adbab653ba25bd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 4 May 2023 17:08:54 -0400
Subject: [PATCH 273/362] Remove unnecessary import.

---
 integration/test_get_put.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/test_get_put.py b/integration/test_get_put.py
index 0aca6954a..e30a34f97 100644
--- a/integration/test_get_put.py
+++ b/integration/test_get_put.py
@@ -6,7 +6,6 @@ and stdout.
 from subprocess import Popen, PIPE, check_output, check_call
 
 import pytest
-from pytest_twisted import blockon
 from twisted.internet import reactor
 from twisted.internet.threads import blockingCallFromThread
 from twisted.internet.defer import Deferred

From 20f55933bdf4742da8b0e68f9cd14fa0522a6323 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 5 May 2023 11:02:24 -0400
Subject: [PATCH 274/362] Update past deprecated runner.

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fe911e34d..77221e552 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -248,7 +248,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - macos-10.15
+          - macos-latest
           - windows-latest
           - ubuntu-latest
         python-version:

From 2e22df60fe6b29706464d3d3fa4f8b2e7baf51d5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 8 May 2023 13:33:34 -0400
Subject: [PATCH 275/362] Try with fewer persistent HTTP connections.

---
 src/allmydata/storage/http_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 64962e7b6..0e12df7ce 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -325,7 +325,7 @@ class StorageClient(object):
         certificate_hash = nurl.user.encode("ascii")
         if pool is None:
             pool = HTTPConnectionPool(reactor)
-            pool.maxPersistentPerHost = 20
+            pool.maxPersistentPerHost = 10
 
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
             cls.TEST_MODE_REGISTER_HTTP_POOL(pool)

From 2bd76058e927c7811b722538abb8127fadde45be Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 8 May 2023 13:48:54 -0400
Subject: [PATCH 276/362] Specific version of macOS.

---
 .github/workflows/ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 77221e552..dc9854ae4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -53,9 +53,9 @@ jobs:
           - "3.11"
         include:
           # On macOS don't bother with 3.8, just to get faster builds.
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.9"
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.11"
           # We only support PyPy on Linux at the moment.
           - os: ubuntu-latest
@@ -165,7 +165,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos-latest
+          - os: macos-12
             python-version: "3.9"
             force-foolscap: false
           - os: windows-latest
@@ -248,7 +248,7 @@ jobs:
       fail-fast: false
       matrix:
         os:
-          - macos-latest
+          - macos-12
           - windows-latest
           - ubuntu-latest
         python-version:

From fea2450c604a3d34a3d78760d1cbc3085bdd521b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 9 May 2023 10:53:17 -0400
Subject: [PATCH 277/362] Test and fix for really bad authorization header.

---
 src/allmydata/storage/http_server.py    | 10 +++++++---
 src/allmydata/test/test_storage_http.py | 19 +++++++++++++++++++
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 7d7398b1e..cc336d1c7 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -116,10 +116,14 @@ def _authorization_decorator(required_secrets):
             ) as ctx:
                 try:
                     # Check Authorization header:
+                    try:
+                        auth_header = request.requestHeaders.getRawHeaders(
+                            "Authorization", [""]
+                        )[0].encode("utf-8")
+                    except UnicodeError:
+                        raise _HTTPError(http.BAD_REQUEST)
                     if not timing_safe_compare(
-                        request.requestHeaders.getRawHeaders("Authorization", [""])[
-                            0
-                        ].encode("utf-8"),
+                        auth_header,
                         swissnum_auth_header(self._swissnum),
                     ):
                         raise _HTTPError(http.UNAUTHORIZED)
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index eca2be1c1..77fb825e6 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -257,6 +257,10 @@ class TestApp(object):
     _add_error_handling(_app)
     _swissnum = SWISSNUM_FOR_TEST  # Match what the test client is using
 
+    @_authorized_route(_app, {}, "/noop", methods=["GET"])
+    def noop(self, request, authorization):
+        return "noop"
+
     @_authorized_route(_app, {Secrets.UPLOAD}, "/upload_secret", methods=["GET"])
     def validate_upload_secret(self, request, authorization):
         if authorization == {Secrets.UPLOAD: b"MAGIC"}:
@@ -339,6 +343,21 @@ class CustomHTTPServerTests(SyncTestCase):
         )
         self._http_server.clock = self.client._clock
 
+    def test_bad_swissnum_in_client(self) -> None:
+        """
+        If the swissnum is invalid, a BAD REQUEST response code is returned.
+        """
+        headers = Headers()
+        headers.addRawHeader("Authorization", b"\x00\xFF\x00\xFF")
+        response = result_of(
+            self.client._treq.request(
+                "GET",
+                DecodedURL.from_text("http://127.0.0.1/noop"),
+                headers=headers,
+            )
+        )
+        self.assertEqual(response.code, 400)
+
     def test_authorization_enforcement(self):
         """
         The requirement for secrets is enforced by the ``_authorized_route``

From 40b930c02c1dd20bd567419ef58b4fa30b6dfe8b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 9 May 2023 16:47:28 -0400
Subject: [PATCH 278/362] Another test.

---
 src/allmydata/test/test_storage_http.py | 30 +++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 77fb825e6..df5dc300c 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -343,11 +343,12 @@ class CustomHTTPServerTests(SyncTestCase):
         )
         self._http_server.clock = self.client._clock
 
-    def test_bad_swissnum_in_client(self) -> None:
+    def test_bad_swissnum_from_client(self) -> None:
         """
         If the swissnum is invalid, a BAD REQUEST response code is returned.
         """
         headers = Headers()
+        # The value is not UTF-8.
         headers.addRawHeader("Authorization", b"\x00\xFF\x00\xFF")
         response = result_of(
             self.client._treq.request(
@@ -358,10 +359,35 @@ class CustomHTTPServerTests(SyncTestCase):
         )
         self.assertEqual(response.code, 400)
 
+    def test_bad_secret(self) -> None:
+        """
+        If the secret is invalid (not base64), a BAD REQUEST
+        response code is returned.
+        """
+        bad_secret = b"upload-secret []<>"
+        headers = Headers()
+        headers.addRawHeader(
+            "X-Tahoe-Authorization",
+            bad_secret,
+        )
+        response = result_of(
+            self.client.request(
+                "GET",
+                DecodedURL.from_text("http://127.0.0.1/upload_secret"),
+                headers=headers,
+            )
+        )
+        self.assertEqual(response.code, 400)
+
+    # TODO test other garbage values
+
     def test_authorization_enforcement(self):
         """
         The requirement for secrets is enforced by the ``_authorized_route``
         decorator; if they are not given, a 400 response code is returned.
+
+        Note that this refers to ``X-Tahoe-Authorization``, not the
+        ``Authorization`` header used for the swissnum.
         """
         # Without secret, get a 400 error.
         response = result_of(
@@ -1474,7 +1500,7 @@ class SharedImmutableMutableTestsMixin:
             self.client.advise_corrupt_share(storage_index, 13, reason)
         )
 
-        for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]:
+        for si, share_number in [(storage_index, 11), (urandom(16), 13)]:
             with assert_fails_with_http_code(self, http.NOT_FOUND):
                 self.http.result_of_with_flush(
                     self.client.advise_corrupt_share(si, share_number, reason)

From 1c9de671049fe133692ed78a0bc35d04b7b3974c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 9 May 2023 16:47:32 -0400
Subject: [PATCH 279/362] Nicer error messages, useful for debugging.

---
 src/allmydata/storage/http_server.py | 31 ++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index cc336d1c7..924ae5a43 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -4,7 +4,7 @@ HTTP server for storage.
 
 from __future__ import annotations
 
-from typing import Any, Callable, Union, cast
+from typing import Any, Callable, Union, cast, Optional
 from functools import wraps
 from base64 import b64decode
 import binascii
@@ -75,7 +75,7 @@ def _extract_secrets(
     secrets, return dictionary mapping secrets to decoded values.
 
     If too few secrets were given, or too many, a ``ClientSecretsException`` is
-    raised.
+    raised; its text is sent in the HTTP response.
     """
     string_key_to_enum = {e.value: e for e in Secrets}
     result = {}
@@ -84,6 +84,10 @@ def _extract_secrets(
             string_key, string_value = header_value.strip().split(" ", 1)
             key = string_key_to_enum[string_key]
             value = b64decode(string_value)
+            if value == b"":
+                raise ClientSecretsException(
+                    "Failed to decode secret {}".format(string_key)
+                )
             if key in (Secrets.LEASE_CANCEL, Secrets.LEASE_RENEW) and len(value) != 32:
                 raise ClientSecretsException("Lease secrets must be 32 bytes long")
             result[key] = value
@@ -91,7 +95,9 @@ def _extract_secrets(
         raise ClientSecretsException("Bad header value(s): {}".format(header_values))
     if result.keys() != required_secrets:
         raise ClientSecretsException(
-            "Expected {} secrets, got {}".format(required_secrets, result.keys())
+            "Expected {} in X-Tahoe-Authorization headers, got {}".format(
+                [r.value for r in required_secrets], list(result.keys())
+            )
         )
     return result
 
@@ -121,12 +127,14 @@ def _authorization_decorator(required_secrets):
                             "Authorization", [""]
                         )[0].encode("utf-8")
                     except UnicodeError:
-                        raise _HTTPError(http.BAD_REQUEST)
+                        raise _HTTPError(http.BAD_REQUEST, "Bad Authorization header")
                     if not timing_safe_compare(
                         auth_header,
                         swissnum_auth_header(self._swissnum),
                     ):
-                        raise _HTTPError(http.UNAUTHORIZED)
+                        raise _HTTPError(
+                            http.UNAUTHORIZED, "Wrong Authorization header"
+                        )
 
                     # Check secrets:
                     authorization = request.requestHeaders.getRawHeaders(
@@ -134,8 +142,8 @@ def _authorization_decorator(required_secrets):
                     )
                     try:
                         secrets = _extract_secrets(authorization, required_secrets)
-                    except ClientSecretsException:
-                        raise _HTTPError(http.BAD_REQUEST)
+                    except ClientSecretsException as e:
+                        raise _HTTPError(http.BAD_REQUEST, str(e))
 
                     # Run the business logic:
                     result = f(self, request, secrets, *args, **kwargs)
@@ -276,8 +284,10 @@ class _HTTPError(Exception):
     Raise from ``HTTPServer`` endpoint to return the given HTTP response code.
     """
 
-    def __init__(self, code: int):
+    def __init__(self, code: int, body: Optional[str] = None):
+        Exception.__init__(self, (code, body))
         self.code = code
+        self.body = body
 
 
 # CDDL schemas.
@@ -503,7 +513,10 @@ def _add_error_handling(app: Klein):
     def _http_error(_, request, failure):
         """Handle ``_HTTPError`` exceptions."""
         request.setResponseCode(failure.value.code)
-        return b""
+        if failure.value.body is not None:
+            return failure.value.body
+        else:
+            return b""
 
     @app.handle_errors(CDDLValidationError)
     def _cddl_validation_error(_, request, failure):

From 36bffb7f60bcf646fb940cff5ffd49caa8514742 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 9 May 2023 16:47:54 -0400
Subject: [PATCH 280/362] News file.

---
 newsfragments/4027.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4027.minor

diff --git a/newsfragments/4027.minor b/newsfragments/4027.minor
new file mode 100644
index 000000000..e69de29bb

From c92c93e6d56975c47bd8b2e0dea3a8cfba81960b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 10 May 2023 16:31:53 -0400
Subject: [PATCH 281/362] Clean up cached HTTP connections on shutdown.

---
 newsfragments/4028.minor             | 0
 src/allmydata/storage/http_client.py | 7 ++++++-
 src/allmydata/storage_client.py      | 5 +++++
 3 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 newsfragments/4028.minor

diff --git a/newsfragments/4028.minor b/newsfragments/4028.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 0e12df7ce..e2b45e30c 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -310,6 +310,7 @@ class StorageClient(object):
     _base_url: DecodedURL
     _swissnum: bytes
     _treq: Union[treq, StubTreq, HTTPClient]
+    _pool: HTTPConnectionPool
     _clock: IReactorTime
 
     @classmethod
@@ -339,7 +340,7 @@ class StorageClient(object):
         )
 
         https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
-        return cls(https_url, swissnum, treq_client, reactor)
+        return cls(https_url, swissnum, treq_client, pool, reactor)
 
     def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
@@ -479,6 +480,10 @@ class StorageClient(object):
                 ).read()
                 raise ClientException(response.code, response.phrase, data)
 
+    def shutdown(self) -> Deferred:
+        """Shutdown any connections."""
+        return self._pool.closeCachedConnections()
+
 
 @define(hash=True)
 class StorageClientGeneral(object):
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index a40e98b03..94aae43f6 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1271,6 +1271,11 @@ class HTTPNativeStorageServer(service.MultiService):
         if self._lc.running:
             self._lc.stop()
         self._failed_to_connect("shut down")
+
+        maybe_storage_server = self.get_storage_server()
+        if maybe_storage_server is not None:
+            result.addCallback(lambda _: maybe_storage_server._http_client.shutdown())
+
         return result
 
 

From ba9946e6ea863b47f5b6a544cf6b70db49dd4bf6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 10 May 2023 16:34:02 -0400
Subject: [PATCH 282/362] Fix tests.

---
 src/allmydata/storage/http_client.py    | 5 +++--
 src/allmydata/test/test_storage_http.py | 5 ++++-
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index e2b45e30c..9c4a5538c 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -310,7 +310,7 @@ class StorageClient(object):
     _base_url: DecodedURL
     _swissnum: bytes
     _treq: Union[treq, StubTreq, HTTPClient]
-    _pool: HTTPConnectionPool
+    _pool: Optional[HTTPConnectionPool]
     _clock: IReactorTime
 
     @classmethod
@@ -482,7 +482,8 @@ class StorageClient(object):
 
     def shutdown(self) -> Deferred:
         """Shutdown any connections."""
-        return self._pool.closeCachedConnections()
+        if self._pool is not None:
+            return self._pool.closeCachedConnections()
 
 
 @define(hash=True)
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index eca2be1c1..64491f7ae 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -331,6 +331,7 @@ class CustomHTTPServerTests(SyncTestCase):
             DecodedURL.from_text("http://127.0.0.1"),
             SWISSNUM_FOR_TEST,
             treq=treq,
+            pool=None,
             # We're using a Treq private API to get the reactor, alas, but only
             # in a test, so not going to worry about it too much. This would be
             # fixed if https://github.com/twisted/treq/issues/226 were ever
@@ -512,6 +513,7 @@ class HttpTestFixture(Fixture):
             DecodedURL.from_text("http://127.0.0.1"),
             SWISSNUM_FOR_TEST,
             treq=self.treq,
+            pool=None,
             clock=self.clock,
         )
 
@@ -624,6 +626,7 @@ class GenericHTTPAPITests(SyncTestCase):
                 DecodedURL.from_text("http://127.0.0.1"),
                 b"something wrong",
                 treq=StubTreq(self.http.http_server.get_resource()),
+                pool=None,
                 clock=self.http.clock,
             )
         )
@@ -1455,7 +1458,7 @@ class SharedImmutableMutableTestsMixin:
             self.client.advise_corrupt_share(storage_index, 13, reason)
         )
 
-        for (si, share_number) in [(storage_index, 11), (urandom(16), 13)]:
+        for si, share_number in [(storage_index, 11), (urandom(16), 13)]:
             with assert_fails_with_http_code(self, http.NOT_FOUND):
                 self.http.result_of_with_flush(
                     self.client.advise_corrupt_share(si, share_number, reason)

From 2ec1c1e43e0bae315897e9d6bb0d7e2df2640cb0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 10 May 2023 17:23:15 -0400
Subject: [PATCH 283/362] Shut down alice.

---
 integration/conftest.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index bf04e4424..6892b33a7 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -401,9 +401,6 @@ def alice(
             reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice",
             web_port="tcp:9980:interface=localhost",
             storage=False,
-            # We're going to kill this ourselves, so no need for finalizer to
-            # do it:
-            finalize=False,
         )
     )
     pytest_twisted.blockon(await_client_ready(process))

From f5acaea134b017a3e9a0b0fa537836b268ae06a3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 11 May 2023 09:05:58 -0400
Subject: [PATCH 284/362] bump the version of klein in the nix-based builds

---
 nix/klein.nix            | 9 +++++++++
 nix/python-overrides.nix | 6 ++++++
 2 files changed, 15 insertions(+)
 create mode 100644 nix/klein.nix

diff --git a/nix/klein.nix b/nix/klein.nix
new file mode 100644
index 000000000..be4426465
--- /dev/null
+++ b/nix/klein.nix
@@ -0,0 +1,9 @@
+{ klein, fetchPypi }:
+klein.overrideAttrs (old: rec {
+  pname = "klein";
+  version = "23.5.0";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc=";
+  };
+})
diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
index 87c42ad58..032b427ae 100644
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@@ -21,6 +21,12 @@ in {
   pycddl = self.callPackage ./pycddl.nix { };
   txi2p = self.callPackage ./txi2p.nix { };
 
+  # Update the version of klein.
+  klein = self.callPackage ./klein.nix {
+    # Avoid infinite recursion.
+    inherit (super) klein;
+  };
+
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
   # we know where a working version lives.
   collections-extended = self.callPackage ./collections-extended.nix {

From f83b73b5f31132340bb043b4dfef8088ce4403bd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 16 May 2023 10:44:34 -0400
Subject: [PATCH 285/362] Make Tor provider available at the right place to
 enable it for HTTP storage client connections.

---
 src/allmydata/client.py            |  8 ++++----
 src/allmydata/storage_client.py    | 12 +++++++++++-
 src/allmydata/util/tor_provider.py | 10 +++-------
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 1d959cb98..cb1fb9fa4 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -10,7 +10,6 @@ import weakref
 from typing import Optional, Iterable
 from base64 import urlsafe_b64encode
 from functools import partial
-# On Python 2 this will be the backported package:
 from configparser import NoSectionError
 
 from foolscap.furl import (
@@ -47,7 +46,7 @@ from allmydata.util.encodingutil import get_filesystem_encoding
 from allmydata.util.abbreviate import parse_abbreviated_size
 from allmydata.util.time_format import parse_duration, parse_date
 from allmydata.util.i2p_provider import create as create_i2p_provider
-from allmydata.util.tor_provider import create as create_tor_provider
+from allmydata.util.tor_provider import create as create_tor_provider, _Provider as TorProvider
 from allmydata.stats import StatsProvider
 from allmydata.history import History
 from allmydata.interfaces import (
@@ -268,7 +267,7 @@ def create_client_from_config(config, _client_factory=None, _introducer_factory=
     introducer_clients = create_introducer_clients(config, main_tub, _introducer_factory)
     storage_broker = create_storage_farm_broker(
         config, default_connection_handlers, foolscap_connection_handlers,
-        tub_options, introducer_clients
+        tub_options, introducer_clients, tor_provider
     )
 
     client = _client_factory(
@@ -464,7 +463,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
     return introducer_clients
 
 
-def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
+def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients, tor_provider: Optional[TorProvider]):
     """
     Create a StorageFarmBroker object, for use by Uploader/Downloader
     (and everybody else who wants to use storage servers)
@@ -500,6 +499,7 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo
         tub_maker=tub_creator,
         node_config=config,
         storage_client_config=storage_client_config,
+        tor_provider=tor_provider,
     )
     for ic in introducer_clients:
         sb.use_introducer(ic)
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index a40e98b03..af14201fb 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -77,6 +77,7 @@ from allmydata.grid_manager import (
 from allmydata.crypto import (
     ed25519,
 )
+from allmydata.util.tor_provider import _Provider as TorProvider
 from allmydata.util import log, base32, connection_status
 from allmydata.util.assertutil import precondition
 from allmydata.util.observer import ObserverList
@@ -202,6 +203,7 @@ class StorageFarmBroker(service.MultiService):
             tub_maker,
             node_config: _Config,
             storage_client_config=None,
+            tor_provider: Optional[TorProvider]=None,
     ):
         service.MultiService.__init__(self)
         assert permute_peers # False not implemented yet
@@ -223,6 +225,7 @@ class StorageFarmBroker(service.MultiService):
         self.introducer_client = None
         self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred)
         self._connected_high_water_mark = 0
+        self._tor_provider = tor_provider
 
     @log_call(action_type=u"storage-client:broker:set-static-servers")
     def set_static_servers(self, servers):
@@ -315,6 +318,7 @@ class StorageFarmBroker(service.MultiService):
                 server_id,
                 server["ann"],
                 grid_manager_verifier=gm_verifier,
+                tor_provider=tor_provider
             )
             s.on_status_changed(lambda _: self._got_connection())
             return s
@@ -1049,7 +1053,7 @@ class HTTPNativeStorageServer(service.MultiService):
     "connected".
     """
 
-    def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None):
+    def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None):
         service.MultiService.__init__(self)
         assert isinstance(server_id, bytes)
         self._server_id = server_id
@@ -1057,6 +1061,8 @@ class HTTPNativeStorageServer(service.MultiService):
         self._on_status_changed = ObserverList()
         self._reactor = reactor
         self._grid_manager_verifier = grid_manager_verifier
+        self._tor_provider = tor_provider
+
         furl = announcement["anonymous-storage-FURL"].encode("utf-8")
         (
             self._nickname,
@@ -1242,6 +1248,8 @@ class HTTPNativeStorageServer(service.MultiService):
                 pool = HTTPConnectionPool(reactor, persistent=False)
                 pool.retryAutomatically = False
                 return StorageClientGeneral(
+                    # TODO if Tor client connections are enabled, use an Agent
+                    # created via tor.
                     StorageClient.from_nurl(nurl, reactor, pool)
                 ).get_version()
 
@@ -1249,6 +1257,8 @@ class HTTPNativeStorageServer(service.MultiService):
 
             # If we've gotten this far, we've found a working NURL.
             self._istorage_server = _HTTPStorageServer.from_http_client(
+                # TODO if Tor client connections are enabled, use an Agent
+                # created via tor.
                 StorageClient.from_nurl(nurl, reactor)
             )
             return self._istorage_server
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index 4ca19c01c..57bb3a83d 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -2,14 +2,10 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import, print_function, with_statement
-from __future__ import division
-from __future__ import unicode_literals
 
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
+from typing import Optional
 import os
 
 from zope.interface import (
@@ -41,7 +37,7 @@ def _import_txtorcon():
     except ImportError: # pragma: no cover
         return None
 
-def create(reactor, config, import_tor=None, import_txtorcon=None):
+def create(reactor, config, import_tor=None, import_txtorcon=None) -> Optional[_Provider]:
     """
     Create a new _Provider service (this is an IService so must be
     hooked up to a parent or otherwise started).

From 3cf03a5c339ab6309016657ab40d7809e1ea1de1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 18 May 2023 09:28:58 -0400
Subject: [PATCH 286/362] More glue to connect Tor up to the HTTP-based storage
 client.

---
 src/allmydata/client.py              |  1 +
 src/allmydata/storage/http_client.py | 25 +++++++++++++++++++++----
 src/allmydata/storage_client.py      | 23 +++++++++++++++++++++--
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index cb1fb9fa4..e85ed4fe2 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -499,6 +499,7 @@ def create_storage_farm_broker(config: _Config, default_connection_handlers, foo
         tub_maker=tub_creator,
         node_config=config,
         storage_client_config=storage_client_config,
+        default_connection_handlers=default_connection_handlers,
         tor_provider=tor_provider,
     )
     for ic in introducer_clients:
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 549fc9719..57ac6706b 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -15,6 +15,7 @@ from typing import (
     TypedDict,
     Set,
     Dict,
+    Callable,
 )
 from base64 import b64encode
 from io import BytesIO
@@ -31,7 +32,7 @@ from collections_extended import RangeMap
 from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
-from twisted.web.iweb import IPolicyForHTTPS, IResponse
+from twisted.web.iweb import IPolicyForHTTPS, IResponse, IAgent
 from twisted.internet.defer import inlineCallbacks, Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
@@ -336,7 +337,13 @@ class StorageClient(object):
 
     @classmethod
     def from_nurl(
-        cls, nurl: DecodedURL, reactor, pool: Optional[HTTPConnectionPool] = None
+        cls,
+        nurl: DecodedURL,
+        reactor,
+        pool: Optional[HTTPConnectionPool] = None,
+        agent_factory: Optional[
+            Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent]
+        ] = None,
     ) -> StorageClient:
         """
         Create a ``StorageClient`` for the given NURL.
@@ -352,11 +359,21 @@ class StorageClient(object):
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
             cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
 
+        def default_agent_factory(
+            reactor: object,
+            tls_context_factory: IPolicyForHTTPS,
+            pool: HTTPConnectionPool,
+        ) -> IAgent:
+            return Agent(reactor, tls_context_factory, pool=pool)
+
+        if agent_factory is None:
+            agent_factory = default_agent_factory
+
         treq_client = HTTPClient(
-            Agent(
+            agent_factory(
                 reactor,
                 _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
-                pool=pool,
+                pool,
             )
         )
 
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index af14201fb..535bc3ffa 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -51,6 +51,7 @@ from zope.interface import (
 )
 from twisted.python.failure import Failure
 from twisted.web import http
+from twisted.web.iweb import IAgent, IPolicyForHTTPS
 from twisted.internet.task import LoopingCall
 from twisted.internet import defer, reactor
 from twisted.application import service
@@ -203,9 +204,13 @@ class StorageFarmBroker(service.MultiService):
             tub_maker,
             node_config: _Config,
             storage_client_config=None,
+            default_connection_handlers=None,
             tor_provider: Optional[TorProvider]=None,
     ):
         service.MultiService.__init__(self)
+        if default_connection_handlers is None:
+            default_connection_handlers = {"tcp": "tcp"}
+
         assert permute_peers # False not implemented yet
         self.permute_peers = permute_peers
         self._tub_maker = tub_maker
@@ -226,6 +231,7 @@ class StorageFarmBroker(service.MultiService):
         self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred)
         self._connected_high_water_mark = 0
         self._tor_provider = tor_provider
+        self._default_connection_handlers = default_connection_handlers
 
     @log_call(action_type=u"storage-client:broker:set-static-servers")
     def set_static_servers(self, servers):
@@ -318,7 +324,8 @@ class StorageFarmBroker(service.MultiService):
                 server_id,
                 server["ann"],
                 grid_manager_verifier=gm_verifier,
-                tor_provider=tor_provider
+                default_connection_handlers=self._default_connection_handlers,
+                tor_provider=self._tor_provider
             )
             s.on_status_changed(lambda _: self._got_connection())
             return s
@@ -1053,7 +1060,7 @@ class HTTPNativeStorageServer(service.MultiService):
     "connected".
     """
 
-    def __init__(self, server_id: bytes, announcement, reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None):
+    def __init__(self, server_id: bytes, announcement, default_connection_handlers: dict[str,str], reactor=reactor, grid_manager_verifier=None, tor_provider: Optional[TorProvider]=None):
         service.MultiService.__init__(self)
         assert isinstance(server_id, bytes)
         self._server_id = server_id
@@ -1062,6 +1069,7 @@ class HTTPNativeStorageServer(service.MultiService):
         self._reactor = reactor
         self._grid_manager_verifier = grid_manager_verifier
         self._tor_provider = tor_provider
+        self._default_connection_handlers = default_connection_handlers
 
         furl = announcement["anonymous-storage-FURL"].encode("utf-8")
         (
@@ -1224,6 +1232,17 @@ class HTTPNativeStorageServer(service.MultiService):
         self._connecting_deferred = connecting
         return connecting
 
+    def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]:
+        """Return a factory for ``twisted.web.iweb.IAgent``."""
+        # TODO default_connection_handlers should really be an object, not a dict...
+        handler = self._default_connection_handlers["tcp"]
+        if handler == "tcp":
+            return None
+        if handler == "tor":
+            raise RuntimeError("TODO implement this next")
+        else:
+            raise RuntimeError(f"Unsupported tcp connection {handler}")
+
     @async_to_deferred
     async def _pick_server_and_get_version(self):
         """

From ffecdf8c773126a4feba6556d83a1d68dd6a70d6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 18 May 2023 10:18:46 -0400
Subject: [PATCH 287/362] Switch to non-deprecated API.

---
 src/allmydata/test/test_tor_provider.py |  8 ++++----
 src/allmydata/util/tor_provider.py      | 16 +++++++---------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py
index 86d54803a..eebb1ceef 100644
--- a/src/allmydata/test/test_tor_provider.py
+++ b/src/allmydata/test/test_tor_provider.py
@@ -94,16 +94,16 @@ class LaunchTor(unittest.TestCase):
         reactor = object()
         private_dir = "private"
         txtorcon = mock.Mock()
-        tpp = mock.Mock
-        tpp.tor_protocol = mock.Mock()
-        txtorcon.launch_tor = mock.Mock(return_value=tpp)
+        tor = mock.Mock
+        tor.protocol = mock.Mock()
+        txtorcon.launch = mock.Mock(return_value=tor)
 
         with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
                         return_value=999999):
             d = tor_provider._launch_tor(reactor, tor_executable, private_dir,
                                          txtorcon)
         tor_control_endpoint, tor_control_proto = self.successResultOf(d)
-        self.assertIs(tor_control_proto, tpp.tor_protocol)
+        self.assertIs(tor_control_proto, tor.protocol)
 
     def test_launch(self):
         return self._do_test_launch(None)
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index 57bb3a83d..4dab6b866 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -100,18 +100,16 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
     # us against one Tor being on $PATH at create-node time, but then a
     # different Tor being present at node startup. OTOH, maybe we don't
     # need to worry about it.
-    tor_config = txtorcon.TorConfig()
-    tor_config.DataDirectory = data_directory(private_dir)
 
     # unix-domain control socket
-    tor_config.ControlPort = "unix:" + os.path.join(private_dir, "tor.control")
-    tor_control_endpoint_desc = tor_config.ControlPort
+    tor_control_endpoint_desc = "unix:" + os.path.join(private_dir, "tor.control")
 
-    tor_config.SOCKSPort = allocate_tcp_port()
-
-    tpp = yield txtorcon.launch_tor(
-        tor_config, reactor,
+    tor = yield txtorcon.launch(
+        reactor,
+        control_port=tor_control_endpoint_desc,
+        data_directory=data_directory(private_dir),
         tor_binary=tor_executable,
+        socks_port=allocate_tcp_port(),
         # can be useful when debugging; mirror Tor's output to ours
         # stdout=sys.stdout,
         # stderr=sys.stderr,
@@ -119,7 +117,7 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
 
     # now tor is launched and ready to be spoken to
     # as a side effect, we've got an ITorControlProtocol ready to go
-    tor_control_proto = tpp.tor_protocol
+    tor_control_proto = tor.protocol
 
     # How/when to shut down the new process? for normal usage, the child
     # tor will exit when it notices its parent (us) quit. Unit tests will

From 34accd694c565c9d9a1c99b2bdf4069ba644085f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 18 May 2023 10:51:31 -0400
Subject: [PATCH 288/362] Refactor to return something more useful.

---
 src/allmydata/test/test_tor_provider.py | 34 ++++++++++++-------------
 src/allmydata/util/tor_provider.py      | 17 ++++++++-----
 2 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py
index eebb1ceef..295b258f0 100644
--- a/src/allmydata/test/test_tor_provider.py
+++ b/src/allmydata/test/test_tor_provider.py
@@ -1,15 +1,8 @@
 """
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+from collections import namedtuple
 import os
 from twisted.trial import unittest
 from twisted.internet import defer, error
@@ -95,15 +88,14 @@ class LaunchTor(unittest.TestCase):
         private_dir = "private"
         txtorcon = mock.Mock()
         tor = mock.Mock
-        tor.protocol = mock.Mock()
         txtorcon.launch = mock.Mock(return_value=tor)
 
         with mock.patch("allmydata.util.tor_provider.allocate_tcp_port",
                         return_value=999999):
             d = tor_provider._launch_tor(reactor, tor_executable, private_dir,
                                          txtorcon)
-        tor_control_endpoint, tor_control_proto = self.successResultOf(d)
-        self.assertIs(tor_control_proto, tor.protocol)
+        tor_control_endpoint, tor_result = self.successResultOf(d)
+        self.assertIs(tor_result, tor)
 
     def test_launch(self):
         return self._do_test_launch(None)
@@ -161,6 +153,12 @@ class ConnectToTor(unittest.TestCase):
         return self._do_test_connect(None, False)
 
 
+class FakeTor:
+    """Pretends to be a ``txtorcon.Tor`` instance."""
+    def __init__(self):
+        self.protocol = object()
+
+
 class CreateOnion(unittest.TestCase):
     def test_no_txtorcon(self):
         with mock.patch("allmydata.util.tor_provider._import_txtorcon",
@@ -171,6 +169,7 @@ class CreateOnion(unittest.TestCase):
             self.assertEqual(str(f.value),
                              "Cannot create onion without txtorcon. "
                              "Please 'pip install tahoe-lafs[tor]' to fix this.")
+
     def _do_test_launch(self, executable):
         basedir = self.mktemp()
         os.mkdir(basedir)
@@ -181,9 +180,9 @@ class CreateOnion(unittest.TestCase):
         if executable:
             args.append("--tor-executable=%s" % executable)
         cli_config = make_cli_config(basedir, *args)
-        protocol = object()
+        tor_instance = FakeTor()
         launch_tor = mock.Mock(return_value=defer.succeed(("control_endpoint",
-                                                           protocol)))
+                                                           tor_instance)))
         txtorcon = mock.Mock()
         ehs = mock.Mock()
         # This appears to be a native string in the real txtorcon object...
@@ -204,8 +203,8 @@ class CreateOnion(unittest.TestCase):
         launch_tor.assert_called_with(reactor, executable,
                                       os.path.abspath(private_dir), txtorcon)
         txtorcon.EphemeralHiddenService.assert_called_with("3457 127.0.0.1:999999")
-        ehs.add_to_tor.assert_called_with(protocol)
-        ehs.remove_from_tor.assert_called_with(protocol)
+        ehs.add_to_tor.assert_called_with(tor_instance.protocol)
+        ehs.remove_from_tor.assert_called_with(tor_instance.protocol)
 
         expected = {"launch": "true",
                     "onion": "true",
@@ -587,13 +586,14 @@ class Provider_Service(unittest.TestCase):
         txtorcon = mock.Mock()
         with mock_txtorcon(txtorcon):
             p = tor_provider.create(reactor, cfg)
+        tor_instance = FakeTor()
         tor_state = mock.Mock()
-        tor_state.protocol = object()
+        tor_state.protocol = tor_instance.protocol
         ehs = mock.Mock()
         ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
         ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
         txtorcon.EphemeralHiddenService = mock.Mock(return_value=ehs)
-        launch_tor = mock.Mock(return_value=defer.succeed((None,tor_state.protocol)))
+        launch_tor = mock.Mock(return_value=defer.succeed((None,tor_instance)))
         with mock.patch("allmydata.util.tor_provider._launch_tor",
                         launch_tor):
             d = p.startService()
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index 4dab6b866..9daf302cf 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -23,6 +23,7 @@ from ..interfaces import (
     IAddressFamily,
 )
 
+
 def _import_tor():
     try:
         from foolscap.connections import tor
@@ -94,6 +95,10 @@ def _try_to_connect(reactor, endpoint_desc, stdout, txtorcon):
 
 @inlineCallbacks
 def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
+    """
+    Launches Tor, returns a corresponding ``(control endpoint string,
+    txtorcon.Tor instance)`` tuple.
+    """
     # TODO: handle default tor-executable
     # TODO: it might be a good idea to find exactly which Tor we used,
     # and record it's absolute path into tahoe.cfg . This would protect
@@ -115,10 +120,6 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
         # stderr=sys.stderr,
     )
 
-    # now tor is launched and ready to be spoken to
-    # as a side effect, we've got an ITorControlProtocol ready to go
-    tor_control_proto = tor.protocol
-
     # How/when to shut down the new process? for normal usage, the child
     # tor will exit when it notices its parent (us) quit. Unit tests will
     # mock out txtorcon.launch_tor(), so there will never be a real Tor
@@ -128,7 +129,8 @@ def _launch_tor(reactor, tor_executable, private_dir, txtorcon):
     # (because it's a TorProcessProtocol) which returns a Deferred
     # that fires when Tor has actually exited.
 
-    returnValue((tor_control_endpoint_desc, tor_control_proto))
+    returnValue((tor_control_endpoint_desc, tor))
+
 
 @inlineCallbacks
 def _connect_to_tor(reactor, cli_config, txtorcon):
@@ -163,8 +165,9 @@ def create_config(reactor, cli_config):
         if tor_executable:
             tahoe_config_tor["tor.executable"] = tor_executable
         print("launching Tor (to allocate .onion address)..", file=stdout)
-        (_, tor_control_proto) = yield _launch_tor(
+        (_, tor) = yield _launch_tor(
             reactor, tor_executable, private_dir, txtorcon)
+        tor_control_proto = tor.protocol
         print("Tor launched", file=stdout)
     else:
         print("connecting to Tor (to allocate .onion address)..", file=stdout)
@@ -288,7 +291,7 @@ class _Provider(service.MultiService):
         returnValue(tor_control_endpoint)
 
     def _get_launched_tor(self, reactor):
-        # this fires with a tuple of (control_endpoint, tor_protocol)
+        # this fires with a tuple of (control_endpoint, txtorcon.Tor instance)
         if not self._tor_launched:
             self._tor_launched = OneShotObserverList()
             private_dir = self._config.get_config_path("private")

From 47991f23fa5cb40eb60283d1503c0afaceae1272 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 18 May 2023 11:05:38 -0400
Subject: [PATCH 289/362] More refactoring to make it easier to get a
 txtorcon.Tor instance.

---
 src/allmydata/test/test_tor_provider.py | 13 ++++++-------
 src/allmydata/util/tor_provider.py      | 13 ++++++++-----
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py
index 295b258f0..bd40a9b4a 100644
--- a/src/allmydata/test/test_tor_provider.py
+++ b/src/allmydata/test/test_tor_provider.py
@@ -628,9 +628,8 @@ class Provider_Service(unittest.TestCase):
         txtorcon = mock.Mock()
         with mock_txtorcon(txtorcon):
             p = tor_provider.create(reactor, cfg)
-        tor_state = mock.Mock()
-        tor_state.protocol = object()
-        txtorcon.build_tor_connection = mock.Mock(return_value=tor_state)
+        tor_instance = FakeTor()
+        txtorcon.connect = mock.Mock(return_value=tor_instance)
         ehs = mock.Mock()
         ehs.add_to_tor = mock.Mock(return_value=defer.succeed(None))
         ehs.remove_from_tor = mock.Mock(return_value=defer.succeed(None))
@@ -642,12 +641,12 @@ class Provider_Service(unittest.TestCase):
             yield flushEventualQueue()
         self.successResultOf(d)
         self.assertIs(p._onion_ehs, ehs)
-        self.assertIs(p._onion_tor_control_proto, tor_state.protocol)
+        self.assertIs(p._onion_tor_control_proto, tor_instance.protocol)
         cfs.assert_called_with(reactor, "ep_desc")
-        txtorcon.build_tor_connection.assert_called_with(tcep)
+        txtorcon.connect.assert_called_with(reactor, tcep)
         txtorcon.EphemeralHiddenService.assert_called_with("456 127.0.0.1:123",
                                                            b"private key")
-        ehs.add_to_tor.assert_called_with(tor_state.protocol)
+        ehs.add_to_tor.assert_called_with(tor_instance.protocol)
 
         yield p.stopService()
-        ehs.remove_from_tor.assert_called_with(tor_state.protocol)
+        ehs.remove_from_tor.assert_called_with(tor_instance.protocol)
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index 9daf302cf..ad36b6986 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -322,17 +322,20 @@ class _Provider(service.MultiService):
             require("external_port")
             require("private_key_file")
 
-    @inlineCallbacks
-    def _start_onion(self, reactor):
+    def _get_tor_instance(self, reactor: object):
+        """Return a ``Deferred`` that fires with a ``txtorcon.Tor`` instance."""
         # launch tor, if necessary
         if self._get_tor_config("launch", False, boolean=True):
-            (_, tor_control_proto) = yield self._get_launched_tor(reactor)
+            return self._get_launched_tor(reactor).addCallback(lambda t: t[1])
         else:
             controlport = self._get_tor_config("control.port", None)
             tcep = clientFromString(reactor, controlport)
-            tor_state = yield self._txtorcon.build_tor_connection(tcep)
-            tor_control_proto = tor_state.protocol
+            return self._txtorcon.connect(reactor, tcep)
 
+    @inlineCallbacks
+    def _start_onion(self, reactor):
+        tor_instance = yield self._get_tor_instance(reactor)
+        tor_control_proto = tor_instance.protocol
         local_port = int(self._get_tor_config("onion.local_port"))
         external_port = int(self._get_tor_config("onion.external_port"))
 

From 2e0e0467fb56f655418cb2131545eb8e2e286432 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 18 May 2023 11:14:51 -0400
Subject: [PATCH 290/362] Hook up HTTP storage client Tor support.

---
 src/allmydata/storage_client.py    | 11 ++++++++---
 src/allmydata/util/tor_provider.py |  4 ++--
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 535bc3ffa..f6b3972f5 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1232,16 +1232,21 @@ class HTTPNativeStorageServer(service.MultiService):
         self._connecting_deferred = connecting
         return connecting
 
-    def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]:
+    async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]:
         """Return a factory for ``twisted.web.iweb.IAgent``."""
         # TODO default_connection_handlers should really be an object, not a dict...
         handler = self._default_connection_handlers["tcp"]
         if handler == "tcp":
             return None
         if handler == "tor":
-            raise RuntimeError("TODO implement this next")
+            tor_instance = await self._tor_provider.get_tor_instance(self._reactor)
+
+            def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent:
+                assert reactor == self._reactor
+                return tor_instance.web_agent(pool=pool, tls_context_factory=tls_context_factory)
+            return agent_factory
         else:
-            raise RuntimeError(f"Unsupported tcp connection {handler}")
+            raise RuntimeError(f"Unsupported tcp connection handler: {handler}")
 
     @async_to_deferred
     async def _pick_server_and_get_version(self):
diff --git a/src/allmydata/util/tor_provider.py b/src/allmydata/util/tor_provider.py
index ad36b6986..aaf43db73 100644
--- a/src/allmydata/util/tor_provider.py
+++ b/src/allmydata/util/tor_provider.py
@@ -322,7 +322,7 @@ class _Provider(service.MultiService):
             require("external_port")
             require("private_key_file")
 
-    def _get_tor_instance(self, reactor: object):
+    def get_tor_instance(self, reactor: object):
         """Return a ``Deferred`` that fires with a ``txtorcon.Tor`` instance."""
         # launch tor, if necessary
         if self._get_tor_config("launch", False, boolean=True):
@@ -334,7 +334,7 @@ class _Provider(service.MultiService):
 
     @inlineCallbacks
     def _start_onion(self, reactor):
-        tor_instance = yield self._get_tor_instance(reactor)
+        tor_instance = yield self.get_tor_instance(reactor)
         tor_control_proto = tor_instance.protocol
         local_port = int(self._get_tor_config("onion.local_port"))
         external_port = int(self._get_tor_config("onion.external_port"))

From 0ccee4e958f9a70148aff3bf6bc82a6fc78a3a1b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 19 May 2023 13:59:18 -0400
Subject: [PATCH 291/362] Hook up the Tor-based Agent when necessary.

---
 src/allmydata/storage_client.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index f6b3972f5..15303265f 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1266,24 +1266,22 @@ class HTTPNativeStorageServer(service.MultiService):
             # version() calls before we are live talking to a server, it could only
             # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992
 
+            agent_factory = await self._agent_factory()
+
             def request(reactor, nurl: DecodedURL):
                 # Since we're just using this one off to check if the NURL
                 # works, no need for persistent pool or other fanciness.
                 pool = HTTPConnectionPool(reactor, persistent=False)
                 pool.retryAutomatically = False
                 return StorageClientGeneral(
-                    # TODO if Tor client connections are enabled, use an Agent
-                    # created via tor.
-                    StorageClient.from_nurl(nurl, reactor, pool)
+                    StorageClient.from_nurl(nurl, reactor, pool, agent_factory=agent_factory)
                 ).get_version()
 
             nurl = await _pick_a_http_server(reactor, self._nurls, request)
 
             # If we've gotten this far, we've found a working NURL.
             self._istorage_server = _HTTPStorageServer.from_http_client(
-                # TODO if Tor client connections are enabled, use an Agent
-                # created via tor.
-                StorageClient.from_nurl(nurl, reactor)
+                StorageClient.from_nurl(nurl, reactor, agent_factory=agent_factory)
             )
             return self._istorage_server
 

From 83d8efbb62ce21cf10303b63d0f92badd0f3dfd3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 19 May 2023 13:59:29 -0400
Subject: [PATCH 292/362] Require the appropriate version of txtorcon.

---
 setup.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index 0453fa63f..ec4936645 100644
--- a/setup.py
+++ b/setup.py
@@ -164,10 +164,9 @@ setup_requires = [
 ]
 
 tor_requires = [
-    # This is exactly what `foolscap[tor]` means but pip resolves the pair of
-    # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose
-    # this if we don't declare it ourselves!
-    "txtorcon >= 0.17.0",
+    # 23.5 added support for custom TLS contexts in web_agent(), which is
+    # needed for the HTTP storage client to run over Tor.
+    "txtorcon >= 23.5.0",
 ]
 
 i2p_requires = [

From a1e00ffc3f5eb359bf10ce05fb279fb3ac3739d9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 19 May 2023 13:59:43 -0400
Subject: [PATCH 293/362] Add a test that triggers client-side HTTP storage
 client to use Tor.

---
 integration/test_tor.py | 51 +++++++++++++++++++++++++++++---------
 integration/util.py     | 54 +++++++++++++++++++++++------------------
 2 files changed, 71 insertions(+), 34 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 10e326e46..35e1581eb 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -19,6 +19,7 @@ from allmydata.test.common import (
     write_introducer,
 )
 from allmydata.client import read_config
+from allmydata.util.deferredutil import async_to_deferred
 
 # see "conftest.py" for the fixtures (e.g. "tor_network")
 
@@ -31,13 +32,26 @@ if sys.platform.startswith('win'):
 
 @pytest_twisted.inlineCallbacks
 def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl):
+    """
+    Two nodes and an introducer all configured to use Tahoe.
+
+    The two nodes can talk to the introducer and each other: we upload to one
+    node, read from the other.
+    """
     carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
     yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
     yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave)
+
+
+@async_to_deferred
+async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: util.TahoeProcess, download_from: util.TahoeProcess):
+    """
+    Ensure both nodes are connected to "a grid" by uploading something via one
+    node, and retrieve it using the other.
+    """
 
-    # ensure both nodes are connected to "a grid" by uploading
-    # something via carol, and retrieve it using dave.
     gold_path = join(temp_dir, "gold")
     with open(gold_path, "w") as f:
         f.write(
@@ -54,12 +68,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         sys.executable,
         (
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
-            '-d', join(temp_dir, 'carol'),
+            '-d', upload_to.node_dir,
             'put', gold_path,
         ),
         env=environ,
     )
-    yield proto.done
+    await proto.done
     cap = proto.output.getvalue().strip().split()[-1]
     print("capability: {}".format(cap))
 
@@ -69,19 +83,18 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
         sys.executable,
         (
             sys.executable, '-b', '-m', 'allmydata.scripts.runner',
-            '-d', join(temp_dir, 'dave'),
+            '-d', download_from.node_dir,
             'get', cap,
         ),
         env=environ,
     )
-    yield proto.done
-
-    dave_got = proto.output.getvalue().strip()
-    assert dave_got == open(gold_path, 'rb').read().strip()
+    await proto.done
+    download_got = proto.output.getvalue().strip()
+    assert download_got == open(gold_path, 'rb').read().strip()
 
 
 @pytest_twisted.inlineCallbacks
-def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl) -> util.TahoeProcess:
     node_dir = FilePath(temp_dir).child(name)
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
@@ -113,9 +126,9 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
+    util.basic_node_configuration(request, flog_gatherer, node_dir.path)
 
     config = read_config(node_dir.path, "tub.port")
-    config.set_config("node", "log_gatherer.furl", flog_gatherer)
     config.set_config("tor", "onion", "true")
     config.set_config("tor", "onion.external_port", "3457")
     config.set_config("tor", "control.port", f"tcp:port={control_port}:host=127.0.0.1")
@@ -125,3 +138,19 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     result = yield util._run_node(reactor, node_dir.path, request, None)
     print("okay, launched")
     return result
+
+
+@pytest_twisted.inlineCallbacks
+def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+    """
+    A normal node (alice) and a normal introducer are configured, and one node
+    (carol) which is configured to be anonymous by talking via Tor.
+
+    Carol should be able to communicate with alice.
+
+    TODO how to ensure that carol is actually using Tor?
+    """
+    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl)
+    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
+
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, carol)
diff --git a/integration/util.py b/integration/util.py
index cbc701fbc..402c14932 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -311,6 +311,36 @@ def _run_node(reactor, node_dir, request, magic_text, finalize=True):
     return d
 
 
+def basic_node_configuration(request, flog_gatherer, node_dir: str):
+    """
+    Setup common configuration options for a node, given a ``pytest`` request
+    fixture.
+    """
+    config_path = join(node_dir, 'tahoe.cfg')
+    config = get_config(config_path)
+    set_config(
+        config,
+        u'node',
+        u'log_gatherer.furl',
+        flog_gatherer,
+    )
+    force_foolscap = request.config.getoption("force_foolscap")
+    assert force_foolscap in (True, False)
+    set_config(
+        config,
+        'storage',
+        'force_foolscap',
+        str(force_foolscap),
+    )
+    set_config(
+        config,
+        'client',
+        'force_foolscap',
+        str(force_foolscap),
+    )
+    write_config(FilePath(config_path), config)
+
+
 def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, name, web_port,
                  storage=True,
                  magic_text=None,
@@ -351,29 +381,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
         created_d = done_proto.done
 
         def created(_):
-            config_path = join(node_dir, 'tahoe.cfg')
-            config = get_config(config_path)
-            set_config(
-                config,
-                u'node',
-                u'log_gatherer.furl',
-                flog_gatherer,
-            )
-            force_foolscap = request.config.getoption("force_foolscap")
-            assert force_foolscap in (True, False)
-            set_config(
-                config,
-                'storage',
-                'force_foolscap',
-                str(force_foolscap),
-            )
-            set_config(
-                config,
-                'client',
-                'force_foolscap',
-                str(force_foolscap),
-            )
-            write_config(FilePath(config_path), config)
+            basic_node_configuration(request, flog_gatherer, node_dir)
         created_d.addCallback(created)
 
     d = Deferred()

From 1b54853d3f8f18f686bc9d7363a79afa0b6935e5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Fri, 19 May 2023 14:01:08 -0400
Subject: [PATCH 294/362] News file.

---
 newsfragments/4029.bugfix | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 newsfragments/4029.bugfix

diff --git a/newsfragments/4029.bugfix b/newsfragments/4029.bugfix
new file mode 100644
index 000000000..3ce4670ec
--- /dev/null
+++ b/newsfragments/4029.bugfix
@@ -0,0 +1,2 @@
+The (still off-by-default) HTTP storage client will now use Tor when Tor-based client-side anonymity was requested.
+Previously it would use normal TCP connections and not be anonymous.
\ No newline at end of file

From f5520fdf74ea5d9f3fafd3605e93a7181f98f5ec Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 22 May 2023 11:42:13 -0400
Subject: [PATCH 295/362] Better name.

---
 integration/test_tor.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 35e1581eb..ec5cc1bc4 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -144,13 +144,13 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
     """
     A normal node (alice) and a normal introducer are configured, and one node
-    (carol) which is configured to be anonymous by talking via Tor.
+    (anonymoose) which is configured to be anonymous by talking via Tor.
 
-    Carol should be able to communicate with alice.
+    Anonymoose should be able to communicate with alice.
 
-    TODO how to ensure that carol is actually using Tor?
+    TODO how to ensure that anonymoose is actually using Tor?
     """
-    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl)
-    yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
+    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl)
+    yield util.await_client_ready(anonymoose, minimum_number_of_servers=2, timeout=600)
 
-    yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, carol)
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, anonymoose)

From 2741fb2b46afa54888d5591d9b757f639d8aae4d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 22 May 2023 12:51:40 -0400
Subject: [PATCH 296/362] Don't persist state unnecessarily (and this appears
 to cause test failures)

---
 integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/integration/conftest.py b/integration/conftest.py
index 6892b33a7..b29b9fe36 100644
--- a/integration/conftest.py
+++ b/integration/conftest.py
@@ -279,7 +279,7 @@ def introducer_furl(introducer, temp_dir):
     return furl
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 @log_call(
     action_type=u"integration:tor:introducer",
     include_args=["temp_dir", "flog_gatherer"],
@@ -342,7 +342,7 @@ def tor_introducer(reactor, temp_dir, flog_gatherer, request):
     return transport
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture
 def tor_introducer_furl(tor_introducer, temp_dir):
     furl_fname = join(temp_dir, 'introducer_tor', 'private', 'introducer.furl')
     while not exists(furl_fname):

From 1ed440812a86ab91289ff1ff21d4f2dfe6e4cb15 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 22 May 2023 13:00:20 -0400
Subject: [PATCH 297/362] Add a safety check.

---
 src/allmydata/storage/http_client.py | 8 ++++++++
 src/allmydata/storage_client.py      | 9 +++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 12d6c5feb..670d84be3 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -341,6 +341,9 @@ class StorageClient(object):
         cls,
         nurl: DecodedURL,
         reactor,
+        # TODO default_connection_handlers should really be a class, not a dict
+        # of strings...
+        default_connection_handlers: dict[str, str],
         pool: Optional[HTTPConnectionPool] = None,
         agent_factory: Optional[
             Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent]
@@ -349,6 +352,11 @@ class StorageClient(object):
         """
         Create a ``StorageClient`` for the given NURL.
         """
+        # Safety check: if we're using normal TCP connections, we better not be
+        # configured for Tor or I2P.
+        if agent_factory is None:
+            assert default_connection_handlers["tcp"] == "tcp"
+
         assert nurl.fragment == "v=1"
         assert nurl.scheme == "pb"
         swissnum = nurl.path[0].encode("ascii")
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 7bf00ad93..326b96ab4 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1274,14 +1274,19 @@ class HTTPNativeStorageServer(service.MultiService):
                 pool = HTTPConnectionPool(reactor, persistent=False)
                 pool.retryAutomatically = False
                 return StorageClientGeneral(
-                    StorageClient.from_nurl(nurl, reactor, pool, agent_factory=agent_factory)
+                    StorageClient.from_nurl(
+                        nurl, reactor, self._default_connection_handlers,
+                        pool=pool, agent_factory=agent_factory)
                 ).get_version()
 
             nurl = await _pick_a_http_server(reactor, self._nurls, request)
 
             # If we've gotten this far, we've found a working NURL.
             self._istorage_server = _HTTPStorageServer.from_http_client(
-                StorageClient.from_nurl(nurl, reactor, agent_factory=agent_factory)
+                StorageClient.from_nurl(
+                    nurl, reactor, self._default_connection_handlers,
+                    agent_factory=agent_factory
+                )
             )
             return self._istorage_server
 

From 084499dd4b53e08ffcab1a87cb5542b54c44998c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 22 May 2023 13:02:58 -0400
Subject: [PATCH 298/362] Fix lint.

---
 src/allmydata/test/test_tor_provider.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/test/test_tor_provider.py b/src/allmydata/test/test_tor_provider.py
index bd40a9b4a..20f947d55 100644
--- a/src/allmydata/test/test_tor_provider.py
+++ b/src/allmydata/test/test_tor_provider.py
@@ -2,7 +2,6 @@
 Ported to Python 3.
 """
 
-from collections import namedtuple
 import os
 from twisted.trial import unittest
 from twisted.internet import defer, error

From 71cb357f45db29561b239fdf53d4eca714502e1f Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 22 May 2023 13:03:46 -0400
Subject: [PATCH 299/362] Upstream code should make sure this doesn't happen.

---
 src/allmydata/storage_client.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 326b96ab4..59f4242f6 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1239,6 +1239,7 @@ class HTTPNativeStorageServer(service.MultiService):
         if handler == "tcp":
             return None
         if handler == "tor":
+            assert self._tor_provider is not None
             tor_instance = await self._tor_provider.get_tor_instance(self._reactor)
 
             def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent:

From d15ea8cb52b87426c82fd1c70581ced87c7d391e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 23 May 2023 13:24:29 -0400
Subject: [PATCH 300/362] Shutdown more immediately.

---
 src/allmydata/storage_client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 94aae43f6..8541bb40c 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1274,7 +1274,8 @@ class HTTPNativeStorageServer(service.MultiService):
 
         maybe_storage_server = self.get_storage_server()
         if maybe_storage_server is not None:
-            result.addCallback(lambda _: maybe_storage_server._http_client.shutdown())
+            client_shutting_down = maybe_storage_server._http_client.shutdown()
+            result.addCallback(lambda _: client_shutting_down)
 
         return result
 

From 1e46e36ee2cde0872b6fd0237f3586c489af0352 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 23 May 2023 13:46:32 -0400
Subject: [PATCH 301/362] More direct approach.

---
 src/allmydata/storage_client.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 8541bb40c..200f693c0 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1272,9 +1272,8 @@ class HTTPNativeStorageServer(service.MultiService):
             self._lc.stop()
         self._failed_to_connect("shut down")
 
-        maybe_storage_server = self.get_storage_server()
-        if maybe_storage_server is not None:
-            client_shutting_down = maybe_storage_server._http_client.shutdown()
+        if self._istorage_server is not None:
+            client_shutting_down = self._istorage_server._http_client.shutdown()
             result.addCallback(lambda _: client_shutting_down)
 
         return result

From 652c179602c36f71b63be502b1ee6e709f66102c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 23 May 2023 14:08:03 -0400
Subject: [PATCH 302/362] Remove comment.

---
 src/allmydata/test/test_storage_http.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index df5dc300c..a6e6205f1 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -379,8 +379,6 @@ class CustomHTTPServerTests(SyncTestCase):
         )
         self.assertEqual(response.code, 400)
 
-    # TODO test other garbage values
-
     def test_authorization_enforcement(self):
         """
         The requirement for secrets is enforced by the ``_authorized_route``

From 0e28c8ed4a88e39e500a8c3902e26a7787db650d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 24 May 2023 08:54:56 -0400
Subject: [PATCH 303/362] bump the nix package of txtorcon

---
 nix/python-overrides.nix | 5 ++++-
 nix/txtorcon.nix         | 9 +++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 nix/txtorcon.nix

diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
index 032b427ae..d1c995e66 100644
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@@ -21,11 +21,14 @@ in {
   pycddl = self.callPackage ./pycddl.nix { };
   txi2p = self.callPackage ./txi2p.nix { };
 
-  # Update the version of klein.
+  # Some packages are of somewhat too-old versions - update them.
   klein = self.callPackage ./klein.nix {
     # Avoid infinite recursion.
     inherit (super) klein;
   };
+  txtorcon = self.callPackage ./txtorcon.nix {
+    inherit (super) txtorcon;
+  };
 
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
   # we know where a working version lives.
diff --git a/nix/txtorcon.nix b/nix/txtorcon.nix
new file mode 100644
index 000000000..552c03fd0
--- /dev/null
+++ b/nix/txtorcon.nix
@@ -0,0 +1,9 @@
+{ txtorcon, fetchPypi }:
+txtorcon.overrideAttrs (old: rec {
+  pname = "txtorcon";
+  version = "23.5.0";
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw=";
+  };
+})

From 96670ded65445aa07628816e6b7523e7d03b315e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 1 Jun 2023 17:27:21 -0400
Subject: [PATCH 304/362] Switch to using officially support constants, now
 part of pyOpenSSL's public API.

The cryptography APIs we were previously using were not supported and aren't
available in all releases.
---
 newsfragments/3998.minor             |  0
 setup.py                             |  7 +++----
 src/allmydata/storage/http_client.py | 13 +++++--------
 3 files changed, 8 insertions(+), 12 deletions(-)
 create mode 100644 newsfragments/3998.minor

diff --git a/newsfragments/3998.minor b/newsfragments/3998.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/setup.py b/setup.py
index 0453fa63f..c40e2dd2e 100644
--- a/setup.py
+++ b/setup.py
@@ -63,11 +63,10 @@ install_requires = [
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
     #   transitively depends on cryptography.  So it's anyone's guess what
     #   version of cryptography will *really* be installed.
+    "cryptography >= 2.6",
 
-    # * cryptography 40 broke constants we need; should really be using them
-    # * via pyOpenSSL; will be fixed in
-    # * https://github.com/pyca/pyopenssl/issues/1201
-    "cryptography >= 2.6, < 40",
+    # * Used for custom HTTPS validation
+    "pyOpenSSL >= 23.2.0",
 
     # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
     #   rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 5464b2e25..65f079aeb 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -46,7 +46,6 @@ import treq
 from treq.client import HTTPClient
 from treq.testing import StubTreq
 from OpenSSL import SSL
-from cryptography.hazmat.bindings.openssl.binding import Binding
 from werkzeug.http import parse_content_range_header
 
 from .http_common import (
@@ -60,8 +59,6 @@ from .common import si_b2a, si_to_human_readable
 from ..util.hashutil import timing_safe_compare
 from ..util.deferredutil import async_to_deferred
 
-_OPENSSL = Binding().lib
-
 
 def _encode_si(si):  # type: (bytes) -> str
     """Encode the storage index into Unicode string."""
@@ -256,11 +253,11 @@ class _TLSContextFactory(CertificateOptions):
             # not the usual TLS concerns about invalid CAs or revoked
             # certificates.
             things_are_ok = (
-                _OPENSSL.X509_V_OK,
-                _OPENSSL.X509_V_ERR_CERT_NOT_YET_VALID,
-                _OPENSSL.X509_V_ERR_CERT_HAS_EXPIRED,
-                _OPENSSL.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT,
-                _OPENSSL.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN,
+                SSL.X509VerificationCodes.OK,
+                SSL.X509VerificationCodes.ERR_CERT_NOT_YET_VALID,
+                SSL.X509VerificationCodes.ERR_CERT_HAS_EXPIRED,
+                SSL.X509VerificationCodes.ERR_DEPTH_ZERO_SELF_SIGNED_CERT,
+                SSL.X509VerificationCodes.ERR_SELF_SIGNED_CERT_IN_CHAIN,
             )
             # TODO can we do this once instead of multiple times?
             if errno in things_are_ok and timing_safe_compare(

From 01bc35f1297bfa5a6be774bc20b071ae7c76b639 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 10:29:57 -0400
Subject: [PATCH 305/362] Try to update nix pyopenssl.

---
 nix/pyopenssl.nix        | 9 +++++++++
 nix/python-overrides.nix | 6 ++++++
 2 files changed, 15 insertions(+)
 create mode 100644 nix/pyopenssl.nix

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
new file mode 100644
index 000000000..8afbf8bc1
--- /dev/null
+++ b/nix/pyopenssl.nix
@@ -0,0 +1,9 @@
+{ pyopenssl, fetchPypi }:
+pyopenssl.overrideAttrs (old: rec {
+  pname = "pyopenssl";
+  version = "23.2.0";
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7";
+  };
+})
diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
index 032b427ae..74cd3a893 100644
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@@ -27,6 +27,12 @@ in {
     inherit (super) klein;
   };
 
+  # Update the version of pyopenssl.
+  pyopenssl = self.callPackage ./pyopenssl.nix {
+    # Avoid infinite recursion.
+    inherit (super) pyopenssl;
+  };
+
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
   # we know where a working version lives.
   collections-extended = self.callPackage ./collections-extended.nix {

From 894cb46304c4a1d4035fa17c8962f4bac37cefa0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 11:27:23 -0400
Subject: [PATCH 306/362] Try merging the two overrides.

---
 nix/pyopenssl.nix        | 3 +++
 nix/python-overrides.nix | 4 ----
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index 8afbf8bc1..0c294a888 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -6,4 +6,7 @@ pyopenssl.overrideAttrs (old: rec {
     inherit pname version;
     sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7";
   };
+  # Building the docs requires sphinx which brings in a dependency on babel,
+  # the test suite of which fails.
+  dontBuildDocs = isPyPy;
 })
diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
index 74cd3a893..423297ef1 100644
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@@ -66,10 +66,6 @@ in {
   # a5f8184fb816a4fd5ae87136838c9981e0d22c67.
   six = onPyPy dontCheck super.six;
 
-  # Building the docs requires sphinx which brings in a dependency on babel,
-  # the test suite of which fails.
-  pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl;
-
   # Likewise for beautifulsoup4.
   beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
 

From 203fd84a887231dc6fc19501a177de1ef2e496ba Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 11:30:11 -0400
Subject: [PATCH 307/362] Need to import it.

---
 nix/pyopenssl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index 0c294a888..b94aa254c 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -1,4 +1,4 @@
-{ pyopenssl, fetchPypi }:
+{ pyopenssl, fetchPypi, isPyPy }:
 pyopenssl.overrideAttrs (old: rec {
   pname = "pyopenssl";
   version = "23.2.0";

From 6e6bae9bf6c008d05fac1e852248cec706de6cb8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 11:34:52 -0400
Subject: [PATCH 308/362] Some random other hash who knows

---
 nix/pyopenssl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index b94aa254c..5006cf1d0 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec {
   version = "23.2.0";
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-1b4bkcpzhmablf592g21rq3l8apbhklp6wcwlvgfflm4algr6vr7";
+    sha256 = "sha256-1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc";
   };
   # Building the docs requires sphinx which brings in a dependency on babel,
   # the test suite of which fails.

From 43e4e1b09a3da502bb0c3433948e03df9f9e05ab Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 11:52:24 -0400
Subject: [PATCH 309/362] Get rid of prefix.

---
 nix/pyopenssl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index 5006cf1d0..fbf377fa7 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec {
   version = "23.2.0";
   src = fetchPypi {
     inherit pname version;
-    sha256 = "sha256-1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc";
+    sha256 = "1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc";
   };
   # Building the docs requires sphinx which brings in a dependency on babel,
   # the test suite of which fails.

From 940600e0ed5a8edc78febd53aac8177bceb56500 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 5 Jun 2023 12:54:51 -0400
Subject: [PATCH 310/362] Link to ticket.

---
 src/allmydata/storage_client.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 1946b5da2..2ea154263 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -1234,7 +1234,10 @@ class HTTPNativeStorageServer(service.MultiService):
 
     async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]:
         """Return a factory for ``twisted.web.iweb.IAgent``."""
-        # TODO default_connection_handlers should really be an object, not a dict...
+        # TODO default_connection_handlers should really be an object, not a
+        # dict, so we can ask "is this using Tor" without poking at a
+        # dictionary with arbitrary strings... See
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032
         handler = self._default_connection_handlers["tcp"]
         if handler == "tcp":
             return None

From 5af0ead5b9b1232526511ab81ad1f59bf370a290 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 10:58:16 -0400
Subject: [PATCH 311/362] Refactor HTTP client creation to be more centralized.

---
 src/allmydata/storage/http_client.py    | 170 +++++++++++++++---------
 src/allmydata/storage_client.py         |  50 ++-----
 src/allmydata/test/common_system.py     |   4 +-
 src/allmydata/test/test_storage_http.py |  12 +-
 4 files changed, 128 insertions(+), 108 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 670d84be3..fe2545c03 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -16,6 +16,7 @@ from typing import (
     Set,
     Dict,
     Callable,
+    ClassVar,
 )
 from base64 import b64encode
 from io import BytesIO
@@ -60,6 +61,15 @@ from .http_common import (
 from .common import si_b2a, si_to_human_readable
 from ..util.hashutil import timing_safe_compare
 from ..util.deferredutil import async_to_deferred
+from ..util.tor_provider import _Provider as TorProvider
+
+try:
+    from txtorcon import Tor  # type: ignore
+except ImportError:
+
+    class Tor:
+        pass
+
 
 _OPENSSL = Binding().lib
 
@@ -302,18 +312,30 @@ class _StorageClientHTTPSPolicy:
         )
 
 
-@define(hash=True)
-class StorageClient(object):
+@define
+class StorageClientFactory:
     """
-    Low-level HTTP client that talks to the HTTP storage server.
+    Create ``StorageClient`` instances, using appropriate
+    ``twisted.web.iweb.IAgent`` for different connection methods: normal TCP,
+    Tor, and eventually I2P.
+
+    There is some caching involved since there might be shared setup work, e.g.
+    connecting to the local Tor service only needs to happen once.
     """
 
-    # If set, we're doing unit testing and we should call this with
-    # HTTPConnectionPool we create.
-    TEST_MODE_REGISTER_HTTP_POOL = None
+    _default_connection_handlers: dict[str, str]
+    _tor_provider: Optional[TorProvider]
+    # Cache the Tor instance created by the provider, if relevant.
+    _tor_instance: Optional[Tor] = None
+
+    # If set, we're doing unit testing and we should call this with any
+    # HTTPConnectionPool that gets passed/created to ``create_agent()``.
+    TEST_MODE_REGISTER_HTTP_POOL = ClassVar[
+        Optional[Callable[[HTTPConnectionPool], None]]
+    ]
 
     @classmethod
-    def start_test_mode(cls, callback):
+    def start_test_mode(cls, callback: Callable[[HTTPConnectionPool], None]) -> None:
         """Switch to testing mode.
 
         In testing mode we register the pool with test system using the given
@@ -328,66 +350,84 @@ class StorageClient(object):
         """Stop testing mode."""
         cls.TEST_MODE_REGISTER_HTTP_POOL = None
 
-    # The URL is a HTTPS URL ("https://...").  To construct from a NURL, use
-    # ``StorageClient.from_nurl()``.
-    _base_url: DecodedURL
-    _swissnum: bytes
-    _treq: Union[treq, StubTreq, HTTPClient]
-    _pool: Optional[HTTPConnectionPool]
-    _clock: IReactorTime
-
-    @classmethod
-    def from_nurl(
-        cls,
+    async def _create_agent(
+        self,
         nurl: DecodedURL,
-        reactor,
-        # TODO default_connection_handlers should really be a class, not a dict
-        # of strings...
-        default_connection_handlers: dict[str, str],
-        pool: Optional[HTTPConnectionPool] = None,
-        agent_factory: Optional[
-            Callable[[object, IPolicyForHTTPS, HTTPConnectionPool], IAgent]
-        ] = None,
-    ) -> StorageClient:
-        """
-        Create a ``StorageClient`` for the given NURL.
-        """
-        # Safety check: if we're using normal TCP connections, we better not be
-        # configured for Tor or I2P.
-        if agent_factory is None:
-            assert default_connection_handlers["tcp"] == "tcp"
+        reactor: object,
+        tls_context_factory: IPolicyForHTTPS,
+        pool: HTTPConnectionPool,
+    ) -> IAgent:
+        """Create a new ``IAgent``, possibly using Tor."""
+        if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
+            self.TEST_MODE_REGISTER_HTTP_POOL(pool)
 
+        # TODO default_connection_handlers should really be an object, not a
+        # dict, so we can ask "is this using Tor" without poking at a
+        # dictionary with arbitrary strings... See
+        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032
+        handler = self._default_connection_handlers["tcp"]
+
+        if handler == "tcp":
+            return Agent(reactor, tls_context_factory, pool=pool)
+        if handler == "tor":  # TODO or nurl.scheme == "pb+tor":
+            assert self._tor_provider is not None
+            if self._tor_instance is None:
+                self._tor_instance = await self._tor_provider.get_tor_instance(reactor)
+            return self._tor_instance.web_agent(
+                pool=pool, tls_context_factory=tls_context_factory
+            )
+        else:
+            raise RuntimeError(f"Unsupported tcp connection handler: {handler}")
+
+    async def create_storage_client(
+        self,
+        nurl: DecodedURL,
+        reactor: IReactorTime,
+        pool: Optional[HTTPConnectionPool] = None,
+    ) -> StorageClient:
+        """Create a new ``StorageClient`` for the given NURL."""
         assert nurl.fragment == "v=1"
-        assert nurl.scheme == "pb"
-        swissnum = nurl.path[0].encode("ascii")
-        certificate_hash = nurl.user.encode("ascii")
+        assert nurl.scheme in ("pb", "pb+tor")
         if pool is None:
             pool = HTTPConnectionPool(reactor)
             pool.maxPersistentPerHost = 10
 
-        if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
-            cls.TEST_MODE_REGISTER_HTTP_POOL(pool)
-
-        def default_agent_factory(
-            reactor: object,
-            tls_context_factory: IPolicyForHTTPS,
-            pool: HTTPConnectionPool,
-        ) -> IAgent:
-            return Agent(reactor, tls_context_factory, pool=pool)
-
-        if agent_factory is None:
-            agent_factory = default_agent_factory
-
-        treq_client = HTTPClient(
-            agent_factory(
-                reactor,
-                _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
-                pool,
-            )
+        certificate_hash = nurl.user.encode("ascii")
+        agent = await self._create_agent(
+            nurl,
+            reactor,
+            _StorageClientHTTPSPolicy(expected_spki_hash=certificate_hash),
+            pool,
+        )
+        treq_client = HTTPClient(agent)
+        https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
+        swissnum = nurl.path[0].encode("ascii")
+        return StorageClient(
+            https_url,
+            swissnum,
+            treq_client,
+            pool,
+            reactor,
+            self.TEST_MODE_REGISTER_HTTP_POOL is not None,
         )
 
-        https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
-        return cls(https_url, swissnum, treq_client, pool, reactor)
+
+@define(hash=True)
+class StorageClient(object):
+    """
+    Low-level HTTP client that talks to the HTTP storage server.
+
+    Create using a ``StorageClientFactory`` instance.
+    """
+
+    # The URL should be a HTTPS URL ("https://...")
+    _base_url: DecodedURL
+    _swissnum: bytes
+    _treq: Union[treq, StubTreq, HTTPClient]
+    _pool: HTTPConnectionPool
+    _clock: IReactorTime
+    # Are we running unit tests?
+    _test_mode: bool
 
     def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
@@ -495,12 +535,11 @@ class StorageClient(object):
             method, url, headers=headers, timeout=timeout, **kwargs
         )
 
-        if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
-            if response.code != 404:
-                # We're doing API queries, HTML is never correct except in 404, but
-                # it's the default for Twisted's web server so make sure nothing
-                # unexpected happened.
-                assert get_content_type(response.headers) != "text/html"
+        if self._test_mode and response.code != 404:
+            # We're doing API queries, HTML is never correct except in 404, but
+            # it's the default for Twisted's web server so make sure nothing
+            # unexpected happened.
+            assert get_content_type(response.headers) != "text/html"
 
         return response
 
@@ -529,8 +568,7 @@ class StorageClient(object):
 
     def shutdown(self) -> Deferred:
         """Shutdown any connections."""
-        if self._pool is not None:
-            return self._pool.closeCachedConnections()
+        return self._pool.closeCachedConnections()
 
 
 @define(hash=True)
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 2ea154263..6a965aaac 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -89,7 +89,8 @@ from allmydata.util.deferredutil import async_to_deferred, race
 from allmydata.storage.http_client import (
     StorageClient, StorageClientImmutables, StorageClientGeneral,
     ClientException as HTTPClientException, StorageClientMutables,
-    ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException
+    ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException,
+    StorageClientFactory
 )
 from .node import _Config
 
@@ -1068,8 +1069,9 @@ class HTTPNativeStorageServer(service.MultiService):
         self._on_status_changed = ObserverList()
         self._reactor = reactor
         self._grid_manager_verifier = grid_manager_verifier
-        self._tor_provider = tor_provider
-        self._default_connection_handlers = default_connection_handlers
+        self._storage_client_factory = StorageClientFactory(
+            default_connection_handlers, tor_provider
+        )
 
         furl = announcement["anonymous-storage-FURL"].encode("utf-8")
         (
@@ -1232,26 +1234,6 @@ class HTTPNativeStorageServer(service.MultiService):
         self._connecting_deferred = connecting
         return connecting
 
-    async def _agent_factory(self) -> Optional[Callable[[object, IPolicyForHTTPS, HTTPConnectionPool],IAgent]]:
-        """Return a factory for ``twisted.web.iweb.IAgent``."""
-        # TODO default_connection_handlers should really be an object, not a
-        # dict, so we can ask "is this using Tor" without poking at a
-        # dictionary with arbitrary strings... See
-        # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4032
-        handler = self._default_connection_handlers["tcp"]
-        if handler == "tcp":
-            return None
-        if handler == "tor":
-            assert self._tor_provider is not None
-            tor_instance = await self._tor_provider.get_tor_instance(self._reactor)
-
-            def agent_factory(reactor: object, tls_context_factory: IPolicyForHTTPS, pool: HTTPConnectionPool) -> IAgent:
-                assert reactor == self._reactor
-                return tor_instance.web_agent(pool=pool, tls_context_factory=tls_context_factory)
-            return agent_factory
-        else:
-            raise RuntimeError(f"Unsupported tcp connection handler: {handler}")
-
     @async_to_deferred
     async def _pick_server_and_get_version(self):
         """
@@ -1270,28 +1252,24 @@ class HTTPNativeStorageServer(service.MultiService):
             # version() calls before we are live talking to a server, it could only
             # be one. See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3992
 
-            agent_factory = await self._agent_factory()
-
-            def request(reactor, nurl: DecodedURL):
+            @async_to_deferred
+            async def request(reactor, nurl: DecodedURL):
                 # Since we're just using this one off to check if the NURL
                 # works, no need for persistent pool or other fanciness.
                 pool = HTTPConnectionPool(reactor, persistent=False)
                 pool.retryAutomatically = False
-                return StorageClientGeneral(
-                    StorageClient.from_nurl(
-                        nurl, reactor, self._default_connection_handlers,
-                        pool=pool, agent_factory=agent_factory)
-                ).get_version()
+                storage_client = await self._storage_client_factory.create_storage_client(
+                    nurl, reactor, pool
+                )
+                return await StorageClientGeneral(storage_client).get_version()
 
             nurl = await _pick_a_http_server(reactor, self._nurls, request)
 
             # If we've gotten this far, we've found a working NURL.
-            self._istorage_server = _HTTPStorageServer.from_http_client(
-                StorageClient.from_nurl(
-                    nurl, reactor, self._default_connection_handlers,
-                    agent_factory=agent_factory
-                )
+            storage_client = await self._storage_client_factory.create_storage_client(
+                    nurl, reactor, None
             )
+            self._istorage_server = _HTTPStorageServer.from_http_client(storage_client)
             return self._istorage_server
 
         try:
diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py
index fa8d943e5..cfb6c9f04 100644
--- a/src/allmydata/test/common_system.py
+++ b/src/allmydata/test/common_system.py
@@ -686,8 +686,8 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
     def setUp(self):
         self._http_client_pools = []
-        http_client.StorageClient.start_test_mode(self._got_new_http_connection_pool)
-        self.addCleanup(http_client.StorageClient.stop_test_mode)
+        http_client.StorageClientFactory.start_test_mode(self._got_new_http_connection_pool)
+        self.addCleanup(http_client.StorageClientFactory.stop_test_mode)
         self.port_assigner = SameProcessStreamEndpointAssigner()
         self.port_assigner.setUp()
         self.addCleanup(self.port_assigner.tearDown)
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 1380ab7e7..233d82989 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -58,6 +58,7 @@ from ..storage.http_server import (
 )
 from ..storage.http_client import (
     StorageClient,
+    StorageClientFactory,
     ClientException,
     StorageClientImmutables,
     ImmutableCreateResult,
@@ -323,10 +324,10 @@ class CustomHTTPServerTests(SyncTestCase):
 
     def setUp(self):
         super(CustomHTTPServerTests, self).setUp()
-        StorageClient.start_test_mode(
+        StorageClientFactory.start_test_mode(
             lambda pool: self.addCleanup(pool.closeCachedConnections)
         )
-        self.addCleanup(StorageClient.stop_test_mode)
+        self.addCleanup(StorageClientFactory.stop_test_mode)
         # Could be a fixture, but will only be used in this test class so not
         # going to bother:
         self._http_server = TestApp()
@@ -341,6 +342,7 @@ class CustomHTTPServerTests(SyncTestCase):
             # fixed if https://github.com/twisted/treq/issues/226 were ever
             # fixed.
             clock=treq._agent._memoryReactor,
+            test_mode=True,
         )
         self._http_server.clock = self.client._clock
 
@@ -529,10 +531,10 @@ class HttpTestFixture(Fixture):
     """
 
     def _setUp(self):
-        StorageClient.start_test_mode(
+        StorageClientFactory.start_test_mode(
             lambda pool: self.addCleanup(pool.closeCachedConnections)
         )
-        self.addCleanup(StorageClient.stop_test_mode)
+        self.addCleanup(StorageClientFactory.stop_test_mode)
         self.clock = Reactor()
         self.tempdir = self.useFixture(TempDir())
         # The global Cooperator used by Twisted (a) used by pull producers in
@@ -558,6 +560,7 @@ class HttpTestFixture(Fixture):
             treq=self.treq,
             pool=None,
             clock=self.clock,
+            test_mode=True,
         )
 
     def result_of_with_flush(self, d):
@@ -671,6 +674,7 @@ class GenericHTTPAPITests(SyncTestCase):
                 treq=StubTreq(self.http.http_server.get_resource()),
                 pool=None,
                 clock=self.http.clock,
+                test_mode=True,
             )
         )
         with assert_fails_with_http_code(self, http.UNAUTHORIZED):

From 74a121da74af8928cb212e550869fc4c7d511cde Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 11:47:36 -0400
Subject: [PATCH 312/362] Fix bug which meant object could not be created.

---
 src/allmydata/storage/http_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index fe2545c03..8cf79843f 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -330,9 +330,9 @@ class StorageClientFactory:
 
     # If set, we're doing unit testing and we should call this with any
     # HTTPConnectionPool that gets passed/created to ``create_agent()``.
-    TEST_MODE_REGISTER_HTTP_POOL = ClassVar[
+    TEST_MODE_REGISTER_HTTP_POOL: ClassVar[
         Optional[Callable[[HTTPConnectionPool], None]]
-    ]
+    ] = None
 
     @classmethod
     def start_test_mode(cls, callback: Callable[[HTTPConnectionPool], None]) -> None:

From e8744f91e5176d6adf2f1fc4c2065f7041162573 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 12:06:51 -0400
Subject: [PATCH 313/362] Hook up HTTP storage for servers listening on .onion
 addresses

---
 src/allmydata/protocol_switch.py     | 14 +++++++++++---
 src/allmydata/storage/http_client.py |  2 +-
 src/allmydata/storage/http_server.py | 11 +++++++++--
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py
index 208efec6c..941b104be 100644
--- a/src/allmydata/protocol_switch.py
+++ b/src/allmydata/protocol_switch.py
@@ -102,8 +102,15 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
         for location_hint in chain.from_iterable(
             hints.split(",") for hints in cls.tub.locationHints
         ):
-            if location_hint.startswith("tcp:"):
-                _, hostname, port = location_hint.split(":")
+            if location_hint.startswith("tcp:") or location_hint.startswith("tor:"):
+                scheme, hostname, port = location_hint.split(":")
+                if scheme == "tcp":
+                    subscheme = None
+                else:
+                    subscheme = "tor"
+                    # If we're listening on Tor, the hostname needs to have an
+                    # .onion TLD.
+                    assert hostname.endswith(".onion")
                 port = int(port)
                 storage_nurls.add(
                     build_nurl(
@@ -111,9 +118,10 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
                         port,
                         str(swissnum, "ascii"),
                         cls.tub.myCertificate.original.to_cryptography(),
+                        subscheme
                     )
                 )
-            # TODO this is probably where we'll have to support Tor and I2P?
+            # TODO this is where we'll have to support Tor and I2P as well.
             # See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3888#comment:9
             # for discussion (there will be separate tickets added for those at
             # some point.)
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 8cf79843f..cd3143924 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -369,7 +369,7 @@ class StorageClientFactory:
 
         if handler == "tcp":
             return Agent(reactor, tls_context_factory, pool=pool)
-        if handler == "tor":  # TODO or nurl.scheme == "pb+tor":
+        if handler == "tor" or nurl.scheme == "pb+tor":
             assert self._tor_provider is not None
             if self._tor_instance is None:
                 self._tor_instance = await self._tor_provider.get_tor_instance(reactor)
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 924ae5a43..028ebf1c7 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -995,13 +995,20 @@ class _TLSEndpointWrapper(object):
 
 
 def build_nurl(
-    hostname: str, port: int, swissnum: str, certificate: CryptoCertificate
+    hostname: str,
+    port: int,
+    swissnum: str,
+    certificate: CryptoCertificate,
+    subscheme: Optional[str] = None,
 ) -> DecodedURL:
     """
     Construct a HTTPS NURL, given the hostname, port, server swissnum, and x509
     certificate for the server.  Clients can then connect to the server using
     this NURL.
     """
+    scheme = "pb"
+    if subscheme is not None:
+        scheme = f"{scheme}+{subscheme}"
     return DecodedURL().replace(
         fragment="v=1",  # how we know this NURL is HTTP-based (i.e. not Foolscap)
         host=hostname,
@@ -1013,7 +1020,7 @@ def build_nurl(
                 "ascii",
             ),
         ),
-        scheme="pb",
+        scheme=scheme,
     )
 
 

From 57a6721670da156bbf94b72c0b904f04633f6757 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 12:07:13 -0400
Subject: [PATCH 314/362] News file.

---
 newsfragments/3910.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3910.minor

diff --git a/newsfragments/3910.minor b/newsfragments/3910.minor
new file mode 100644
index 000000000..e69de29bb

From a977180baf0b138c8dd3824d95ae3f66bec1540d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 12:15:31 -0400
Subject: [PATCH 315/362] Fix lint

---
 src/allmydata/storage_client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 6a965aaac..a614c17db 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -51,7 +51,6 @@ from zope.interface import (
 )
 from twisted.python.failure import Failure
 from twisted.web import http
-from twisted.web.iweb import IAgent, IPolicyForHTTPS
 from twisted.internet.task import LoopingCall
 from twisted.internet import defer, reactor
 from twisted.application import service

From 20d4175abcbe2948f6e219fd4dd564e7fea47ae3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Tue, 6 Jun 2023 12:18:02 -0400
Subject: [PATCH 316/362] Fix typecheck complaint

---
 src/allmydata/storage/http_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index cd3143924..e7df3709d 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -67,7 +67,7 @@ try:
     from txtorcon import Tor  # type: ignore
 except ImportError:
 
-    class Tor:
+    class Tor:  # type: ignore[no-redef]
         pass
 
 

From 4b495bbe854237b97fc87036268fdbd4265d4a6a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 12 Jun 2023 09:54:45 -0400
Subject: [PATCH 317/362] Slightly improved logging.

---
 integration/util.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index 402c14932..b9f784af9 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -140,7 +140,8 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def outReceived(self, data):
         data = str(data, sys.stdout.encoding)
-        sys.stdout.write(self.name + data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")
         self._output.write(data)
         if not self.magic_seen.called and self._magic_text in self._output.getvalue():
             print("Saw '{}' in the logs".format(self._magic_text))
@@ -148,7 +149,8 @@ class _MagicTextProtocol(ProcessProtocol):
 
     def errReceived(self, data):
         data = str(data, sys.stderr.encoding)
-        sys.stdout.write(self.name + data)
+        for line in data.splitlines():
+            sys.stdout.write(self.name + line + "\n")
 
 
 def _cleanup_process_async(transport: IProcessTransport, allow_missing: bool) -> None:

From eba1ed02269836457e663ace391a035286aa2187 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 12 Jun 2023 12:00:05 -0400
Subject: [PATCH 318/362] More isolated test setup.

---
 integration/test_tor.py | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index ec5cc1bc4..c14cb717e 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -38,8 +38,8 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
     The two nodes can talk to the introducer and each other: we upload to one
     node, read from the other.
     """
-    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
-    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl)
+    carol = yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
+    dave = yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, tor_network, tor_introducer_furl, 2)
     yield util.await_client_ready(carol, minimum_number_of_servers=2, timeout=600)
     yield util.await_client_ready(dave, minimum_number_of_servers=2, timeout=600)
     yield upload_to_one_download_from_the_other(reactor, temp_dir, carol, dave)
@@ -94,7 +94,7 @@ async def upload_to_one_download_from_the_other(reactor, temp_dir, upload_to: ut
 
 
 @pytest_twisted.inlineCallbacks
-def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl) -> util.TahoeProcess:
+def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl, shares_total: int) -> util.TahoeProcess:
     node_dir = FilePath(temp_dir).child(name)
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
@@ -116,7 +116,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 '--listen', 'tor',
                 '--shares-needed', '1',
                 '--shares-happy', '1',
-                '--shares-total', '2',
+                '--shares-total', str(shares_total),
                 node_dir.path,
             ),
             env=environ,
@@ -141,16 +141,23 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
 
 @pytest_twisted.inlineCallbacks
-def test_anonymous_client(reactor, alice, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
+def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
     """
-    A normal node (alice) and a normal introducer are configured, and one node
+    A normal node (normie) and a normal introducer are configured, and one node
     (anonymoose) which is configured to be anonymous by talking via Tor.
 
-    Anonymoose should be able to communicate with alice.
+    Anonymoose should be able to communicate with normie.
 
     TODO how to ensure that anonymoose is actually using Tor?
     """
-    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl)
-    yield util.await_client_ready(anonymoose, minimum_number_of_servers=2, timeout=600)
+    normie = yield util._create_node(
+        reactor, request, temp_dir, introducer_furl, flog_gatherer, "normie",
+        web_port="tcp:9989:interface=localhost",
+        storage=True, needed=1, happy=1, total=1,
+    )
+    yield util.await_client_ready(normie)
 
-    yield upload_to_one_download_from_the_other(reactor, temp_dir, alice, anonymoose)
+    anonymoose = yield _create_anonymous_node(reactor, 'anonymoose', 8008, request, temp_dir, flog_gatherer, tor_network, introducer_furl, 1)
+    yield util.await_client_ready(anonymoose, minimum_number_of_servers=1, timeout=600)
+
+    yield upload_to_one_download_from_the_other(reactor, temp_dir, normie, anonymoose)

From 939f0ded25ab83b0f25cc97994d75fc1135d46a8 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 12 Jun 2023 12:00:12 -0400
Subject: [PATCH 319/362] It's OK if some nodes are down.

---
 integration/util.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/integration/util.py b/integration/util.py
index b9f784af9..31d351bc1 100644
--- a/integration/util.py
+++ b/integration/util.py
@@ -631,16 +631,9 @@ def await_client_ready(tahoe, timeout=10, liveness=60*2, minimum_number_of_serve
             server['last_received_data']
             for server in servers
         ]
-        # if any times are null/None that server has never been
-        # contacted (so it's down still, probably)
-        never_received_data = server_times.count(None)
-        if never_received_data > 0:
-            print(f"waiting because {never_received_data} server(s) not contacted")
-            time.sleep(1)
-            continue
-
-        # check that all times are 'recent enough'
-        if any([time.time() - t > liveness for t in server_times]):
+        # check that all times are 'recent enough' (it's OK if _some_ servers
+        # are down, we just want to make sure a sufficient number are up)
+        if len([time.time() - t <= liveness for t in server_times if t is not None]) < minimum_number_of_servers:
             print("waiting because at least one server too old")
             time.sleep(1)
             continue

From e8150015ad4a555c564fcc3a3702f0e9435cc41a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 12 Jun 2023 12:34:22 -0400
Subject: [PATCH 320/362] Try newer Python in the hopes this will speed things
 up.

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dc9854ae4..1061657b9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -166,7 +166,7 @@ jobs:
       matrix:
         include:
           - os: macos-12
-            python-version: "3.9"
+            python-version: "3.11"
             force-foolscap: false
           - os: windows-latest
             python-version: "3.11"

From 7ff20a34e0d5a52092d88146122c5db626a33140 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 12 Jun 2023 13:22:45 -0400
Subject: [PATCH 321/362] Skip on macOS :(

---
 integration/test_tor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index c14cb717e..af83e2ba1 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -139,7 +139,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
     print("okay, launched")
     return result
 
-
+@pytest.mark.skipif(sys.platform.startswith('darwin'), reason='This test has issues on macOS')
 @pytest_twisted.inlineCallbacks
 def test_anonymous_client(reactor, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
     """

From e5b6049329a7eb22e1f3bc56a849bead18ffaa59 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 13 Jun 2023 10:16:50 -0400
Subject: [PATCH 322/362] match the package name on pypi, case and all

otherwise urls are misconstructed and stuff fails
---
 nix/pyopenssl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index fbf377fa7..14155491c 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -1,6 +1,6 @@
 { pyopenssl, fetchPypi, isPyPy }:
 pyopenssl.overrideAttrs (old: rec {
-  pname = "pyopenssl";
+  pname = "pyOpenSSL";
   version = "23.2.0";
   src = fetchPypi {
     inherit pname version;

From 608fbce9f9be25f5f770b618b210864f47b5f4b1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 13 Jun 2023 10:18:56 -0400
Subject: [PATCH 323/362] match the source tarball hash

---
 nix/pyopenssl.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index 14155491c..e2a23f38c 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -4,7 +4,7 @@ pyopenssl.overrideAttrs (old: rec {
   version = "23.2.0";
   src = fetchPypi {
     inherit pname version;
-    sha256 = "1qgarxcmlrrrlyjnsry47lz04z8bviy7rrlbbp9874kdj799rckc";
+    sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";
   };
   # Building the docs requires sphinx which brings in a dependency on babel,
   # the test suite of which fails.

From 8421d406e9b246b5ce14809a408841f9f7c2537e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 13 Jun 2023 10:33:54 -0400
Subject: [PATCH 324/362] Fix the name metadata as well

It was already computed for the derivation we're going to override.  It won't
be recomputed again as a result of `overrideAttrs` so we recompute it and
include it in the override.
---
 nix/pyopenssl.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index e2a23f38c..3428b0eb7 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -2,6 +2,7 @@
 pyopenssl.overrideAttrs (old: rec {
   pname = "pyOpenSSL";
   version = "23.2.0";
+  name = "${pname}-${version}";
   src = fetchPypi {
     inherit pname version;
     sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";

From 0b0e5c5c93243b79a8f6e70f5b9ed9ca7c8f2020 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 13 Jun 2023 10:34:36 -0400
Subject: [PATCH 325/362] Keep using our `dontBuildDocs` helper function

It does the necessary overrides for stopping doc builds and excluding certain
inputs and outputs.  We can't just set `dontBuildDocs` in the derivation
because that's not a setting recognized by the Nixpkgs Python build system.
---
 nix/pyopenssl.nix        | 3 ---
 nix/python-overrides.nix | 8 ++++++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix
index 3428b0eb7..b8966fad1 100644
--- a/nix/pyopenssl.nix
+++ b/nix/pyopenssl.nix
@@ -7,7 +7,4 @@ pyopenssl.overrideAttrs (old: rec {
     inherit pname version;
     sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";
   };
-  # Building the docs requires sphinx which brings in a dependency on babel,
-  # the test suite of which fails.
-  dontBuildDocs = isPyPy;
 })
diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix
index 423297ef1..4a332b3fc 100644
--- a/nix/python-overrides.nix
+++ b/nix/python-overrides.nix
@@ -29,8 +29,12 @@ in {
 
   # Update the version of pyopenssl.
   pyopenssl = self.callPackage ./pyopenssl.nix {
-    # Avoid infinite recursion.
-    inherit (super) pyopenssl;
+    pyopenssl =
+      # Building the docs requires sphinx which brings in a dependency on babel,
+      # the test suite of which fails.
+      onPyPy (dontBuildDocs { sphinx-rtd-theme = null; })
+        # Avoid infinite recursion.
+        super.pyopenssl;
   };
 
   # collections-extended is currently broken for Python 3.11 in nixpkgs but

From 6bc232745afba2c1121ceac0a4d05dfef3eee459 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 09:56:25 -0400
Subject: [PATCH 326/362] News fragment.

---
 newsfragments/4035.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/4035.minor

diff --git a/newsfragments/4035.minor b/newsfragments/4035.minor
new file mode 100644
index 000000000..e69de29bb

From 5561e11cfd3804a080fc9e85ee52fa01a7a11a1d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 10:31:11 -0400
Subject: [PATCH 327/362] Upgrade versions, install dependencies since mypy
 might want them

---
 tox.ini | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tox.ini b/tox.ini
index 2edb15a0b..89dbda748 100644
--- a/tox.ini
+++ b/tox.ini
@@ -121,20 +121,18 @@ commands =
 
 [testenv:typechecks]
 basepython = python3
-skip_install = True
 deps =
-    mypy
-    mypy-zope
+    mypy==1.3.0
+    # When 0.9.2 comes out it will work with 1.3, it's just unreleased at the moment...
+    git+https://github.com/shoobx/mypy-zope
     types-mock
     types-six
     types-PyYAML
     types-pkg_resources
     types-pyOpenSSL
-    git+https://github.com/warner/foolscap
-    # Twisted 21.2.0 introduces some type hints which we are not yet
-    # compatible with.
-    # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3622
-    twisted<21.2.0
+    foolscap
+    # Upgrade when new releases come out:
+    Twisted==22.10.0
 commands = mypy src
 
 

From b45ee20ba8bd9b582ad479b228be47390145d0de Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 11:07:48 -0400
Subject: [PATCH 328/362] MyPy fixes for allmydata.storage.

---
 src/allmydata/storage/http_client.py  | 23 +++++++++++------------
 src/allmydata/storage/http_common.py  |  2 +-
 src/allmydata/storage/http_server.py  | 13 +++++++++----
 src/allmydata/storage/lease.py        |  4 +++-
 src/allmydata/storage/lease_schema.py |  2 +-
 src/allmydata/storage/server.py       |  4 +++-
 6 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 7f53a4378..59213417c 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -34,7 +34,7 @@ from werkzeug.datastructures import Range, ContentRange
 from twisted.web.http_headers import Headers
 from twisted.web import http
 from twisted.web.iweb import IPolicyForHTTPS, IResponse, IAgent
-from twisted.internet.defer import inlineCallbacks, Deferred, succeed
+from twisted.internet.defer import Deferred, succeed
 from twisted.internet.interfaces import (
     IOpenSSLClientConnectionCreator,
     IReactorTime,
@@ -70,7 +70,6 @@ except ImportError:
         pass
 
 
-
 def _encode_si(si):  # type: (bytes) -> str
     """Encode the storage index into Unicode string."""
     return str(si_b2a(si), "ascii")
@@ -179,24 +178,24 @@ def limited_content(
     This will time out if no data is received for 60 seconds; so long as a
     trickle of data continues to arrive, it will continue to run.
     """
-    d = succeed(None)
+    result_deferred = succeed(None)
 
     # Sadly, addTimeout() won't work because we need access to the IDelayedCall
     # in order to reset it on each data chunk received.
-    timeout = clock.callLater(60, d.cancel)
+    timeout = clock.callLater(60, result_deferred.cancel)
     collector = _LengthLimitedCollector(max_length, timeout)
 
     with start_action(
         action_type="allmydata:storage:http-client:limited-content",
         max_length=max_length,
     ).context():
-        d = DeferredContext(d)
+        d = DeferredContext(result_deferred)
 
     # Make really sure everything gets called in Deferred context, treq might
     # call collector directly...
     d.addCallback(lambda _: treq.collect(response, collector))
 
-    def done(_):
+    def done(_: object) -> BytesIO:
         timeout.cancel()
         collector.f.seek(0)
         return collector.f
@@ -659,15 +658,15 @@ class UploadProgress(object):
     required: RangeMap
 
 
-@inlineCallbacks
-def read_share_chunk(
+@async_to_deferred
+async def read_share_chunk(
     client: StorageClient,
     share_type: str,
     storage_index: bytes,
     share_number: int,
     offset: int,
     length: int,
-) -> Deferred[bytes]:
+) -> bytes:
     """
     Download a chunk of data from a share.
 
@@ -688,7 +687,7 @@ def read_share_chunk(
     # The default 60 second timeout is for getting the response, so it doesn't
     # include the time it takes to download the body... so we will will deal
     # with that later, via limited_content().
-    response = yield client.request(
+    response = await client.request(
         "GET",
         url,
         headers=Headers(
@@ -725,7 +724,7 @@ def read_share_chunk(
             raise ValueError("Server sent more than we asked for?!")
         # It might also send less than we asked for. That's (probably) OK, e.g.
         # if we went past the end of the file.
-        body = yield limited_content(response, client._clock, supposed_length)
+        body = await limited_content(response, client._clock, supposed_length)
         body.seek(0, SEEK_END)
         actual_length = body.tell()
         if actual_length != supposed_length:
@@ -751,7 +750,7 @@ async def advise_corrupt_share(
     storage_index: bytes,
     share_number: int,
     reason: str,
-):
+) -> None:
     assert isinstance(reason, str)
     url = client.relative_url(
         "/storage/v1/{}/{}/{}/corrupt".format(
diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py
index e5f07898e..f16a16785 100644
--- a/src/allmydata/storage/http_common.py
+++ b/src/allmydata/storage/http_common.py
@@ -22,7 +22,7 @@ def get_content_type(headers: Headers) -> Optional[str]:
 
     Returns ``None`` if no content-type was set.
     """
-    values = headers.getRawHeaders("content-type") or [None]
+    values = headers.getRawHeaders("content-type", [None]) or [None]
     content_type = parse_options_header(values[0])[0] or None
     return content_type
 
diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py
index 028ebf1c7..c63a4ca08 100644
--- a/src/allmydata/storage/http_server.py
+++ b/src/allmydata/storage/http_server.py
@@ -386,13 +386,16 @@ class _ReadRangeProducer:
     a request.
     """
 
-    request: Request
+    request: Optional[Request]
     read_data: ReadData
-    result: Deferred
+    result: Optional[Deferred[bytes]]
     start: int
     remaining: int
 
     def resumeProducing(self):
+        if self.result is None or self.request is None:
+            return
+
         to_read = min(self.remaining, 65536)
         data = self.read_data(self.start, to_read)
         assert len(data) <= to_read
@@ -441,7 +444,7 @@ class _ReadRangeProducer:
 
 def read_range(
     request: Request, read_data: ReadData, share_length: int
-) -> Union[Deferred, bytes]:
+) -> Union[Deferred[bytes], bytes]:
     """
     Read an optional ``Range`` header, reads data appropriately via the given
     callable, writes the data to the request.
@@ -478,6 +481,8 @@ def read_range(
         raise _HTTPError(http.REQUESTED_RANGE_NOT_SATISFIABLE)
 
     offset, end = range_header.ranges[0]
+    assert end is not None  # should've exited in block above this if so
+
     # If we're being ask to read beyond the length of the share, just read
     # less:
     end = min(end, share_length)
@@ -496,7 +501,7 @@ def read_range(
         ContentRange("bytes", offset, end).to_header(),
     )
 
-    d = Deferred()
+    d: Deferred[bytes] = Deferred()
     request.registerProducer(
         _ReadRangeProducer(
             request, read_data_with_error_handling, d, offset, end - offset
diff --git a/src/allmydata/storage/lease.py b/src/allmydata/storage/lease.py
index c056a7d28..c0d11abfd 100644
--- a/src/allmydata/storage/lease.py
+++ b/src/allmydata/storage/lease.py
@@ -173,7 +173,9 @@ class LeaseInfo(object):
         """
         return attr.assoc(
             self,
-            _expiration_time=new_expire_time,
+            # MyPy is unhappy with this; long-term solution is likely switch to
+            # new @frozen attrs API, with type annotations.
+            _expiration_time=new_expire_time,  # type: ignore[call-arg]
         )
 
     def is_renew_secret(self, candidate_secret):
diff --git a/src/allmydata/storage/lease_schema.py b/src/allmydata/storage/lease_schema.py
index 63d3d4ed8..ba7dc991a 100644
--- a/src/allmydata/storage/lease_schema.py
+++ b/src/allmydata/storage/lease_schema.py
@@ -56,7 +56,7 @@ class HashedLeaseSerializer(object):
         """
         Hash a lease secret for storage.
         """
-        return blake2b(secret, digest_size=32, encoder=RawEncoder())
+        return blake2b(secret, digest_size=32, encoder=RawEncoder)
 
     @classmethod
     def _hash_lease_info(cls, lease_info):
diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index 6099636f8..d805df1c1 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -55,7 +55,9 @@ class StorageServer(service.MultiService):
     """
     Implement the business logic for the storage server.
     """
-    name = 'storage'
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = 'storage'  # type: ignore
     # only the tests change this to anything else
     LeaseCheckerClass = LeaseCheckingCrawler
 

From 8493b42024c3e22438605b446a544747b27c8c6c Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:02:24 -0400
Subject: [PATCH 329/362] Fix types.

---
 src/allmydata/test/cli/wormholetesting.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 647798bc8..91849901a 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -63,7 +63,7 @@ class MemoryWormholeServer(object):
         specific application id and relay URL combination.
     """
     _apps: dict[ApplicationKey, _WormholeApp] = field(default=Factory(dict))
-    _waiters: dict[ApplicationKey, Deferred] = field(default=Factory(dict))
+    _waiters: dict[ApplicationKey, Deferred[IWormhole]] = field(default=Factory(dict))
 
     def create(
         self,
@@ -130,7 +130,7 @@ class TestingHelper(object):
         key = (relay_url, appid)
         if key in self._server._waiters:
             raise ValueError(f"There is already a waiter for {key}")
-        d = Deferred()
+        d : Deferred[IWormhole] = Deferred()
         self._server._waiters[key] = d
         wormhole = await d
         return wormhole
@@ -166,7 +166,7 @@ class _WormholeApp(object):
     appid/relay_url scope.
     """
     wormholes: dict[WormholeCode, IWormhole] = field(default=Factory(dict))
-    _waiting: dict[WormholeCode, List[Deferred]] = field(default=Factory(dict))
+    _waiting: dict[WormholeCode, List[Deferred[_MemoryWormhole]]] = field(default=Factory(dict))
     _counter: Iterator[int] = field(default=Factory(count))
 
     def allocate_code(self, wormhole: IWormhole, code: Optional[WormholeCode]) -> WormholeCode:
@@ -192,13 +192,13 @@ class _WormholeApp(object):
 
         return code
 
-    def wait_for_wormhole(self, code: WormholeCode) -> Awaitable[_MemoryWormhole]:
+    def wait_for_wormhole(self, code: WormholeCode) -> Deferred[_MemoryWormhole]:
         """
         Return a ``Deferred`` which fires with the next wormhole to be associated
         with the given code.  This is used to let the first end of a wormhole
         rendezvous with the second end.
         """
-        d = Deferred()
+        d : Deferred[_MemoryWormhole] = Deferred()
         self._waiting.setdefault(code, []).append(d)
         return d
 
@@ -242,8 +242,8 @@ class _MemoryWormhole(object):
 
     _view: _WormholeServerView
     _code: Optional[WormholeCode] = None
-    _payload: DeferredQueue = field(default=Factory(DeferredQueue))
-    _waiting_for_code: list[Deferred] = field(default=Factory(list))
+    _payload: DeferredQueue[WormholeMessage] = field(default=Factory(DeferredQueue))
+    _waiting_for_code: list[Deferred[WormholeCode]] = field(default=Factory(list))
 
     def allocate_code(self) -> None:
         if self._code is not None:
@@ -265,7 +265,7 @@ class _MemoryWormhole(object):
 
     def when_code(self) -> Deferred[WormholeCode]:
         if self._code is None:
-            d = Deferred()
+            d : Deferred[WormholeCode] = Deferred()
             self._waiting_for_code.append(d)
             return d
         return succeed(self._code)

From 257aa289cdd5a7954261a03e93725394afbe8ca0 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:04:45 -0400
Subject: [PATCH 330/362] Remote interfaces don't interact well with mypy.

---
 src/allmydata/introducer/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py
index 07f8a5f7a..a64596f0e 100644
--- a/src/allmydata/introducer/client.py
+++ b/src/allmydata/introducer/client.py
@@ -35,7 +35,7 @@ class InvalidCacheError(Exception):
 
 V2 = b"http://allmydata.org/tahoe/protocols/introducer/v2"
 
-@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient)
+@implementer(RIIntroducerSubscriberClient_v2, IIntroducerClient)  # type: ignore[misc]
 class IntroducerClient(service.Service, Referenceable):
 
     def __init__(self, tub, introducer_furl,

From 1fd81116cb9b7c29fdf5e3bb89430cfdb79b9f36 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:08:30 -0400
Subject: [PATCH 331/362] Fix mypy complaint.

---
 src/allmydata/protocol_switch.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/protocol_switch.py b/src/allmydata/protocol_switch.py
index 941b104be..6a6bf8061 100644
--- a/src/allmydata/protocol_switch.py
+++ b/src/allmydata/protocol_switch.py
@@ -16,9 +16,10 @@ later in the configuration process.
 from __future__ import annotations
 
 from itertools import chain
+from typing import cast
 
 from twisted.internet.protocol import Protocol
-from twisted.internet.interfaces import IDelayedCall
+from twisted.internet.interfaces import IDelayedCall, IReactorFromThreads
 from twisted.internet.ssl import CertificateOptions
 from twisted.web.server import Site
 from twisted.protocols.tls import TLSMemoryBIOFactory
@@ -89,7 +90,7 @@ class _FoolscapOrHttps(Protocol, metaclass=_PretendToBeNegotiation):
             certificate=cls.tub.myCertificate.original,
         )
 
-        http_storage_server = HTTPServer(reactor, storage_server, swissnum)
+        http_storage_server = HTTPServer(cast(IReactorFromThreads, reactor), storage_server, swissnum)
         cls.https_factory = TLSMemoryBIOFactory(
             certificate_options,
             False,

From 3b5c6695d5fca060ac51575d8b81ac347dac1f18 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:09:51 -0400
Subject: [PATCH 332/362] Pacify mypy.

---
 src/allmydata/testing/web.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py
index 4f68b3774..95e92825b 100644
--- a/src/allmydata/testing/web.py
+++ b/src/allmydata/testing/web.py
@@ -276,6 +276,15 @@ class _SynchronousProducer(object):
         consumer.write(self.body)
         return succeed(None)
 
+    def stopProducing(self):
+        pass
+
+    def pauseProducing(self):
+        pass
+
+    def resumeProducing(self):
+        pass
+
 
 def create_tahoe_treq_client(root=None):
     """

From cab24e4c7b1241c14e2f010e6a7d117755e732e4 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:12:39 -0400
Subject: [PATCH 333/362] Another service name issue.

---
 src/allmydata/immutable/upload.py | 4 +++-
 src/allmydata/storage/server.py   | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index a331cc5db..36bd86fa6 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -1843,7 +1843,9 @@ class Uploader(service.MultiService, log.PrefixingLogMixin):
     """I am a service that allows file uploading. I am a service-child of the
     Client.
     """
-    name = "uploader"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "uploader"  # type: ignore[assignment]
     URI_LIT_SIZE_THRESHOLD = 55
 
     def __init__(self, helper_furl=None, stats_provider=None, history=None):
diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py
index d805df1c1..858b87b1f 100644
--- a/src/allmydata/storage/server.py
+++ b/src/allmydata/storage/server.py
@@ -57,7 +57,7 @@ class StorageServer(service.MultiService):
     """
     # The type in Twisted for services is wrong in 22.10...
     # https://github.com/twisted/twisted/issues/10135
-    name = 'storage'  # type: ignore
+    name = 'storage'  # type: ignore[assignment]
     # only the tests change this to anything else
     LeaseCheckerClass = LeaseCheckingCrawler
 

From 054c893539762449c377866fbc8c204536f9f2c5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:16:10 -0400
Subject: [PATCH 334/362] Pacify mypy

---
 src/allmydata/util/deferredutil.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 695915ceb..9e8d7bad4 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -14,6 +14,7 @@ from typing import (
     TypeVar,
     Optional,
     Coroutine,
+    Generator
 )
 from typing_extensions import ParamSpec
 
@@ -212,7 +213,7 @@ class WaitForDelayedCallsMixin(PollMixin):
 def until(
         action: Callable[[], defer.Deferred[Any]],
         condition: Callable[[], bool],
-) -> defer.Deferred[None]:
+) -> Generator[Any, None, None]:
     """
     Run a Deferred-returning function until a condition is true.
 

From f42fb1e551e6b09ec4fa6812f06423f4afa8c828 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:16:35 -0400
Subject: [PATCH 335/362] Unused import

---
 src/allmydata/test/cli/wormholetesting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/wormholetesting.py b/src/allmydata/test/cli/wormholetesting.py
index 91849901a..3bcad1ebf 100644
--- a/src/allmydata/test/cli/wormholetesting.py
+++ b/src/allmydata/test/cli/wormholetesting.py
@@ -40,7 +40,7 @@ from itertools import count
 from sys import stderr
 
 from attrs import frozen, define, field, Factory
-from twisted.internet.defer import Deferred, DeferredQueue, succeed, Awaitable
+from twisted.internet.defer import Deferred, DeferredQueue, succeed
 from wormhole._interfaces import IWormhole
 from wormhole.wormhole import create
 from zope.interface import implementer

From ff1c1f700ee4d94ec5a3276383420f4690703a2d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:52:54 -0400
Subject: [PATCH 336/362] Remove unused methods.

---
 src/allmydata/interfaces.py | 41 -------------------------------------
 1 file changed, 41 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 201ab082e..1ebc23c75 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -493,47 +493,6 @@ class IStorageBroker(Interface):
         @return: unicode nickname, or None
         """
 
-    # methods moved from IntroducerClient, need review
-    def get_all_connections():
-        """Return a frozenset of (nodeid, service_name, rref) tuples, one for
-        each active connection we've established to a remote service. This is
-        mostly useful for unit tests that need to wait until a certain number
-        of connections have been made."""
-
-    def get_all_connectors():
-        """Return a dict that maps from (nodeid, service_name) to a
-        RemoteServiceConnector instance for all services that we are actively
-        trying to connect to. Each RemoteServiceConnector has the following
-        public attributes::
-
-          service_name: the type of service provided, like 'storage'
-          last_connect_time: when we last established a connection
-          last_loss_time: when we last lost a connection
-
-          version: the peer's version, from the most recent connection
-          oldest_supported: the peer's oldest supported version, same
-
-          rref: the RemoteReference, if connected, otherwise None
-
-        This method is intended for monitoring interfaces, such as a web page
-        that describes connecting and connected peers.
-        """
-
-    def get_all_peerids():
-        """Return a frozenset of all peerids to whom we have a connection (to
-        one or more services) established. Mostly useful for unit tests."""
-
-    def get_all_connections_for(service_name):
-        """Return a frozenset of (nodeid, service_name, rref) tuples, one
-        for each active connection that provides the given SERVICE_NAME."""
-
-    def get_permuted_peers(service_name, key):
-        """Returns an ordered list of (peerid, rref) tuples, selecting from
-        the connections that provide SERVICE_NAME, using a hash-based
-        permutation keyed by KEY. This randomizes the service list in a
-        repeatable way, to distribute load over many peers.
-        """
-
 
 class IDisplayableServer(Interface):
     def get_nickname():

From 65775cd6bde8514fe5d3e0b809f7dd2a0dc50678 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 14:54:24 -0400
Subject: [PATCH 337/362] Not used externally.

---
 src/allmydata/interfaces.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 1ebc23c75..0379e8633 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -510,16 +510,6 @@ class IServer(IDisplayableServer):
     def start_connecting(trigger_cb):
         pass
 
-    def get_rref():
-        """Obsolete.  Use ``get_storage_server`` instead.
-
-        Once a server is connected, I return a RemoteReference.
-        Before a server is connected for the first time, I return None.
-
-        Note that the rref I return will start producing DeadReferenceErrors
-        once the connection is lost.
-        """
-
     def upload_permitted():
         """
         :return: True if we should use this server for uploads, False

From 11e01518388e6844f8ae086456f2cd6e81d08480 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 16:02:04 -0400
Subject: [PATCH 338/362] Fix some type issues in storage_client.py

---
 src/allmydata/interfaces.py          |  7 ++++++-
 src/allmydata/storage/http_client.py |  5 +++--
 src/allmydata/storage_client.py      | 10 ++++++----
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 0379e8633..0f00c5417 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -17,11 +17,13 @@ if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, range, max, min  # noqa: F401
 
 from past.builtins import long
+from typing import Dict
 
 from zope.interface import Interface, Attribute
 from twisted.plugin import (
     IPlugin,
 )
+from twisted.internet.defer import Deferred
 from foolscap.api import StringConstraint, ListOf, TupleOf, SetOf, DictOf, \
      ChoiceOf, IntegerConstraint, Any, RemoteInterface, Referenceable
 
@@ -307,12 +309,15 @@ class RIStorageServer(RemoteInterface):
         store that on disk.
         """
 
+# The result of IStorageServer.get_version():
+VersionMessage = Dict[bytes, object]
+
 
 class IStorageServer(Interface):
     """
     An object capable of storing shares for a storage client.
     """
-    def get_version():
+    def get_version() -> Deferred[VersionMessage]:
         """
         :see: ``RIStorageServer.get_version``
         """
diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 59213417c..9b44d2a73 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -57,6 +57,7 @@ from .http_common import (
     CBOR_MIME_TYPE,
     get_spki_hash,
 )
+from ..interfaces import VersionMessage
 from .common import si_b2a, si_to_human_readable
 from ..util.hashutil import timing_safe_compare
 from ..util.deferredutil import async_to_deferred
@@ -576,7 +577,7 @@ class StorageClientGeneral(object):
     _client: StorageClient
 
     @async_to_deferred
-    async def get_version(self) -> dict[bytes, object]:
+    async def get_version(self) -> VersionMessage:
         """
         Return the version metadata for the server.
         """
@@ -585,7 +586,7 @@ class StorageClientGeneral(object):
         ):
             return await self._get_version()
 
-    async def _get_version(self) -> dict[bytes, object]:
+    async def _get_version(self) -> VersionMessage:
         """Implementation of get_version()."""
         url = self._client.relative_url("/storage/v1/version")
         response = await self._client.request("GET", url)
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index a614c17db..4efc845b4 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -33,7 +33,7 @@ Ported to Python 3.
 from __future__ import annotations
 
 from six import ensure_text
-from typing import Union, Callable, Any, Optional
+from typing import Union, Callable, Any, Optional, cast
 from os import urandom
 import re
 import time
@@ -53,6 +53,7 @@ from twisted.python.failure import Failure
 from twisted.web import http
 from twisted.internet.task import LoopingCall
 from twisted.internet import defer, reactor
+from twisted.internet.interfaces import IReactorTime
 from twisted.application import service
 from twisted.plugin import (
     getPlugins,
@@ -70,6 +71,7 @@ from allmydata.interfaces import (
     IServer,
     IStorageServer,
     IFoolscapStoragePlugin,
+    VersionMessage
 )
 from allmydata.grid_manager import (
     create_grid_manager_verifier,
@@ -1089,7 +1091,7 @@ class HTTPNativeStorageServer(service.MultiService):
         self._connection_status = connection_status.ConnectionStatus.unstarted()
         self._version = None
         self._last_connect_time = None
-        self._connecting_deferred = None
+        self._connecting_deferred : Optional[defer.Deferred[object]]= None
 
     def get_permutation_seed(self):
         return self._permutation_seed
@@ -1266,7 +1268,7 @@ class HTTPNativeStorageServer(service.MultiService):
 
             # If we've gotten this far, we've found a working NURL.
             storage_client = await self._storage_client_factory.create_storage_client(
-                    nurl, reactor, None
+                    nurl, cast(IReactorTime, reactor), None
             )
             self._istorage_server = _HTTPStorageServer.from_http_client(storage_client)
             return self._istorage_server
@@ -1507,7 +1509,7 @@ class _HTTPStorageServer(object):
         """
         return _HTTPStorageServer(http_client=http_client)
 
-    def get_version(self):
+    def get_version(self) -> defer.Deferred[VersionMessage]:
         return StorageClientGeneral(self._http_client).get_version()
 
     @defer.inlineCallbacks

From 55d62d609b286f3778bb290f344ef03c652c1176 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 16:50:02 -0400
Subject: [PATCH 339/362] Fix some mypy errors.

---
 src/allmydata/web/common.py     | 8 ++++++--
 src/allmydata/web/operations.py | 5 +++--
 src/allmydata/webish.py         | 4 +++-
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index bd1e3838e..a8de54c0d 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -86,6 +86,8 @@ from allmydata.util.encodingutil import (
 )
 from allmydata.util import abbreviate
 from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string
+from ..webish import TahoeLAFSRequest
+
 
 class WebError(Exception):
     def __init__(self, text, code=http.BAD_REQUEST):
@@ -723,13 +725,15 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *,
 
     :return: Either bytes or tuple of bytes.
     """
+    # This is not... obvious to callers, let's say, but it does happen.
+    assert isinstance(req, TahoeLAFSRequest)
     if isinstance(argname, str):
         argname_bytes = argname.encode("utf-8")
     else:
         argname_bytes = argname
 
-    results = []
-    if argname_bytes in req.args:
+    results : list[bytes] = []
+    if req.args is not None and argname_bytes in req.args:
         results.extend(req.args[argname_bytes])
     argname_unicode = str(argname_bytes, "utf-8")
     if req.fields and argname_unicode in req.fields:
diff --git a/src/allmydata/web/operations.py b/src/allmydata/web/operations.py
index aedf33f37..a564f8484 100644
--- a/src/allmydata/web/operations.py
+++ b/src/allmydata/web/operations.py
@@ -43,8 +43,9 @@ DAY = 24*HOUR
 
 class OphandleTable(resource.Resource, service.Service):
     """Renders /operations/%d."""
-
-    name = "operations"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "operations"  # type: ignore[assignment]
 
     UNCOLLECTED_HANDLE_LIFETIME = 4*DAY
     COLLECTED_HANDLE_LIFETIME = 1*DAY
diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py
index 1b2b8192a..ec2582f80 100644
--- a/src/allmydata/webish.py
+++ b/src/allmydata/webish.py
@@ -242,7 +242,9 @@ class TahoeLAFSSite(Site, object):
 
 
 class WebishServer(service.MultiService):
-    name = "webish"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "webish"  # type: ignore[assignment]
 
     def __init__(self, client, webport, tempdir, nodeurl_path=None, staticdir=None,
                  clock=None, now_fn=time.time):

From af323d2bbb7753da7ec25c70ecb40f711c071c0b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:05:51 -0400
Subject: [PATCH 340/362] Get the code working again.

---
 src/allmydata/web/common.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index a8de54c0d..0685d60a0 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -86,7 +86,6 @@ from allmydata.util.encodingutil import (
 )
 from allmydata.util import abbreviate
 from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string
-from ..webish import TahoeLAFSRequest
 
 
 class WebError(Exception):
@@ -725,8 +724,11 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *,
 
     :return: Either bytes or tuple of bytes.
     """
-    # This is not... obvious to callers, let's say, but it does happen.
-    assert isinstance(req, TahoeLAFSRequest)
+    # This is not... obvious to callers, let's say, but it does happen in
+    # pretty much all non-test real code. We have to import here to prevent
+    # circular import.
+    from ..webish import TahoeLAFSRequest
+
     if isinstance(argname, str):
         argname_bytes = argname.encode("utf-8")
     else:
@@ -736,7 +738,7 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *,
     if req.args is not None and argname_bytes in req.args:
         results.extend(req.args[argname_bytes])
     argname_unicode = str(argname_bytes, "utf-8")
-    if req.fields and argname_unicode in req.fields:
+    if isinstance(req, TahoeLAFSRequest) and req.fields and argname_unicode in req.fields:
         value = req.fields[argname_unicode].value
         if isinstance(value, str):
             value = value.encode("utf-8")

From 44b752c87d97ba78fce3726e34d9ee0ee7b3fcdf Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:43:39 -0400
Subject: [PATCH 341/362] Fix mypy issues

---
 src/allmydata/frontends/sftpd.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py
index d2d614c77..14f17e12d 100644
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@@ -1925,7 +1925,11 @@ class FakeTransport(object):
     def loseConnection(self):
         logmsg("FakeTransport.loseConnection()", level=NOISY)
 
-    # getPeer and getHost can just raise errors, since we don't know what to return
+    def getHost(self):
+        pass
+
+    def getPeer(self):
+        pass
 
 
 @implementer(ISession)
@@ -1990,15 +1994,18 @@ class Dispatcher(object):
     def __init__(self, client):
         self._client = client
 
-    def requestAvatar(self, avatarID, mind, interface):
+    def requestAvatar(self, avatarId, mind, *interfaces):
+        [interface] = interfaces
         _assert(interface == IConchUser, interface=interface)
-        rootnode = self._client.create_node_from_uri(avatarID.rootcap)
-        handler = SFTPUserHandler(self._client, rootnode, avatarID.username)
+        rootnode = self._client.create_node_from_uri(avatarId.rootcap)
+        handler = SFTPUserHandler(self._client, rootnode, avatarId.username)
         return (interface, handler, handler.logout)
 
 
 class SFTPServer(service.MultiService):
-    name = "frontend:sftp"
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "frontend:sftp"  # type: ignore[assignment]
 
     def __init__(self, client, accountfile,
                  sftp_portstr, pubkey_file, privkey_file):

From 27243ccfdfadd51db8a5d1fef260d6125b10f3bd Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:45:15 -0400
Subject: [PATCH 342/362] Fix mypy issues

---
 src/allmydata/introducer/server.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py
index 5dad89ae8..157a1b73c 100644
--- a/src/allmydata/introducer/server.py
+++ b/src/allmydata/introducer/server.py
@@ -142,9 +142,12 @@ def stringify_remote_address(rref):
     return str(remote)
 
 
+# MyPy doesn't work well with remote interfaces...
 @implementer(RIIntroducerPublisherAndSubscriberService_v2)
-class IntroducerService(service.MultiService, Referenceable):
-    name = "introducer"
+class IntroducerService(service.MultiService, Referenceable):  # type: ignore[misc]
+    # The type in Twisted for services is wrong in 22.10...
+    # https://github.com/twisted/twisted/issues/10135
+    name = "introducer"  # type: ignore[assignment]
     # v1 is the original protocol, added in 1.0 (but only advertised starting
     # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
     # TODO: reconcile bytes/str for keys

From 9306f5edab0ed72e7acde86af6ff2d57fbe378b9 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:48:13 -0400
Subject: [PATCH 343/362] Fix mypy issues

---
 src/allmydata/scripts/admin.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py
index 02fd9a143..be700bcca 100644
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@@ -112,6 +112,9 @@ class AddGridManagerCertOptions(BaseOptions):
         return "Usage: tahoe [global-options] admin add-grid-manager-cert [options]"
 
     def postOptions(self) -> None:
+        assert self.parent is not None
+        assert self.parent.parent is not None
+        
         if self['name'] is None:
             raise usage.UsageError(
                 "Must provide --name option"
@@ -123,8 +126,8 @@ class AddGridManagerCertOptions(BaseOptions):
 
         data: str
         if self['filename'] == '-':
-            print("reading certificate from stdin", file=self.parent.parent.stderr)
-            data = self.parent.parent.stdin.read()
+            print("reading certificate from stdin", file=self.parent.parent.stderr)  # type: ignore[attr-defined]
+            data = self.parent.parent.stdin.read()  # type: ignore[attr-defined]
             if len(data) == 0:
                 raise usage.UsageError(
                     "Reading certificate from stdin failed"

From ce1839f2033c1700caa6d7f9b1de5ef713b3e142 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:51:54 -0400
Subject: [PATCH 344/362] Pacify mypy

---
 src/allmydata/test/test_storage_http.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 233d82989..2b4023bc5 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -514,8 +514,8 @@ class Reactor(Clock):
         Clock.__init__(self)
         self._queue = Queue()
 
-    def callFromThread(self, f, *args, **kwargs):
-        self._queue.put((f, args, kwargs))
+    def callFromThread(self, callable, *args, **kwargs):
+        self._queue.put((callable, args, kwargs))
 
     def advance(self, *args, **kwargs):
         Clock.advance(self, *args, **kwargs)

From 0f8100b1e9646993a22175a77ca5a0ee1e707cce Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:52:01 -0400
Subject: [PATCH 345/362] Fix whitespace

---
 src/allmydata/scripts/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py
index be700bcca..3acd52267 100644
--- a/src/allmydata/scripts/admin.py
+++ b/src/allmydata/scripts/admin.py
@@ -114,7 +114,7 @@ class AddGridManagerCertOptions(BaseOptions):
     def postOptions(self) -> None:
         assert self.parent is not None
         assert self.parent.parent is not None
-        
+
         if self['name'] is None:
             raise usage.UsageError(
                 "Must provide --name option"

From 96afb0743ac2ad9eb4af3c9ce94fcb8d65ac49ba Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 17:57:25 -0400
Subject: [PATCH 346/362] Pacify mypy

---
 src/allmydata/test/test_consumer.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py
index 234fc2594..ee1908ba7 100644
--- a/src/allmydata/test/test_consumer.py
+++ b/src/allmydata/test/test_consumer.py
@@ -39,6 +39,12 @@ class Producer(object):
         self.consumer = consumer
         self.done = False
 
+    def stopProducing(self):
+        pass
+
+    def pauseProducing(self):
+        pass
+
     def resumeProducing(self):
         """Kick off streaming."""
         self.iterate()

From bf5213cb016923c757780779afc02ac2847d3347 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Wed, 14 Jun 2023 18:01:52 -0400
Subject: [PATCH 347/362] Pacify mypy

---
 src/allmydata/test/mutable/test_version.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/mutable/test_version.py b/src/allmydata/test/mutable/test_version.py
index 87050424b..c91c1d4f1 100644
--- a/src/allmydata/test/mutable/test_version.py
+++ b/src/allmydata/test/mutable/test_version.py
@@ -78,18 +78,21 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
         fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir)))
                         for (i,ss,storedir)
                         in self.iterate_servers()]
-        fso.stdout = StringIO()
-        fso.stderr = StringIO()
+        # This attribute isn't defined on FindSharesOptions but `find_shares()`
+        # definitely expects it...
+        fso.stdout = StringIO()  # type: ignore[attr-defined]
         debug.find_shares(fso)
-        sharefiles = fso.stdout.getvalue().splitlines()
+        sharefiles = fso.stdout.getvalue().splitlines()  # type: ignore[attr-defined]
         expected = self.nm.default_encoding_parameters["n"]
         self.assertThat(sharefiles, HasLength(expected))
 
+        # This attribute isn't defined on DebugOptions but `dump_share()`
+        # definitely expects it...
         do = debug.DumpOptions()
         do["filename"] = sharefiles[0]
-        do.stdout = StringIO()
+        do.stdout = StringIO()  # type: ignore[attr-defined]
         debug.dump_share(do)
-        output = do.stdout.getvalue()
+        output = do.stdout.getvalue()  # type: ignore[attr-defined]
         lines = set(output.splitlines())
         self.assertTrue("Mutable slot found:" in lines, output)
         self.assertTrue(" share_type: MDMF" in lines, output)
@@ -104,10 +107,12 @@ class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
         self.assertTrue("  verify-cap: %s" % vcap in lines, output)
         cso = debug.CatalogSharesOptions()
         cso.nodedirs = fso.nodedirs
-        cso.stdout = StringIO()
-        cso.stderr = StringIO()
+        # Definitely not options on CatalogSharesOptions, but the code does use
+        # stdout and stderr...
+        cso.stdout = StringIO()  # type: ignore[attr-defined]
+        cso.stderr = StringIO()  # type: ignore[attr-defined]
         debug.catalog_shares(cso)
-        shares = cso.stdout.getvalue().splitlines()
+        shares = cso.stdout.getvalue().splitlines()  # type: ignore[attr-defined]
         oneshare = shares[0] # all shares should be MDMF
         self.failIf(oneshare.startswith("UNKNOWN"), oneshare)
         self.assertTrue(oneshare.startswith("MDMF"), oneshare)

From 8d99ddc542825e4e893b92546fe0856bb283b701 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Thu, 15 Jun 2023 17:14:08 -0400
Subject: [PATCH 348/362] Pacify mypy

---
 src/allmydata/test/cli/test_invite.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py
index 31992a54d..1302e5970 100644
--- a/src/allmydata/test/cli/test_invite.py
+++ b/src/allmydata/test/cli/test_invite.py
@@ -8,7 +8,7 @@ import json
 import os
 from functools import partial
 from os.path import join
-from typing import Awaitable, Callable, Optional, Sequence, TypeVar, Union
+from typing import Callable, Optional, Sequence, TypeVar, Union, Coroutine, Any, Tuple, cast, Generator
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -60,7 +60,7 @@ def make_simple_peer(
         server: MemoryWormholeServer,
         helper: TestingHelper,
         messages: Sequence[JSONable],
-) -> Callable[[], Awaitable[IWormhole]]:
+) -> Callable[[], Coroutine[defer.Deferred[IWormhole], Any, IWormhole]]:
     """
     Make a wormhole peer that just sends the given messages.
 
@@ -102,18 +102,24 @@ A = TypeVar("A")
 B = TypeVar("B")
 
 def concurrently(
-    client: Callable[[], Awaitable[A]],
-    server: Callable[[], Awaitable[B]],
-) -> defer.Deferred[tuple[A, B]]:
+    client: Callable[[], Union[
+        Coroutine[defer.Deferred[A], Any, A],
+        Generator[defer.Deferred[A], Any, A],
+    ]],
+    server: Callable[[], Union[
+        Coroutine[defer.Deferred[B], Any, B],
+        Generator[defer.Deferred[B], Any, B],
+    ]],
+) -> defer.Deferred[Tuple[A, B]]:
     """
     Run two asynchronous functions concurrently and asynchronously return a
     tuple of both their results.
     """
-    return defer.gatherResults([
+    result = defer.gatherResults([
         defer.Deferred.fromCoroutine(client()),
         defer.Deferred.fromCoroutine(server()),
-    ])
-
+    ]).addCallback(tuple)  # type: ignore
+    return cast(defer.Deferred[Tuple[A, B]], result)
 
 class Join(GridTestMixin, CLITestMixin, unittest.TestCase):
 

From db9597ee1995a5f20de3a58fe0c5660b693fd8ca Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Mon, 19 Jun 2023 16:07:31 -0600
Subject: [PATCH 349/362] add --allow-stdin-close option

---
 src/allmydata/scripts/tahoe_run.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py
index aaf234b61..d54fe9af3 100644
--- a/src/allmydata/scripts/tahoe_run.py
+++ b/src/allmydata/scripts/tahoe_run.py
@@ -104,6 +104,11 @@ class RunOptions(BasedirOptions):
          " [default: %s]" % quote_local_unicode_path(_default_nodedir)),
         ]
 
+    optFlags = [
+        ("allow-stdin-close", None,
+         'Do not exit when stdin closes ("tahoe run" otherwise will exit).'),
+    ]
+
     def parseArgs(self, basedir=None, *twistd_args):
         # This can't handle e.g. 'tahoe run --reactor=foo', since
         # '--reactor=foo' looks like an option to the tahoe subcommand, not to
@@ -156,6 +161,7 @@ class DaemonizeTheRealService(Service, HookMixin):
             "running": None,
         }
         self.stderr = options.parent.stderr
+        self._close_on_stdin_close = False if options["allow-stdin-close"] else True
 
     def startService(self):
 
@@ -202,7 +208,8 @@ class DaemonizeTheRealService(Service, HookMixin):
                 srv.setServiceParent(self.parent)
                 # exiting on stdin-closed facilitates cleanup when run
                 # as a subprocess
-                on_stdin_close(reactor, reactor.stop)
+                if self._close_on_stdin_close:
+                    on_stdin_close(reactor, reactor.stop)
             d.addCallback(created)
             d.addErrback(handle_config_error)
             d.addBoth(self._call_hook, 'running')
@@ -213,11 +220,13 @@ class DaemonizeTheRealService(Service, HookMixin):
 
 class DaemonizeTahoeNodePlugin(object):
     tapname = "tahoenode"
-    def __init__(self, nodetype, basedir):
+    def __init__(self, nodetype, basedir, allow_stdin_close):
         self.nodetype = nodetype
         self.basedir = basedir
+        self.allow_stdin_close = allow_stdin_close
 
     def makeService(self, so):
+        so["allow-stdin-close"] = self.allow_stdin_close
         return DaemonizeTheRealService(self.nodetype, self.basedir, so)
 
 
@@ -304,7 +313,9 @@ def run(reactor, config, runApp=twistd.runApp):
         print(config, file=err)
         print("tahoe %s: usage error from twistd: %s\n" % (config.subcommand_name, ue), file=err)
         return 1
-    twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)}
+    twistd_config.loadedPlugins = {
+        "DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir, config["allow-stdin-close"])
+    }
 
     # our own pid-style file contains PID and process creation time
     pidfile = FilePath(get_pidfile(config['basedir']))

From a107e163351faa5f62dfc33781d5e12a5781a07e Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Mon, 19 Jun 2023 16:08:17 -0600
Subject: [PATCH 350/362] news

---
 newsfragments/4036.enhancement | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/4036.enhancement

diff --git a/newsfragments/4036.enhancement b/newsfragments/4036.enhancement
new file mode 100644
index 000000000..36c062718
--- /dev/null
+++ b/newsfragments/4036.enhancement
@@ -0,0 +1 @@
+tahoe run now accepts --allow-stdin-close to mean "keep running if stdin closes"
\ No newline at end of file

From 2fcb190c2f5a8550284eaa27a0ee0f8d2f2f068b Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Mon, 19 Jun 2023 17:53:57 -0600
Subject: [PATCH 351/362] add tests for both close-stdin cases

---
 src/allmydata/scripts/tahoe_run.py |  3 +-
 src/allmydata/test/cli/test_run.py | 95 ++++++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py
index d54fe9af3..ff3ff9efd 100644
--- a/src/allmydata/scripts/tahoe_run.py
+++ b/src/allmydata/scripts/tahoe_run.py
@@ -205,7 +205,8 @@ class DaemonizeTheRealService(Service, HookMixin):
             d = service_factory()
 
             def created(srv):
-                srv.setServiceParent(self.parent)
+                if self.parent is not None:
+                    srv.setServiceParent(self.parent)
                 # exiting on stdin-closed facilitates cleanup when run
                 # as a subprocess
                 if self._close_on_stdin_close:
diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index e84f52096..96640d45a 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -31,6 +31,12 @@ from twisted.python.filepath import (
 from twisted.internet.testing import (
     MemoryReactor,
 )
+from twisted.python.failure import (
+    Failure,
+)
+from twisted.internet.error import (
+    ConnectionDone,
+)
 from twisted.internet.test.modulehelpers import (
     AlternateReactor,
 )
@@ -147,6 +153,95 @@ class DaemonizeTheRealServiceTests(SyncTestCase):
         )
 
 
+class DaemonizeStopTests(SyncTestCase):
+    """
+    Tests relating to stopping the daemon
+    """
+    def setUp(self):
+        self.nodedir = FilePath(self.mktemp())
+        self.nodedir.makedirs()
+        config = ""
+        self.nodedir.child("tahoe.cfg").setContent(config.encode("ascii"))
+        self.nodedir.child("tahoe-client.tac").touch()
+
+        # arrange to know when reactor.stop() is called
+        self.reactor = MemoryReactor()
+        self.stop_calls = []
+
+        def record_stop():
+            self.stop_calls.append(object())
+        self.reactor.stop = record_stop
+
+        super().setUp()
+
+    def test_stop_on_stdin_close(self):
+        """
+        We stop when stdin is closed.
+        """
+        options = parse_options(["run", self.nodedir.path])
+        stdout = options.stdout = StringIO()
+        stderr = options.stderr = StringIO()
+        stdin = options.stdin = StringIO()
+        run_options = options.subOptions
+
+        with AlternateReactor(self.reactor):
+            service = DaemonizeTheRealService(
+                "client",
+                self.nodedir.path,
+                run_options,
+            )
+            service.startService()
+
+            # We happen to know that the service uses reactor.callWhenRunning
+            # to schedule all its work (though I couldn't tell you *why*).
+            # Make sure those scheduled calls happen.
+            waiting = self.reactor.whenRunningHooks[:]
+            del self.reactor.whenRunningHooks[:]
+            for f, a, k in waiting:
+                f(*a, **k)
+
+            # there should be a single reader: our StandardIO process
+            # reader for stdin. Simulate it closing.
+            for r in self.reactor.getReaders():
+                r.connectionLost(Failure(ConnectionDone()))
+
+            self.assertEqual(len(self.stop_calls), 1)
+
+    def test_allow_stdin_close(self):
+        """
+        If --allow-stdin-close is specified then closing stdin doesn't
+        stop the process
+        """
+        options = parse_options(["run", "--allow-stdin-close", self.nodedir.path])
+        stdout = options.stdout = StringIO()
+        stderr = options.stderr = StringIO()
+        stdin = options.stdin = StringIO()
+        run_options = options.subOptions
+
+        with AlternateReactor(self.reactor):
+            service = DaemonizeTheRealService(
+                "client",
+                self.nodedir.path,
+                run_options,
+            )
+            service.startService()
+
+            # We happen to know that the service uses reactor.callWhenRunning
+            # to schedule all its work (though I couldn't tell you *why*).
+            # Make sure those scheduled calls happen.
+            waiting = self.reactor.whenRunningHooks[:]
+            del self.reactor.whenRunningHooks[:]
+            for f, a, k in waiting:
+                f(*a, **k)
+
+            # kind of cheating -- there are no readers, because we
+            # never instantiated a StandardIO in this case..
+            for r in self.reactor.getReaders():
+                r.connectionLost(Failure(ConnectionDone()))
+
+            self.assertEqual(self.stop_calls, [])
+
+
 class RunTests(SyncTestCase):
     """
     Tests for ``run``.

From e765c8db6fb237e161b2c4830601a7486b88a4e9 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Mon, 19 Jun 2023 17:55:30 -0600
Subject: [PATCH 352/362] move news

---
 newsfragments/{4036.enhancement => 4036.feature} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename newsfragments/{4036.enhancement => 4036.feature} (100%)

diff --git a/newsfragments/4036.enhancement b/newsfragments/4036.feature
similarity index 100%
rename from newsfragments/4036.enhancement
rename to newsfragments/4036.feature

From 357c9b003f1b5866f9eb250b6c1b5313aacf86e3 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Mon, 19 Jun 2023 17:55:36 -0600
Subject: [PATCH 353/362] flake8

---
 src/allmydata/test/cli/test_run.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index 96640d45a..731269d3d 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -179,9 +179,9 @@ class DaemonizeStopTests(SyncTestCase):
         We stop when stdin is closed.
         """
         options = parse_options(["run", self.nodedir.path])
-        stdout = options.stdout = StringIO()
-        stderr = options.stderr = StringIO()
-        stdin = options.stdin = StringIO()
+        options.stdout = StringIO()
+        options.stderr = StringIO()
+        options.stdin = StringIO()
         run_options = options.subOptions
 
         with AlternateReactor(self.reactor):
@@ -213,9 +213,9 @@ class DaemonizeStopTests(SyncTestCase):
         stop the process
         """
         options = parse_options(["run", "--allow-stdin-close", self.nodedir.path])
-        stdout = options.stdout = StringIO()
-        stderr = options.stderr = StringIO()
-        stdin = options.stdin = StringIO()
+        options.stdout = StringIO()
+        options.stderr = StringIO()
+        options.stdin = StringIO()
         run_options = options.subOptions
 
         with AlternateReactor(self.reactor):

From 02fba3b2b67e062fb8ca0f2aeb8177309002a451 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 20 Jun 2023 07:45:51 -0400
Subject: [PATCH 354/362] factor some duplication out of the tests

---
 src/allmydata/test/cli/test_run.py | 90 ++++++++++++++----------------
 1 file changed, 43 insertions(+), 47 deletions(-)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index 731269d3d..30d5cd893 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -174,37 +174,55 @@ class DaemonizeStopTests(SyncTestCase):
 
         super().setUp()
 
-    def test_stop_on_stdin_close(self):
+    def _make_daemon(self, extra_argv: list[str]) -> DaemonizeTheRealService:
         """
-        We stop when stdin is closed.
+        Create the daemonization service.
+
+        :param extra_argv: Extra arguments to pass between ``run`` and the
+            node path.
         """
-        options = parse_options(["run", self.nodedir.path])
+        options = parse_options(["run"] + extra_argv + [self.nodedir.path])
         options.stdout = StringIO()
         options.stderr = StringIO()
         options.stdin = StringIO()
         run_options = options.subOptions
+        return DaemonizeTheRealService(
+            "client",
+            self.nodedir.path,
+            run_options,
+        )
 
+    def _run_daemon(self) -> None:
+        """
+        Simulate starting up the reactor so the daemon plugin can do its
+        stuff.
+        """
+        # We happen to know that the service uses reactor.callWhenRunning
+        # to schedule all its work (though I couldn't tell you *why*).
+        # Make sure those scheduled calls happen.
+        waiting = self.reactor.whenRunningHooks[:]
+        del self.reactor.whenRunningHooks[:]
+        for f, a, k in waiting:
+            f(*a, **k)
+
+    def _close_stdin(self) -> None:
+        """
+        Simulate closing the daemon plugin's stdin.
+        """
+        # there should be a single reader: our StandardIO process
+        # reader for stdin. Simulate it closing.
+        for r in self.reactor.getReaders():
+            r.connectionLost(Failure(ConnectionDone()))
+
+    def test_stop_on_stdin_close(self):
+        """
+        We stop when stdin is closed.
+        """
         with AlternateReactor(self.reactor):
-            service = DaemonizeTheRealService(
-                "client",
-                self.nodedir.path,
-                run_options,
-            )
+            service = self._make_daemon([])
             service.startService()
-
-            # We happen to know that the service uses reactor.callWhenRunning
-            # to schedule all its work (though I couldn't tell you *why*).
-            # Make sure those scheduled calls happen.
-            waiting = self.reactor.whenRunningHooks[:]
-            del self.reactor.whenRunningHooks[:]
-            for f, a, k in waiting:
-                f(*a, **k)
-
-            # there should be a single reader: our StandardIO process
-            # reader for stdin. Simulate it closing.
-            for r in self.reactor.getReaders():
-                r.connectionLost(Failure(ConnectionDone()))
-
+            self._run_daemon()
+            self._close_stdin()
             self.assertEqual(len(self.stop_calls), 1)
 
     def test_allow_stdin_close(self):
@@ -212,33 +230,11 @@ class DaemonizeStopTests(SyncTestCase):
         If --allow-stdin-close is specified then closing stdin doesn't
         stop the process
         """
-        options = parse_options(["run", "--allow-stdin-close", self.nodedir.path])
-        options.stdout = StringIO()
-        options.stderr = StringIO()
-        options.stdin = StringIO()
-        run_options = options.subOptions
-
         with AlternateReactor(self.reactor):
-            service = DaemonizeTheRealService(
-                "client",
-                self.nodedir.path,
-                run_options,
-            )
+            service = self._make_daemon(["--allow-stdin-close"])
             service.startService()
-
-            # We happen to know that the service uses reactor.callWhenRunning
-            # to schedule all its work (though I couldn't tell you *why*).
-            # Make sure those scheduled calls happen.
-            waiting = self.reactor.whenRunningHooks[:]
-            del self.reactor.whenRunningHooks[:]
-            for f, a, k in waiting:
-                f(*a, **k)
-
-            # kind of cheating -- there are no readers, because we
-            # never instantiated a StandardIO in this case..
-            for r in self.reactor.getReaders():
-                r.connectionLost(Failure(ConnectionDone()))
-
+            self._run_daemon()
+            self._close_stdin()
             self.assertEqual(self.stop_calls, [])
 
 

From 7257851565c16ce200a0cfd64c2e92d3cc552783 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 20 Jun 2023 07:46:43 -0400
Subject: [PATCH 355/362] python 2/3 porting boilerplate cleanup

---
 src/allmydata/test/cli/test_run.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index 30d5cd893..6254a6259 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -1,16 +1,6 @@
 """
 Tests for ``allmydata.scripts.tahoe_run``.
-
-Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import re
 from six.moves import (

From 592e77beca60c997d06ec62b51e054e5ae59b05f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 20 Jun 2023 08:12:14 -0400
Subject: [PATCH 356/362] allow `list` as a generic container annotation

---
 src/allmydata/test/cli/test_run.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py
index 6254a6259..2adcfea19 100644
--- a/src/allmydata/test/cli/test_run.py
+++ b/src/allmydata/test/cli/test_run.py
@@ -2,6 +2,8 @@
 Tests for ``allmydata.scripts.tahoe_run``.
 """
 
+from __future__ import annotations
+
 import re
 from six.moves import (
     StringIO,

From 122e0a73a979f379e6bc7a3f795847be8dc6db0b Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 22 Jun 2023 01:29:55 -0600
Subject: [PATCH 357/362] more-generic testing hook

---
 src/allmydata/storage/http_client.py    |  9 ++-------
 src/allmydata/test/test_storage_http.py | 17 ++++++++++++++---
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index e7df3709d..8c0100656 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -427,7 +427,7 @@ class StorageClient(object):
     _pool: HTTPConnectionPool
     _clock: IReactorTime
     # Are we running unit tests?
-    _test_mode: bool
+    _analyze_response: Callable[[IResponse], None] = lambda _: None
 
     def relative_url(self, path: str) -> DecodedURL:
         """Get a URL relative to the base URL."""
@@ -534,12 +534,7 @@ class StorageClient(object):
         response = await self._treq.request(
             method, url, headers=headers, timeout=timeout, **kwargs
         )
-
-        if self._test_mode and response.code != 404:
-            # We're doing API queries, HTML is never correct except in 404, but
-            # it's the default for Twisted's web server so make sure nothing
-            # unexpected happened.
-            assert get_content_type(response.headers) != "text/html"
+        self._analyze_response(response)
 
         return response
 
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index 233d82989..aaa858db4 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -316,6 +316,17 @@ def result_of(d):
         + "This is probably a test design issue."
     )
 
+def response_is_not_html(response):
+    """
+    During tests, this is registered so we can ensure the web server
+    doesn't give us text/html.
+
+    HTML is never correct except in 404, but it's the default for
+    Twisted's web server so we assert nothing unexpected happened.
+    """
+    if response.code != 404:
+        assert get_content_type(response.headers) != "text/html"
+
 
 class CustomHTTPServerTests(SyncTestCase):
     """
@@ -342,7 +353,7 @@ class CustomHTTPServerTests(SyncTestCase):
             # fixed if https://github.com/twisted/treq/issues/226 were ever
             # fixed.
             clock=treq._agent._memoryReactor,
-            test_mode=True,
+            analyze_response=response_is_not_html,
         )
         self._http_server.clock = self.client._clock
 
@@ -560,7 +571,7 @@ class HttpTestFixture(Fixture):
             treq=self.treq,
             pool=None,
             clock=self.clock,
-            test_mode=True,
+            analyze_response=response_is_not_html,
         )
 
     def result_of_with_flush(self, d):
@@ -674,7 +685,7 @@ class GenericHTTPAPITests(SyncTestCase):
                 treq=StubTreq(self.http.http_server.get_resource()),
                 pool=None,
                 clock=self.http.clock,
-                test_mode=True,
+                analyze_response=response_is_not_html,
             )
         )
         with assert_fails_with_http_code(self, http.UNAUTHORIZED):

From 75b9c59846bffadbeca2c5941931d335790a23bd Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Thu, 22 Jun 2023 01:54:53 -0600
Subject: [PATCH 358/362] refactor

---
 src/allmydata/storage/http_client.py    |  7 ++++++-
 src/allmydata/storage/http_common.py    | 13 +++++++++++++
 src/allmydata/test/test_storage_http.py | 18 +++++-------------
 3 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/allmydata/storage/http_client.py b/src/allmydata/storage/http_client.py
index 8c0100656..f2165ffda 100644
--- a/src/allmydata/storage/http_client.py
+++ b/src/allmydata/storage/http_client.py
@@ -57,6 +57,7 @@ from .http_common import (
     get_content_type,
     CBOR_MIME_TYPE,
     get_spki_hash,
+    response_is_not_html,
 )
 from .common import si_b2a, si_to_human_readable
 from ..util.hashutil import timing_safe_compare
@@ -402,13 +403,17 @@ class StorageClientFactory:
         treq_client = HTTPClient(agent)
         https_url = DecodedURL().replace(scheme="https", host=nurl.host, port=nurl.port)
         swissnum = nurl.path[0].encode("ascii")
+        response_check = lambda _: None
+        if self.TEST_MODE_REGISTER_HTTP_POOL is not None:
+            response_check = response_is_not_html
+
         return StorageClient(
             https_url,
             swissnum,
             treq_client,
             pool,
             reactor,
-            self.TEST_MODE_REGISTER_HTTP_POOL is not None,
+            response_check,
         )
 
 
diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py
index e5f07898e..7ee137e1d 100644
--- a/src/allmydata/storage/http_common.py
+++ b/src/allmydata/storage/http_common.py
@@ -12,6 +12,7 @@ from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
 
 from werkzeug.http import parse_options_header
 from twisted.web.http_headers import Headers
+from twisted.web.iweb import IResponse
 
 CBOR_MIME_TYPE = "application/cbor"
 
@@ -27,6 +28,18 @@ def get_content_type(headers: Headers) -> Optional[str]:
     return content_type
 
 
+def response_is_not_html(response: IResponse) -> None:
+    """
+    During tests, this is registered so we can ensure the web server
+    doesn't give us text/html.
+
+    HTML is never correct except in 404, but it's the default for
+    Twisted's web server so we assert nothing unexpected happened.
+    """
+    if response.code != 404:
+        assert get_content_type(response.headers) != "text/html"
+
+
 def swissnum_auth_header(swissnum: bytes) -> bytes:
     """Return value for ``Authorization`` header."""
     return b"Tahoe-LAFS " + b64encode(swissnum).strip()
diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py
index aaa858db4..f660342ae 100644
--- a/src/allmydata/test/test_storage_http.py
+++ b/src/allmydata/test/test_storage_http.py
@@ -43,7 +43,11 @@ from testtools.matchers import Equals
 from zope.interface import implementer
 
 from .common import SyncTestCase
-from ..storage.http_common import get_content_type, CBOR_MIME_TYPE
+from ..storage.http_common import (
+    get_content_type,
+    CBOR_MIME_TYPE,
+    response_is_not_html,
+)
 from ..storage.common import si_b2a
 from ..storage.lease import LeaseInfo
 from ..storage.server import StorageServer
@@ -316,18 +320,6 @@ def result_of(d):
         + "This is probably a test design issue."
     )
 
-def response_is_not_html(response):
-    """
-    During tests, this is registered so we can ensure the web server
-    doesn't give us text/html.
-
-    HTML is never correct except in 404, but it's the default for
-    Twisted's web server so we assert nothing unexpected happened.
-    """
-    if response.code != 404:
-        assert get_content_type(response.headers) != "text/html"
-
-
 class CustomHTTPServerTests(SyncTestCase):
     """
     Tests that use a custom HTTP server.

From 992687a8b9a9390ac7534e3eb9d89519001d3bd5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 26 Jun 2023 09:05:36 -0400
Subject: [PATCH 359/362] News fragment

---
 newsfragments/3622.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3622.minor

diff --git a/newsfragments/3622.minor b/newsfragments/3622.minor
new file mode 100644
index 000000000..e69de29bb

From 5f9e784964ebbee5575cd6d57a2c65480ab99a15 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 26 Jun 2023 09:06:28 -0400
Subject: [PATCH 360/362] Better explanation

---
 src/allmydata/web/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/web/common.py b/src/allmydata/web/common.py
index 0685d60a0..1a0ba433b 100644
--- a/src/allmydata/web/common.py
+++ b/src/allmydata/web/common.py
@@ -724,9 +724,7 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *,
 
     :return: Either bytes or tuple of bytes.
     """
-    # This is not... obvious to callers, let's say, but it does happen in
-    # pretty much all non-test real code. We have to import here to prevent
-    # circular import.
+    # Need to import here to prevent circular import:
     from ..webish import TahoeLAFSRequest
 
     if isinstance(argname, str):
@@ -739,6 +737,8 @@ def get_arg(req: IRequest, argname: str | bytes, default: Optional[T] = None, *,
         results.extend(req.args[argname_bytes])
     argname_unicode = str(argname_bytes, "utf-8")
     if isinstance(req, TahoeLAFSRequest) and req.fields and argname_unicode in req.fields:
+        # In all but one or two unit tests, the request will be a
+        # TahoeLAFSRequest.
         value = req.fields[argname_unicode].value
         if isinstance(value, str):
             value = value.encode("utf-8")

From a7f45ab35564682511860314ac05ea012f8e8606 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 26 Jun 2023 11:09:32 -0400
Subject: [PATCH 361/362] If this ever does get called, make the error less
 obscure.

---
 src/allmydata/frontends/sftpd.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py
index 14f17e12d..7ef9a8820 100644
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@@ -1926,10 +1926,10 @@ class FakeTransport(object):
         logmsg("FakeTransport.loseConnection()", level=NOISY)
 
     def getHost(self):
-        pass
+        raise NotImplementedError()
 
     def getPeer(self):
-        pass
+        raise NotImplementedError()
 
 
 @implementer(ISession)

From a7100c749d400be0024c60eb5d93e92c45e32105 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@pythonspeed.com>
Date: Mon, 26 Jun 2023 11:20:46 -0400
Subject: [PATCH 362/362] Specific commit

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 89dbda748..2f245f2ed 100644
--- a/tox.ini
+++ b/tox.ini
@@ -124,7 +124,7 @@ basepython = python3
 deps =
     mypy==1.3.0
     # When 0.9.2 comes out it will work with 1.3, it's just unreleased at the moment...
-    git+https://github.com/shoobx/mypy-zope
+    git+https://github.com/shoobx/mypy-zope@f276030
     types-mock
     types-six
     types-PyYAML