Merge remote-tracking branch 'origin/master' into 3935-happy-eyeballs

.github/workflows/ci.yml

@@ -51,12 +51,13 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
         include:
           # On macOS don't bother with 3.8, just to get faster builds.
           - os: macos-latest
             python-version: "3.9"
           - os: macos-latest
-            python-version: "3.10"
+            python-version: "3.11"
           # We only support PyPy on Linux at the moment.
           - os: ubuntu-latest
             python-version: "pypy-3.8"
@@ -174,7 +175,7 @@ jobs:
           # 22.04 has some issue with Tor at the moment:
           # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
           - os: ubuntu-20.04
-            python-version: "3.9"
+            python-version: "3.11"
             force-foolscap: false
     steps:
 

integration/util.py

@@ -367,6 +367,12 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
                 'force_foolscap',
                 str(force_foolscap),
             )
+            set_config(
+                config,
+                'client',
+                'force_foolscap',
+                str(force_foolscap),
+            )
             write_config(FilePath(config_path), config)
         created_d.addCallback(created)
 

newsfragments/3982.feature

@@ -0,0 +1 @@
+Added support for Python 3.11.

setup.py

@@ -55,7 +55,9 @@ install_requires = [
     # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs
     # * foolscap 0.13.2 drops i2p support completely
     # * foolscap >= 21.7 is necessary for Python 3 with i2p support.
+    # * foolscap >= 23.3 is necessary for Python 3.11.
     "foolscap >= 21.7.0",
+    "foolscap >= 23.3.0; python_version > '3.10'",
 
     # * cryptography 2.6 introduced some ed25519 APIs we rely on.  Note that
     #   Twisted[conch] also depends on cryptography and Twisted[tls]
@@ -380,8 +382,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
       package_dir = {'':'src'},
       packages=find_packages('src') + ['allmydata.test.plugins'],
       classifiers=trove_classifiers,
-      # We support Python 3.8 or later. 3.11 is not supported yet.
-      python_requires=">=3.8, <3.11",
+      # We support Python 3.8 or later, 3.12 is untested for now
+      python_requires=">=3.8, <3.12",
       install_requires=install_requires,
       extras_require={
           # Duplicate the Twisted pywin32 dependency here.  See

src/allmydata/client.py

@@ -60,7 +60,7 @@ from allmydata.interfaces import (
 )
 from allmydata.nodemaker import NodeMaker
 from allmydata.blacklist import Blacklist
-
+from allmydata.node import _Config
 
 KiB=1024
 MiB=1024*KiB
@@ -94,6 +94,7 @@ _client_config = configutil.ValidConfiguration(
             "shares.total",
             "shares._max_immutable_segment_size_for_testing",
             "storage.plugins",
+            "force_foolscap",
         ),
         "storage": (
             "debug_discard",
@@ -465,7 +466,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
     return introducer_clients
 
 
-def create_storage_farm_broker(config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
+def create_storage_farm_broker(config: _Config, default_connection_handlers, foolscap_connection_handlers, tub_options, introducer_clients):
     """
     Create a StorageFarmBroker object, for use by Uploader/Downloader
     (and everybody else who wants to use storage servers)

src/allmydata/storage_client.py

@@ -88,6 +88,7 @@ from allmydata.storage.http_client import (
     ClientException as HTTPClientException, StorageClientMutables,
     ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException
 )
+from .node import _Config
 
 ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs"
 
@@ -198,7 +199,7 @@ class StorageFarmBroker(service.MultiService):
             self,
             permute_peers,
             tub_maker,
-            node_config,
+            node_config: _Config,
             storage_client_config=None,
     ):
         service.MultiService.__init__(self)
@@ -217,9 +218,9 @@ class StorageFarmBroker(service.MultiService):
         # own Reconnector, and will give us a RemoteReference when we ask
         # them for it.
         self.servers = BytesKeyDict()
-        self._static_server_ids = set() # ignore announcements for these
+        self._static_server_ids : set[bytes] = set() # ignore announcements for these
         self.introducer_client = None
-        self._threshold_listeners = [] # tuples of (threshold, Deferred)
+        self._threshold_listeners : list[tuple[float,defer.Deferred[Any]]]= [] # tuples of (threshold, Deferred)
         self._connected_high_water_mark = 0
 
     @log_call(action_type=u"storage-client:broker:set-static-servers")
@@ -273,6 +274,16 @@ class StorageFarmBroker(service.MultiService):
             in self.storage_client_config.storage_plugins.items()
         })
 
+    @staticmethod
+    def _should_we_use_http(node_config: _Config, announcement: dict) -> bool:
+        """
+        Given an announcement dictionary and config, return whether we should
+        connect to storage server over HTTP.
+        """
+        return not node_config.get_config(
+            "client", "force_foolscap", default=True, boolean=True,
+        ) and len(announcement.get(ANONYMOUS_STORAGE_NURLS, [])) > 0
+
     @log_call(
         action_type=u"storage-client:broker:make-storage-server",
         include_args=["server_id"],
@@ -298,7 +309,7 @@ class StorageFarmBroker(service.MultiService):
             "pub-{}".format(str(server_id, "ascii")),  # server_id is v0-<key> not pub-v0-key .. for reasons?
         )
 
-        if len(server["ann"].get(ANONYMOUS_STORAGE_NURLS, [])) > 0:
+        if self._should_we_use_http(self.node_config, server["ann"]):
             s = HTTPNativeStorageServer(
                 server_id,
                 server["ann"],

src/allmydata/test/cli/wormholetesting.py

@@ -34,7 +34,7 @@ from __future__ import annotations
 
 from typing import Iterator, Optional, List, Tuple
 from collections.abc import Awaitable
-from inspect import getargspec
+from inspect import getfullargspec
 from itertools import count
 from sys import stderr
 
@@ -141,8 +141,8 @@ def _verify():
     """
     # Poor man's interface verification.
 
-    a = getargspec(create)
-    b = getargspec(MemoryWormholeServer.create)
+    a = getfullargspec(create)
+    b = getfullargspec(MemoryWormholeServer.create)
     # I know it has a `self` argument at the beginning.  That's okay.
     b = b._replace(args=b.args[1:])
     assert a == b, "{} != {}".format(a, b)

src/allmydata/test/common_system.py

@@ -871,6 +871,7 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
         setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,))
         setconf(config, which, "storage", "force_foolscap", str(force_foolscap))
+        setconf(config, which, "client", "force_foolscap", str(force_foolscap))
 
         tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor)
         setnode("tub.port", tub_port_endpoint)

src/allmydata/test/test_storage_client.py

@@ -89,11 +89,14 @@ from allmydata.storage_client import (
     StorageFarmBroker,
     _FoolscapStorage,
     _NullStorage,
-    _pick_a_http_server
+    _pick_a_http_server,
+    ANONYMOUS_STORAGE_NURLS,
 )
 from ..storage.server import (
     StorageServer,
 )
+from ..client import config_from_string
+
 from allmydata.interfaces import (
     IConnectionStatus,
     IStorageServer,
@@ -736,6 +739,45 @@ storage:
         yield done
         self.assertTrue(done.called)
 
+    def test_should_we_use_http_default(self):
+        """Default is to not use HTTP; this will change eventually"""
+        basedir = self.mktemp()
+        node_config = config_from_string(basedir, "", "")
+        announcement = {ANONYMOUS_STORAGE_NURLS: ["pb://..."]}
+        self.assertFalse(
+            StorageFarmBroker._should_we_use_http(node_config, announcement)
+        )
+        self.assertFalse(
+            StorageFarmBroker._should_we_use_http(node_config, {})
+        )
+
+    def test_should_we_use_http(self):
+        """
+        If HTTP is allowed, it will only be used if the announcement includes
+        some NURLs.
+        """
+        basedir = self.mktemp()
+
+        no_nurls = {}
+        empty_nurls = {ANONYMOUS_STORAGE_NURLS: []}
+        has_nurls = {ANONYMOUS_STORAGE_NURLS: ["pb://.."]}
+
+        for force_foolscap, announcement, expected_http_usage in [
+                ("false", no_nurls, False),
+                ("false", empty_nurls, False),
+                ("false", has_nurls, True),
+                ("true", empty_nurls, False),
+                ("true", no_nurls, False),
+                ("true", has_nurls, False),
+        ]:
+            node_config = config_from_string(
+                basedir, "", f"[client]\nforce_foolscap = {force_foolscap}"
+            )
+            self.assertEqual(
+                StorageFarmBroker._should_we_use_http(node_config, announcement),
+                expected_http_usage
+            )
+
 
 class PickHTTPServerTests(unittest.SynchronousTestCase):
     """Tests for ``_pick_a_http_server``."""