Merge remote-tracking branch 'origin/master' into 3961.test-vectors

.circleci/config.yml

@@ -15,16 +15,11 @@ workflows:
   ci:
     jobs:
       # Start with jobs testing various platforms.
-      - "debian-10":
-          {}
       - "debian-11":
           {}
 
       - "ubuntu-20-04":
           {}
-      - "ubuntu-18-04":
-          requires:
-            - "ubuntu-20-04"
 
       # Equivalent to RHEL 8; CentOS 8 is dead.
       - "oraclelinux-8":
@@ -85,11 +80,7 @@ workflows:
       # Contexts are managed in the CircleCI web interface:
       #
       #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-      - "build-image-debian-10": &DOCKERHUB_CONTEXT
-          context: "dockerhub-auth"
-      - "build-image-debian-11":
-          <<: *DOCKERHUB_CONTEXT
-      - "build-image-ubuntu-18-04":
+      - "build-image-debian-11": &DOCKERHUB_CONTEXT
           <<: *DOCKERHUB_CONTEXT
       - "build-image-ubuntu-20-04":
           <<: *DOCKERHUB_CONTEXT
@@ -167,12 +158,7 @@ jobs:
           command: |
             dist/Tahoe-LAFS/tahoe --version
 
-  debian-10: &DEBIAN
-    docker:
-      - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/debian:10-py3.7"
-        user: "nobody"
-
+  debian-11: &DEBIAN
     environment: &UTF_8_ENVIRONMENT
       # In general, the test suite is not allowed to fail while the job
       # succeeds.  But you can set this to "yes" if you want it to be
@@ -184,7 +170,7 @@ jobs:
       # filenames and argv).
       LANG: "en_US.UTF-8"
       # Select a tox environment to run for this job.
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
       # The path in which test artifacts will be placed.
@@ -252,15 +238,11 @@ jobs:
               /tmp/venv/bin/codecov
             fi
 
-  debian-11:
-    <<: *DEBIAN
     docker:
       - <<: *DOCKERHUB_AUTH
         image: "tahoelafsci/debian:11-py3.9"
         user: "nobody"
-    environment:
-      <<: *UTF_8_ENVIRONMENT
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py39"
+
 
   # Restore later using PyPy3.8
   # pypy27-buster:
@@ -312,22 +294,6 @@ jobs:
       - run: *SETUP_VIRTUALENV
       - run: *RUN_TESTS
 
-  ubuntu-18-04: &UBUNTU_18_04
-    <<: *DEBIAN
-    docker:
-      - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/ubuntu:18.04-py3.7"
-        user: "nobody"
-
-    environment:
-      <<: *UTF_8_ENVIRONMENT
-      # The default trial args include --rterrors which is incompatible with
-      # this reporter on Python 3.  So drop that and just specify the
-      # reporter.
-      TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py37"
-
-
   ubuntu-20-04:
     <<: *DEBIAN
     docker:
@@ -445,7 +411,7 @@ jobs:
   typechecks:
     docker:
       - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/ubuntu:18.04-py3.7"
+        image: "tahoelafsci/ubuntu:20.04-py3.9"
 
     steps:
       - "checkout"
@@ -457,7 +423,7 @@ jobs:
   docs:
     docker:
       - <<: *DOCKERHUB_AUTH
-        image: "tahoelafsci/ubuntu:18.04-py3.7"
+        image: "tahoelafsci/ubuntu:20.04-py3.9"
 
     steps:
       - "checkout"
@@ -508,15 +474,6 @@ jobs:
             docker push tahoelafsci/${DISTRO}:${TAG}-py${PYTHON_VERSION}
 
 
-  build-image-debian-10:
-    <<: *BUILD_IMAGE
-
-    environment:
-      DISTRO: "debian"
-      TAG: "10"
-      PYTHON_VERSION: "3.7"
-
-
   build-image-debian-11:
     <<: *BUILD_IMAGE
 
@@ -525,14 +482,6 @@ jobs:
       TAG: "11"
       PYTHON_VERSION: "3.9"
 
-  build-image-ubuntu-18-04:
-    <<: *BUILD_IMAGE
-
-    environment:
-      DISTRO: "ubuntu"
-      TAG: "18.04"
-      PYTHON_VERSION: "3.7"
-
 
   build-image-ubuntu-20-04:
     <<: *BUILD_IMAGE

.github/workflows/ci.yml

@@ -48,21 +48,20 @@ jobs:
           - windows-latest
           - ubuntu-latest
         python-version:
-          - "3.7"
           - "3.8"
           - "3.9"
           - "3.10"
         include:
-          # On macOS don't bother with 3.7-3.8, just to get faster builds.
+          # On macOS don't bother with 3.8, just to get faster builds.
           - os: macos-latest
             python-version: "3.9"
           - os: macos-latest
             python-version: "3.10"
           # We only support PyPy on Linux at the moment.
-          - os: ubuntu-latest
-            python-version: "pypy-3.7"
           - os: ubuntu-latest
             python-version: "pypy-3.8"
+          - os: ubuntu-latest
+            python-version: "pypy-3.9"
 
     steps:
       # See https://github.com/actions/checkout. A fetch-depth of 0
@@ -153,19 +152,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - windows-latest
+        include:
+          - os: macos-latest
+            python-version: "3.9"
+            force-foolscap: false
+          - os: windows-latest
+            python-version: "3.9"
+            force-foolscap: false
           # 22.04 has some issue with Tor at the moment:
           # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3943
-          - ubuntu-20.04
-        python-version:
-          - 3.7
-          - 3.9
-        include:
-          # On macOS don't bother with 3.7, just to get faster builds.
-          - os: macos-latest
-            python-version: 3.9
-
+          - os: ubuntu-20.04
+            python-version: "3.9"
+            force-foolscap: false
     steps:
 
       - name: Install Tor [Ubuntu]
@@ -206,12 +204,24 @@ jobs:
         run: python misc/build_helpers/show-tool-versions.py
 
       - name: Run "Python 3 integration tests"
+        if: "${{ !matrix.force-foolscap }}"
         env:
           # On macOS this is necessary to ensure unix socket paths for tor
           # aren't too long. On Windows tox won't pass it through so it has no
           # effect. On Linux it doesn't make a difference one way or another.
           TMPDIR: "/tmp"
-        run: tox -e integration
+        run: |
+          tox -e integration
+
+      - name: Run "Python 3 integration tests (force Foolscap)"
+        if: "${{ matrix.force-foolscap }}"
+        env:
+          # On macOS this is necessary to ensure unix socket paths for tor
+          # aren't too long. On Windows tox won't pass it through so it has no
+          # effect. On Linux it doesn't make a difference one way or another.
+          TMPDIR: "/tmp"
+        run: |
+          tox -e integration -- --force-foolscap integration/
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v3

README.rst

@@ -56,7 +56,7 @@ Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS <docs/running.rst>`__
 🐍 Python 2
 -----------
 
-Python 3.7 or later is now required.
+Python 3.8 or later is required.
 If you are still using Python 2.7, use Tahoe-LAFS version 1.17.1.
 
 

default.nix

@@ -29,7 +29,7 @@ in
 , pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
                                   # for dependency resolution
 
-, pythonVersion ? "python37" # a string choosing the python derivation from
+, pythonVersion ? "python39" # a string choosing the python derivation from
                              # nixpkgs to target
 
 , extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,

integration/conftest.py

@@ -1,15 +1,6 @@
 """
 Ported to Python 3.
 """
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-
 import sys
 import shutil
 from time import sleep
@@ -65,6 +56,13 @@ def pytest_addoption(parser):
         "--coverage", action="store_true", dest="coverage",
         help="Collect coverage statistics",
     )
+    parser.addoption(
+        "--force-foolscap", action="store_true", default=False,
+        dest="force_foolscap",
+        help=("If set, force Foolscap only for the storage protocol. " +
+              "Otherwise HTTP will be used.")
+    )
+
 
 @pytest.fixture(autouse=True, scope='session')
 def eliot_logging():

integration/util.py

@@ -322,6 +322,14 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
                 u'log_gatherer.furl',
                 flog_gatherer,
             )
+            force_foolscap = request.config.getoption("force_foolscap")
+            assert force_foolscap in (True, False)
+            set_config(
+                config,
+                'storage',
+                'force_foolscap',
+                str(force_foolscap),
+            )
             write_config(FilePath(config_path), config)
         created_d.addCallback(created)
 

newsfragments/3942.minor

@@ -0,0 +1 @@
+

newsfragments/3964.removed

@@ -0,0 +1 @@
+Python 3.7 is no longer supported, and Debian 10 and Ubuntu 18.04 are no longer tested.

setup.py

@@ -223,7 +223,7 @@ def run_command(args, cwd=None):
     use_shell = sys.platform == "win32"
     try:
         p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell)
-    except EnvironmentError as e:  # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.7+
+    except EnvironmentError as e:  # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 3.8+
         print("Warning: unable to run %r." % (" ".join(args),))
         print(e)
         return None
@@ -374,8 +374,8 @@ setup(name="tahoe-lafs", # also set in __init__.py
       package_dir = {'':'src'},
       packages=find_packages('src') + ['allmydata.test.plugins'],
       classifiers=trove_classifiers,
-      # We support Python 3.7 or later. 3.11 is not supported yet.
-      python_requires=">=3.7, <3.11",
+      # We support Python 3.8 or later. 3.11 is not supported yet.
+      python_requires=">=3.8, <3.11",
       install_requires=install_requires,
       extras_require={
           # Duplicate the Twisted pywin32 dependency here.  See
@@ -388,9 +388,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
           ],
           "test": [
               "flake8",
-              # On Python 3.7, importlib_metadata v5 breaks flake8.
-              # https://github.com/python/importlib_metadata/issues/407
-              "importlib_metadata<5; python_version < '3.8'",
               # Pin a specific pyflakes so we don't have different folks
               # disagreeing on what is or is not a lint issue.  We can bump
               # this version from time to time, but we will do it

src/allmydata/crypto/rsa.py

@@ -12,14 +12,9 @@ on any of their methods.
 
 Ported to Python 3.
 """
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from __future__ import annotations
 
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from functools import partial
 
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends import default_backend
@@ -72,20 +67,39 @@ def create_signing_keypair_from_string(private_key_der):
 
     :returns: 2-tuple of (private_key, public_key)
     """
-    priv_key = load_der_private_key(
+    load = partial(
+        load_der_private_key,
         private_key_der,
         password=None,
         backend=default_backend(),
     )
-    if not isinstance(priv_key, rsa.RSAPrivateKey):
+
+    try:
+        # Load it once without the potentially expensive OpenSSL validation
+        # checks.  These have superlinear complexity.  We *will* run them just
+        # below - but first we'll apply our own constant-time checks.
+        unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True)
+    except TypeError:
+        # cryptography<39 does not support this parameter, so just load the
+        # key with validation...
+        unsafe_priv_key = load()
+        # But avoid *reloading* it since that will run the expensive
+        # validation *again*.
+        load = lambda: unsafe_priv_key
+
+    if not isinstance(unsafe_priv_key, rsa.RSAPrivateKey):
         raise ValueError(
             "Private Key did not decode to an RSA key"
         )
-    if priv_key.key_size != 2048:
+    if unsafe_priv_key.key_size != 2048:
         raise ValueError(
             "Private Key must be 2048 bits"
         )
-    return priv_key, priv_key.public_key()
+
+    # Now re-load it with OpenSSL's validation applied.
+    safe_priv_key = load()
+
+    return safe_priv_key, safe_priv_key.public_key()
 
 
 def der_string_from_signing_key(private_key):

src/allmydata/storage/http_client.py

@@ -323,6 +323,7 @@ class StorageClient(object):
         swissnum = nurl.path[0].encode("ascii")
         certificate_hash = nurl.user.encode("ascii")
         pool = HTTPConnectionPool(reactor)
+        pool.maxPersistentPerHost = 20
 
         if cls.TEST_MODE_REGISTER_HTTP_POOL is not None:
             cls.TEST_MODE_REGISTER_HTTP_POOL(pool)

src/allmydata/storage/http_server.py

@@ -100,7 +100,7 @@ def _authorization_decorator(required_secrets):
         @wraps(f)
         def route(self, request, *args, **kwargs):
             if not timing_safe_compare(
-                request.requestHeaders.getRawHeaders("Authorization", [None])[0].encode(
+                request.requestHeaders.getRawHeaders("Authorization", [""])[0].encode(
                     "utf-8"
                 ),
                 swissnum_auth_header(self._swissnum),

src/allmydata/test/test_storage_http.py

@@ -37,6 +37,7 @@ from twisted.web import http
 from twisted.web.http_headers import Headers
 from werkzeug import routing
 from werkzeug.exceptions import NotFound as WNotFound
+from testtools.matchers import Equals
 
 from .common import SyncTestCase
 from ..storage.http_common import get_content_type, CBOR_MIME_TYPE
@@ -555,6 +556,20 @@ class GenericHTTPAPITests(SyncTestCase):
         super(GenericHTTPAPITests, self).setUp()
         self.http = self.useFixture(HttpTestFixture())
 
+    def test_missing_authentication(self) -> None:
+        """
+        If nothing is given in the ``Authorization`` header at all an
+        ``Unauthorized`` response is returned.
+        """
+        client = StubTreq(self.http.http_server.get_resource())
+        response = self.http.result_of_with_flush(
+            client.request(
+                "GET",
+                "http://127.0.0.1/storage/v1/version",
+            ),
+        )
+        self.assertThat(response.code, Equals(http.UNAUTHORIZED))
+
     def test_bad_authentication(self):
         """
         If the wrong swissnum is used, an ``Unauthorized`` response code is

tests.nix

@@ -5,7 +5,7 @@ in
 { pkgsVersion ? "nixpkgs-21.11"
 , pkgs ? import sources.${pkgsVersion} { }
 , pypiData ? sources.pypi-deps-db
-, pythonVersion ? "python37"
+, pythonVersion ? "python39"
 , mach-nix ? import sources.mach-nix {
     inherit pkgs pypiData;
     python = pythonVersion;
@@ -21,7 +21,7 @@ let
     inherit pkgs;
     lib = pkgs.lib;
   };
-  tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test;
+  tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test;
 
   # Get the Tahoe-LAFS package itself.  This does not include test
   # requirements and we don't ask for test requirements so that we can just

tox.ini

@@ -7,11 +7,9 @@
 # the tox-gh-actions package.
 [gh-actions]
 python =
-    3.7: py37-coverage,typechecks,codechecks
     3.8: py38-coverage
     3.9: py39-coverage
     3.10: py310-coverage
-    pypy-3.7: pypy37
     pypy-3.8: pypy38
     pypy-3.9: pypy39
 
@@ -19,7 +17,7 @@ python =
 twisted = 1
 
 [tox]
-envlist = typechecks,codechecks,py{37,38,39,310}-{coverage},pypy27,pypy37,pypy38,pypy39,integration
+envlist = typechecks,codechecks,py{38,39,310}-{coverage},pypy27,pypy38,pypy39,integration
 minversion = 2.4
 
 [testenv]
@@ -49,8 +47,6 @@ deps =
      # regressions in new releases of this package that cause us the kind of
      # suffering we're trying to avoid with the above pins.
      certifi
-     # VCS hooks support
-     py37,!coverage: pre-commit
 
 # We add usedevelop=False because testing against a true installation gives
 # more useful results.