Merge remote-tracking branch 'origin/master' into 3935-happy-eyeballs

.circleci/config.yml

@@ -71,12 +71,12 @@ workflows:
           {}
 
       - "nixos":
-          name: "NixOS 21.05"
-          nixpkgs: "21.05"
+          name: "NixOS 22.11"
+          nixpkgs: "22.11"
 
       - "nixos":
-          name: "NixOS 21.11"
-          nixpkgs: "21.11"
+          name: "NixOS unstable"
+          nixpkgs: "unstable"
 
       # Eventually, test against PyPy 3.8
       #- "pypy27-buster":
@@ -412,33 +412,18 @@ jobs:
               --run 'python setup.py update_version'
 
       - "run":
-          name: "Build"
+          name: "Test"
           command: |
             # CircleCI build environment looks like it has a zillion and a
             # half cores.  Don't let Nix autodetect this high core count
             # because it blows up memory usage and fails the test run.  Pick a
             # number of cores that suites the build environment we're paying
             # for (the free one!).
-            #
-            # Also, let it run more than one job at a time because we have to
-            # build a couple simple little dependencies that don't take
-            # advantage of multiple cores and we get a little speedup by doing
-            # them in parallel.
-            source .circleci/lib.sh
-            cache_if_able nix-build \
-              --cores 3 \
-              --max-jobs 2 \
-              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
-
-      - "run":
-          name: "Test"
-          command: |
-            # Let it go somewhat wild for the test suite itself
             source .circleci/lib.sh
             cache_if_able nix-build \
               --cores 8 \
               --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
-              tests.nix
+              nix/tests.nix
 
   typechecks:
     docker:

default.nix

@@ -1,8 +1,7 @@
 let
   # sources.nix contains information about which versions of some of our
-  # dependencies we should use.  since we use it to pin nixpkgs and the PyPI
-  # package database, roughly all the rest of our dependencies are *also*
-  # pinned - indirectly.
+  # dependencies we should use.  since we use it to pin nixpkgs, all the rest
+  # of our dependencies are *also* pinned - indirectly.
   #
   # sources.nix is managed using a tool called `niv`.  as an example, to
   # update to the most recent version of nixpkgs from the 21.11 maintenance
@@ -10,93 +9,45 @@ let
   #
   #   niv update nixpkgs-21.11
   #
-  # or, to update the PyPI package database -- which is necessary to make any
-  # newly released packages visible -- you likewise run:
-  #
-  #   niv update pypi-deps-db
-  #
   # niv also supports chosing a specific revision, following a different
   # branch, etc.  find complete documentation for the tool at
   # https://github.com/nmattia/niv
   sources = import nix/sources.nix;
 in
 {
-  pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
+  pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
                                 # niv-managed sources data
 
 , pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
 
-, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
-                                  # for dependency resolution
+, pythonVersion ? "python310" # a string choosing the python derivation from
+                              # nixpkgs to target
 
-, pythonVersion ? "python39" # a string choosing the python derivation from
-                             # nixpkgs to target
+, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
+                                # the dependencies of which the resulting
+                                # package will also depend on.  Include all of the
+                                # runtime extras by default because the incremental
+                                # cost of including them is a lot smaller than the
+                                # cost of re-building the whole thing to add them.
 
-, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
-                           # the dependencies of which the resulting package
-                           # will also depend on.  Include all of the runtime
-                           # extras by default because the incremental cost of
-                           # including them is a lot smaller than the cost of
-                           # re-building the whole thing to add them.
-
-, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
-                                       # the tahoe-lafs package
-    inherit pkgs pypiData;
-    python = pythonVersion;
-}
 }:
-# The project name, version, and most other metadata are automatically
-# extracted from the source.  Some requirements are not properly extracted
-# and those cases are handled below.  The version can only be extracted if
-# `setup.py update_version` has been run (this is not at all ideal but it
-# seems difficult to fix) - so for now just be sure to run that first.
-mach-nix.buildPythonPackage rec {
-  # Define the location of the Tahoe-LAFS source to be packaged.  Clean up all
-  # as many of the non-source files (eg the `.git` directory, `~` backup
-  # files, nix's own `result` symlink, etc) as possible to avoid needing to
-  # re-build when files that make no difference to the package have changed.
-  src = pkgs.lib.cleanSource ./.;
-
+with (pkgs.${pythonVersion}.override {
+  packageOverrides = self: super: {
+    # Some dependencies aren't packaged in nixpkgs so supply our own packages.
+    pycddl = self.callPackage ./nix/pycddl.nix { };
+    txi2p = self.callPackage ./nix/txi2p.nix { };
+  };
+}).pkgs;
+callPackage ./nix/tahoe-lafs.nix {
   # Select whichever package extras were requested.
-  inherit extras;
+  inherit extrasNames;
 
-  # Define some extra requirements that mach-nix does not automatically detect
-  # from inspection of the source.  We typically don't need to put version
-  # constraints on any of these requirements.  The pypi-deps-db we're
-  # operating with makes dependency resolution deterministic so as long as it
-  # works once it will always work.  It could be that in the future we update
-  # pypi-deps-db and an incompatibility arises - in which case it would make
-  # sense to apply some version constraints here.
-  requirementsExtra = ''
-    # mach-nix does not yet support pyproject.toml which means it misses any
-    # build-time requirements of our dependencies which are declared in such a
-    # file.  Tell it about them here.
-    setuptools_rust
+  # Define the location of the Tahoe-LAFS source to be packaged (the same
+  # directory as contains this file).  Clean up as many of the non-source
+  # files (eg the `.git` directory, `~` backup files, nix's own `result`
+  # symlink, etc) as possible to avoid needing to re-build when files that
+  # make no difference to the package have changed.
+  tahoe-lafs-src = pkgs.lib.cleanSource ./.;
 
-    # mach-nix does not yet parse environment markers (e.g. "python > '3.0'")
-    # correctly. It misses all of our requirements which have an environment marker.
-    # Duplicate them here.
-    foolscap
-    eliot
-    pyrsistent
-    collections-extended
-  '';
-
-  # Specify where mach-nix should find packages for our Python dependencies.
-  # There are some reasonable defaults so we only need to specify certain
-  # packages where the default configuration runs into some issue.
-  providers = {
-  };
-
-  # Define certain overrides to the way Python dependencies are built.
-  _ = {
-    # Remove a click-default-group patch for a test suite problem which no
-    # longer applies because the project apparently no longer has a test suite
-    # in its source distribution.
-    click-default-group.patches = [];
-  };
-
-  passthru.meta.mach-nix = {
-    inherit providers _;
-  };
+  doCheck = false;
 }

integration/test_get_put.py

@@ -3,7 +3,7 @@ Integration tests for getting and putting files, including reading from stdin
 and stdout.
 """
 
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import sys
 
 import pytest
@@ -67,6 +67,24 @@ def test_get_to_stdout(alice, get_put_alias, tmpdir):
     assert p.wait() == 0
 
 
+def test_large_file(alice, get_put_alias, tmp_path):
+    """
+    It's possible to upload and download a larger file.
+
+    We avoid stdin/stdout since that's flaky on Windows.
+    """
+    tempfile = tmp_path / "file"
+    with tempfile.open("wb") as f:
+        f.write(DATA * 1_000_000)
+    cli(alice, "put", str(tempfile), "getput:largefile")
+
+    outfile = tmp_path / "out"
+    check_call(
+        ["tahoe", "--node-directory", alice.node_dir, "get", "getput:largefile", str(outfile)],
+    )
+    assert outfile.read_bytes() == tempfile.read_bytes()
+
+
 @pytest.mark.skipif(
     sys.platform.startswith("win"),
     reason="reconfigure() has issues on Windows"

mypy.ini

@@ -7,4 +7,18 @@ show_error_codes = True
 warn_unused_configs =True
 no_implicit_optional = True
 warn_redundant_casts = True
-strict_equality = True
+strict_equality = True
+
+[mypy-allmydata.test.cli.wormholetesting]
+disallow_any_generics = True
+disallow_subclassing_any = True
+disallow_untyped_calls = True
+disallow_untyped_defs = True
+disallow_incomplete_defs = True
+check_untyped_defs = True
+disallow_untyped_decorators = True
+warn_unused_ignores = True
+warn_return_any = True
+no_implicit_reexport = True
+strict_equality = True
+strict_concatenate = True

newsfragments/3989.installation

@@ -0,0 +1 @@
+tenacity is no longer a dependency.

nix/pycddl.nix

@@ -0,0 +1,51 @@
+# package https://gitlab.com/tahoe-lafs/pycddl
+#
+# also in the process of being pushed upstream
+# https://github.com/NixOS/nixpkgs/pull/221220
+#
+# we should switch to the upstream package when it is available from our
+# minimum version of nixpkgs.
+#
+# if you need to update this package to a new pycddl release then
+#
+# 1. change value given to `buildPythonPackage` for `version` to match the new
+#    release
+#
+# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
+#
+# 3. run `nix-build`
+#
+# 4. there will be an error about a hash mismatch.  change the value given to
+#    `fetchPypi` for `sha256` to the "actual" hash value report.
+#
+# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`.
+#
+# 6. run `nix-build`
+#
+# 7. there will be an error about a hash mismatch.  change the value given to
+#    `cargoDeps` for `hash` to the "actual" hash value report.
+#
+# 8. run `nix-build`.  it should succeed.  if it does not, seek assistance.
+#
+{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
+buildPythonPackage rec {
+  pname = "pycddl";
+  version = "0.4.0";
+  format = "pyproject";
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
+  };
+
+  nativeBuildInputs = with rustPlatform; [
+    maturinBuildHook
+    cargoSetupHook
+  ];
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    name = "${pname}-${version}";
+    hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs=";
+  };
+}

nix/sources.json

@@ -1,16 +1,4 @@
 {
-    "mach-nix": {
-        "branch": "switch-to-nix-pypi-fetcher-2",
-        "description": "Create highly reproducible python environments",
-        "homepage": "",
-        "owner": "PrivateStorageio",
-        "repo": "mach-nix",
-        "rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4",
-        "sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls",
-        "type": "tarball",
-        "url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
     "niv": {
         "branch": "master",
         "description": "Easy dependency management for Nix projects",
@@ -23,40 +11,28 @@
         "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixpkgs-21.05": {
-        "branch": "nixos-21.05",
+    "nixpkgs-22.11": {
+        "branch": "nixos-22.11",
         "description": "Nix Packages collection",
         "homepage": "",
-        "owner": "NixOS",
+        "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
-        "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
+        "rev": "970402e6147c49603f4d06defe44d27fe51884ce",
+        "sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
+        "url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
-    "nixpkgs-21.11": {
-        "branch": "nixos-21.11",
-        "description": "Nix Packages collection",
-        "homepage": "",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "838eefb4f93f2306d4614aafb9b2375f315d917f",
-        "sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz",
-        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
-    },
-    "pypi-deps-db": {
+    "nixpkgs-unstable": {
         "branch": "master",
-        "description": "Probably the most complete python dependency database",
+        "description": "Nix Packages collection",
         "homepage": "",
-        "owner": "DavHau",
-        "repo": "pypi-deps-db",
-        "rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6",
-        "sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "d0c9a536331227ab883b4f6964be638fa436d81f",
+        "sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1",
         "type": "tarball",
-        "url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz",
+        "url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     }
 }

nix/tahoe-lafs.nix

@@ -0,0 +1,83 @@
+{ lib
+, pythonPackages
+, buildPythonPackage
+, tahoe-lafs-src
+, extrasNames
+
+# control how the test suite is run
+, doCheck
+}:
+let
+  pname = "tahoe-lafs";
+  version = "1.18.0.post1";
+
+  pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum  ++ deps.${extra}) [] extras;
+
+  pythonExtraDependencies = with pythonPackages; {
+    tor = [ txtorcon ];
+    i2p = [ txi2p ];
+  };
+
+  pythonPackageDependencies = with pythonPackages; [
+    attrs
+    autobahn
+    cbor2
+    click
+    collections-extended
+    cryptography
+    distro
+    eliot
+    filelock
+    foolscap
+    future
+    klein
+    magic-wormhole
+    netifaces
+    psutil
+    pycddl
+    pyrsistent
+    pyutil
+    six
+    treq
+    twisted
+    # Get the dependencies for the Twisted extras we depend on, too.
+    twisted.passthru.optional-dependencies.tls
+    twisted.passthru.optional-dependencies.conch
+    werkzeug
+    zfec
+    zope_interface
+  ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
+
+  pythonCheckDependencies = with pythonPackages; [
+    beautifulsoup4
+    fixtures
+    hypothesis
+    mock
+    paramiko
+    prometheus-client
+    pytest
+    pytest-timeout
+    pytest-twisted
+    testtools
+    towncrier
+  ];
+in
+buildPythonPackage {
+  inherit pname version;
+  src = tahoe-lafs-src;
+  propagatedBuildInputs = pythonPackageDependencies;
+
+  inherit doCheck;
+  checkInputs = pythonCheckDependencies;
+  checkPhase = ''
+    export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
+    python -m twisted.trial -j $NIX_BUILD_CORES allmydata
+  '';
+
+  meta = with lib; {
+    homepage = "https://tahoe-lafs.org/";
+    description = "secure, decentralized, fault-tolerant file store";
+    # Also TGPPL
+    license = licenses.gpl2Plus;
+  };
+}

nix/tests.nix

@@ -0,0 +1,4 @@
+# Build the package with the test suite enabled.
+args@{...}: (import ../. args).override {
+  doCheck = true;
+}

nix/txi2p.nix

@@ -0,0 +1,39 @@
+# package https://github.com/tahoe-lafs/txi2p
+#
+# if you need to update this package to a new txi2p release then
+#
+# 1. change value given to `buildPythonPackage` for `version` to match the new
+#    release
+#
+# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
+#
+# 3. run `nix-build`
+#
+# 4. there will be an error about a hash mismatch.  change the value given to
+#    `fetchPypi` for `sha256` to the "actual" hash value report.
+#
+# 5. if there are new runtime dependencies then add them to the argument list
+#    at the top.  if there are new test dependencies add them to the
+#    `checkInputs` list.
+#
+# 6. run `nix-build`.  it should succeed.  if it does not, seek assistance.
+#
+{ fetchPypi
+, buildPythonPackage
+, parsley
+, twisted
+, unittestCheckHook
+}:
+buildPythonPackage rec {
+  pname = "txi2p-tahoe";
+  version = "0.3.7";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY=";
+  };
+
+  propagatedBuildInputs = [ twisted parsley ];
+  checkInputs = [ unittestCheckHook ];
+  pythonImportsCheck = [ "parsley" "ometa"];
+}

setup.py

@@ -118,7 +118,7 @@ install_requires = [
     "attrs >= 18.2.0",
 
     # WebSocket library for twisted and asyncio
-    "autobahn < 22.4.1",  # remove this when 22.4.3 is released
+    "autobahn",
 
     # Support for Python 3 transition
     "future >= 0.18.2",
@@ -413,7 +413,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               "beautifulsoup4",
               "html5lib",
               "junitxml",
-              "tenacity",
               # Pin old version until
               # https://github.com/paramiko/paramiko/issues/1961 is fixed.
               "paramiko < 2.9",

src/allmydata/storage/http_server.py

@@ -606,7 +606,10 @@ class HTTPServer(object):
     async def allocate_buckets(self, request, authorization, storage_index):
         """Allocate buckets."""
         upload_secret = authorization[Secrets.UPLOAD]
-        info = await self._read_encoded(request, _SCHEMAS["allocate_buckets"])
+        # It's just a list of up to ~256 shares, shouldn't use many bytes.
+        info = await self._read_encoded(
+            request, _SCHEMAS["allocate_buckets"], max_size=8192
+        )
 
         # We do NOT validate the upload secret for existing bucket uploads.
         # Another upload may be happening in parallel, with a different upload
@@ -773,7 +776,11 @@ class HTTPServer(object):
         except KeyError:
             raise _HTTPError(http.NOT_FOUND)
 
-        info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"])
+        # The reason can be a string with explanation, so in theory it could be
+        # longish?
+        info = await self._read_encoded(
+            request, _SCHEMAS["advise_corrupt_share"], max_size=32768,
+        )
         bucket.advise_corrupt_share(info["reason"].encode("utf-8"))
         return b""
 
@@ -872,7 +879,11 @@ class HTTPServer(object):
         }:
             raise _HTTPError(http.NOT_FOUND)
 
-        info = await self._read_encoded(request, _SCHEMAS["advise_corrupt_share"])
+        # The reason can be a string with explanation, so in theory it could be
+        # longish?
+        info = await self._read_encoded(
+            request, _SCHEMAS["advise_corrupt_share"], max_size=32768
+        )
         self._storage_server.advise_corrupt_share(
             b"mutable", storage_index, share_number, info["reason"].encode("utf-8")
         )

src/allmydata/test/cli/test_invite.py

@@ -19,7 +19,8 @@ from ...util.jsonbytes import dumps_bytes
 from ..common_util import run_cli
 from ..no_network import GridTestMixin
 from .common import CLITestMixin
-from .wormholetesting import IWormhole, MemoryWormholeServer, TestingHelper, memory_server
+from .wormholetesting import MemoryWormholeServer, TestingHelper, memory_server, IWormhole
+
 
 # Logically:
 #   JSONable = dict[str, Union[JSONable, None, int, float, str, list[JSONable]]]

src/allmydata/test/cli/wormholetesting.py

@@ -32,14 +32,15 @@ For example::
 
 from __future__ import annotations
 
-from typing import Iterator, Optional, List, Tuple
-from collections.abc import Awaitable
+__all__ = ['MemoryWormholeServer', 'TestingHelper', 'memory_server', 'IWormhole']
+
+from typing import Iterator, Optional, List, Tuple, Any, TextIO
 from inspect import getfullargspec
 from itertools import count
 from sys import stderr
 
 from attrs import frozen, define, field, Factory
-from twisted.internet.defer import Deferred, DeferredQueue, succeed
+from twisted.internet.defer import Deferred, DeferredQueue, succeed, Awaitable
 from wormhole._interfaces import IWormhole
 from wormhole.wormhole import create
 from zope.interface import implementer
@@ -66,18 +67,18 @@ class MemoryWormholeServer(object):
 
     def create(
         self,
-        appid,
-        relay_url,
-        reactor,
-        versions={},
-        delegate=None,
-        journal=None,
-        tor=None,
-        timing=None,
-        stderr=stderr,
-        _eventual_queue=None,
-        _enable_dilate=False,
-    ):
+        appid: str,
+        relay_url: str,
+        reactor: Any,
+        versions: Any={},
+        delegate: Optional[Any]=None,
+        journal: Optional[Any]=None,
+        tor: Optional[Any]=None,
+        timing: Optional[Any]=None,
+        stderr: TextIO=stderr,
+        _eventual_queue: Optional[Any]=None,
+        _enable_dilate: bool=False,
+    ) -> _MemoryWormhole:
         """
         Create a wormhole.  It will be able to connect to other wormholes created
         by this instance (and constrained by the normal appid/relay_url
@@ -134,7 +135,7 @@ class TestingHelper(object):
         return wormhole
 
 
-def _verify():
+def _verify() -> None:
     """
     Roughly confirm that the in-memory wormhole creation function matches the
     interface of the real implementation.
@@ -145,7 +146,13 @@ def _verify():
     b = getfullargspec(MemoryWormholeServer.create)
     # I know it has a `self` argument at the beginning.  That's okay.
     b = b._replace(args=b.args[1:])
-    assert a == b, "{} != {}".format(a, b)
+
+    # Just compare the same information to check function signature
+    assert a.varkw == b.varkw
+    assert a.args == b.args
+    assert a.varargs == b.varargs
+    assert a.kwonlydefaults == b.kwonlydefaults
+    assert a.defaults == b.defaults
 
 
 _verify()
@@ -262,7 +269,7 @@ class _MemoryWormhole(object):
             return d
         return succeed(self._code)
 
-    def get_welcome(self):
+    def get_welcome(self) -> Deferred[str]:
         return succeed("welcome")
 
     def send_message(self, payload: WormholeMessage) -> None:
@@ -276,8 +283,8 @@ class _MemoryWormhole(object):
             )
         d = self._view.wormhole_by_code(self._code, exclude=self)
 
-        def got_wormhole(wormhole):
-            msg = wormhole._payload.get()
+        def got_wormhole(wormhole: _MemoryWormhole) -> Deferred[WormholeMessage]:
+            msg: Deferred[WormholeMessage] = wormhole._payload.get()
             return msg
 
         d.addCallback(got_wormhole)

src/allmydata/test/test_iputil.py

@@ -4,18 +4,13 @@ Tests for allmydata.util.iputil.
 Ported to Python 3.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from future.utils import PY2, native_str
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+from __future__ import annotations
 
 import os, socket
 import gc
+from functools import wraps
 
+from typing import TypeVar, Callable
 from testtools.matchers import (
     MatchesAll,
     IsInstance,
@@ -25,8 +20,6 @@ from testtools.matchers import (
 
 from twisted.trial import unittest
 
-from tenacity import retry, stop_after_attempt
-
 from foolscap.api import Tub
 
 from allmydata.util import iputil, gcutil
@@ -39,6 +32,43 @@ from .common import (
     SyncTestCase,
 )
 
+T = TypeVar("T", contravariant=True)
+U = TypeVar("U", covariant=True)
+
+def retry(stop: Callable[[], bool]) -> Callable[[Callable[[T], U]], Callable[[T], U]]:
+    """
+    Call a function until the predicate says to stop or the function stops
+    raising an exception.
+
+    :param stop: A callable to call after the decorated function raises an
+        exception.  The decorated function will be called again if ``stop``
+        returns ``False``.
+
+    :return: A decorator function.
+    """
+    def decorate(f: Callable[[T], U]) -> Callable[[T], U]:
+        @wraps(f)
+        def decorator(self: T) -> U:
+            while True:
+                try:
+                    return f(self)
+                except Exception:
+                    if stop():
+                        raise
+        return decorator
+    return decorate
+
+def stop_after_attempt(limit: int) -> Callable[[], bool]:
+    """
+    Stop after ``limit`` calls.
+    """
+    counter = 0
+    def check():
+        nonlocal counter
+        counter += 1
+        return counter < limit
+    return check
+
 class ListenOnUsed(unittest.TestCase):
     """Tests for listenOnUnused."""
 
@@ -127,7 +157,7 @@ class GetLocalAddressesSyncTests(SyncTestCase):
                 IsInstance(list),
                 AllMatch(
                     MatchesAll(
-                        IsInstance(native_str),
+                        IsInstance(str),
                         MatchesPredicate(
                             lambda addr: socket.inet_pton(socket.AF_INET, addr),
                             "%r is not an IPv4 address.",

src/allmydata/test/test_system.py

@@ -33,6 +33,7 @@ from allmydata.util import log, base32
 from allmydata.util.encodingutil import quote_output, unicode_to_argv
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.consumer import MemoryConsumer, download_to_data
+from allmydata.util.deferredutil import async_to_deferred
 from allmydata.interfaces import IDirectoryNode, IFileNode, \
      NoSuchChildError, NoSharesError, SDMF_VERSION, MDMF_VERSION
 from allmydata.monitor import Monitor
@@ -657,7 +658,25 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
             self.failUnlessEqual(res, NEWERDATA)
         d.addCallback(_check_download_5)
 
-        def _corrupt_shares(res):
+        # The previous checks upload a complete replacement. This uses a
+        # different API that is supposed to do a partial write at an offset.
+        @async_to_deferred
+        async def _check_write_at_offset(newnode):
+            log.msg("writing at offset")
+            start = b"abcdef"
+            expected = b"abXYef"
+            uri = self._mutable_node_1.get_uri()
+            newnode = self.clients[0].create_node_from_uri(uri)
+            await newnode.overwrite(MutableData(start))
+            version = await newnode.get_mutable_version()
+            await version.update(MutableData(b"XY"), 2)
+            result = await newnode.download_best_version()
+            self.assertEqual(result, expected)
+            # Revert to previous version
+            await newnode.overwrite(MutableData(NEWERDATA))
+        d.addCallback(_check_write_at_offset)
+
+        def _corrupt_shares(_res):
             # run around and flip bits in all but k of the shares, to test
             # the hash checks
             shares = self._find_all_shares(self.basedir)

tests.nix

@@ -1,88 +0,0 @@
-let
-  sources = import nix/sources.nix;
-in
-# See default.nix for documentation about parameters.
-{ pkgsVersion ? "nixpkgs-21.11"
-, pkgs ? import sources.${pkgsVersion} { }
-, pypiData ? sources.pypi-deps-db
-, pythonVersion ? "python39"
-, mach-nix ? import sources.mach-nix {
-    inherit pkgs pypiData;
-    python = pythonVersion;
-  }
-}@args:
-let
-  # We would like to know the test requirements but mach-nix does not directly
-  # expose this information to us.  However, it is perfectly capable of
-  # determining it if we ask right...  This is probably not meant to be a
-  # public mach-nix API but we pinned mach-nix so we can deal with mach-nix
-  # upgrade breakage in our own time.
-  mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" {
-    inherit pkgs;
-    lib = pkgs.lib;
-  };
-  tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test;
-
-  # Get the Tahoe-LAFS package itself.  This does not include test
-  # requirements and we don't ask for test requirements so that we can just
-  # re-use the normal package if it is already built.
-  tahoe-lafs = import ./. args;
-
-  # If we want to get tahoe-lafs into a Python environment with a bunch of
-  # *other* Python modules and let them interact in the usual way then we have
-  # to ask mach-nix for tahoe-lafs and those other Python modules in the same
-  # way - i.e., using `requirements`.  The other tempting mechanism,
-  # `packagesExtra`, inserts an extra layer of Python environment and prevents
-  # normal interaction between Python modules (as well as usually producing
-  # file collisions in the packages that are both runtime and test
-  # dependencies).  To get the tahoe-lafs we just built into the environment,
-  # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs
-  # from nixpkgs.
-  overridesPre = [(self: super: { inherit tahoe-lafs; })];
-  providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; };
-
-  # Make the Python environment in which we can run the tests.
-  python-env = mach-nix.mkPython {
-    # Get the packaging fixes we already know we need from putting together
-    # the runtime package.
-    inherit (tahoe-lafs.meta.mach-nix) _;
-    # Share the runtime package's provider configuration - combined with our
-    # own that causes the right tahoe-lafs to be picked up.
-    inherit providers overridesPre;
-    requirements = ''
-      # Here we pull in the Tahoe-LAFS package itself.
-      tahoe-lafs
-
-      # Unfortunately mach-nix misses all of the Python dependencies of the
-      # tahoe-lafs satisfied from nixpkgs.  Drag them in here.  This gives a
-      # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will
-      # fix this soon.
-      #
-      # https://github.com/DavHau/mach-nix/issues/123
-      # https://github.com/DavHau/mach-nix/pull/386
-      ${tahoe-lafs.requirements}
-
-      # And then all of the test-only dependencies.
-      ${builtins.concatStringsSep "\n" tests_require}
-
-      # txi2p-tahoe is another dependency with an environment marker that
-      # mach-nix doesn't automatically pick up.
-      txi2p-tahoe
-    '';
-  };
-in
-# Make a derivation that runs the unit test suite.
-pkgs.runCommand "tahoe-lafs-tests" { } ''
-  export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
-  ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
-
-  # It's not cool to put the whole _trial_temp into $out because it has weird
-  # files in it we don't want in the store.  Plus, even all of the less weird
-  # files are mostly just trash that's not meaningful if the test suite passes
-  # (which is the only way we get $out anyway).
-  #
-  # The build log itself is typically available from `nix-store --read-log` so
-  # we don't need to record that either.
-  echo "passed" >$out
-
-''