Merge remote-tracking branch 'origin/master' into 3855-immutable-http-apis-part-1

.circleci/config.yml

@@ -39,11 +39,13 @@ workflows:
       - "centos-8":
           {}
 
-      - "nixos-19-09":
+      - "nixos":
-          {}
+          name: "NixOS 21.05"
+          nixpkgs: "21.05"
 
-      - "nixos-21-05":
+      - "nixos":
-          {}
+          name: "NixOS 21.11"
+          nixpkgs: "21.11"
 
       # Test against PyPy 2.7
       - "pypy27-buster":
@@ -441,20 +443,58 @@ jobs:
         image: "tahoelafsci/fedora:29-py"
         user: "nobody"
 
-  nixos-19-09: &NIXOS
+  nixos:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+
     docker:
       # Run in a highly Nix-capable environment.
       - <<: *DOCKERHUB_AUTH
-        image: "nixorg/nix:circleci"
+        image: "nixos/nix:2.3.16"
 
     environment:
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
-      SOURCE: "nix/"
+      # allows us to push to CACHIX_NAME.  We only need this set for
+      # `cachix use` in this step.
+      CACHIX_NAME: "tahoe-lafs-opensource"
 
     steps:
-      - "checkout"
       - "run":
-          name: "Build and Test"
+          # The nixos/nix image does not include ssh.  Install it so the
+          # `checkout` step will succeed.  We also want cachix for
+          # Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            nix-env \
+              --file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
+              --install \
+              -A openssh cachix bash
+
+      - "checkout"
+
+      - run:
+          name: "Cachix setup"
+          # Record the store paths that exist before we did much.  There's no
+          # reason to cache these, they're either in the image or have to be
+          # retrieved before we can use cachix to restore from cache.
+          command: |
+            cachix use "${CACHIX_NAME}"
+            nix path-info --all > /tmp/store-path-pre-build
+
+      - "run":
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build"
           command: |
             # CircleCI build environment looks like it has a zillion and a
             # half cores.  Don't let Nix autodetect this high core count
@@ -466,17 +506,50 @@ jobs:
             # build a couple simple little dependencies that don't take
             # advantage of multiple cores and we get a little speedup by doing
             # them in parallel.
-            nix-build --cores 3 --max-jobs 2 "$SOURCE"
+            nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
 
-  nixos-21-05:
+      - "run":
-    <<: *NIXOS
+          name: "Test"
+          command: |
+            # Let it go somewhat wild for the test suite itself
+            nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
 
-    environment:
+      - run:
-      # Note this doesn't look more similar to the 19.09 NIX_PATH URL because
+          # Send any new store objects to cachix.
-      # there was some internal shuffling by the NixOS project about how they
+          name: "Push to Cachix"
-      # publish stable revisions.
+          when: "always"
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz"
+          command: |
-      SOURCE: "nix/py3.nix"
+            # Cribbed from
+            # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
+              # number of the PR being built".  Sorry, dear reader, you have
+              # guessed poorly.  It is also conditionally set based on whether
+              # this is a PR from a fork or not.
+              #
+              # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
+              echo "Skipping Cachix push for forked PR."
+            else
+              # If this *isn't* a build from a fork then we have the Cachix
+              # write key in our environment and we can push any new objects
+              # to Cachix.
+              #
+              # To decide what to push, we inspect the list of store objects
+              # that existed before and after we did most of our work.  Any
+              # that are new after the work is probably a useful thing to have
+              # around so push it to the cache.  We exclude all derivation
+              # objects (.drv files) because they're cheap to reconstruct and
+              # by the time you know their cache key you've already done all
+              # the work anyway.
+              #
+              # This shell expression for finding the objects and pushing them
+              # was from the Cachix docs:
+              #
+              # https://docs.cachix.org/continuous-integration-setup/circleci.html
+              #
+              # but they seem to have removed it now.
+              bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
+            fi
 
   typechecks:
     docker:

default.nix

@@ -0,0 +1,115 @@
+let
+  # sources.nix contains information about which versions of some of our
+  # dependencies we should use.  since we use it to pin nixpkgs and the PyPI
+  # package database, roughly all the rest of our dependencies are *also*
+  # pinned - indirectly.
+  #
+  # sources.nix is managed using a tool called `niv`.  as an example, to
+  # update to the most recent version of nixpkgs from the 21.11 maintenance
+  # release, in the top-level tahoe-lafs checkout directory you run:
+  #
+  #   niv update nixpkgs-21.11
+  #
+  # or, to update the PyPI package database -- which is necessary to make any
+  # newly released packages visible -- you likewise run:
+  #
+  #   niv update pypi-deps-db
+  #
+  # niv also supports chosing a specific revision, following a different
+  # branch, etc.  find complete documentation for the tool at
+  # https://github.com/nmattia/niv
+  sources = import nix/sources.nix;
+in
+{
+  pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
+                                # niv-managed sources data
+
+, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
+
+, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
+                                  # for dependency resolution
+
+, pythonVersion ? "python37" # a string choosing the python derivation from
+                             # nixpkgs to target
+
+, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
+                           # the dependencies of which the resulting package
+                           # will also depend on.  Include all of the runtime
+                           # extras by default because the incremental cost of
+                           # including them is a lot smaller than the cost of
+                           # re-building the whole thing to add them.
+
+, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
+                                       # the tahoe-lafs package
+    inherit pkgs pypiData;
+    python = pythonVersion;
+}
+}:
+# The project name, version, and most other metadata are automatically
+# extracted from the source.  Some requirements are not properly extracted
+# and those cases are handled below.  The version can only be extracted if
+# `setup.py update_version` has been run (this is not at all ideal but it
+# seems difficult to fix) - so for now just be sure to run that first.
+mach-nix.buildPythonPackage rec {
+  # Define the location of the Tahoe-LAFS source to be packaged.  Clean up all
+  # as many of the non-source files (eg the `.git` directory, `~` backup
+  # files, nix's own `result` symlink, etc) as possible to avoid needing to
+  # re-build when files that make no difference to the package have changed.
+  src = pkgs.lib.cleanSource ./.;
+
+  # Select whichever package extras were requested.
+  inherit extras;
+
+  # Define some extra requirements that mach-nix does not automatically detect
+  # from inspection of the source.  We typically don't need to put version
+  # constraints on any of these requirements.  The pypi-deps-db we're
+  # operating with makes dependency resolution deterministic so as long as it
+  # works once it will always work.  It could be that in the future we update
+  # pypi-deps-db and an incompatibility arises - in which case it would make
+  # sense to apply some version constraints here.
+  requirementsExtra = ''
+    # mach-nix does not yet support pyproject.toml which means it misses any
+    # build-time requirements of our dependencies which are declared in such a
+    # file.  Tell it about them here.
+    setuptools_rust
+
+    # mach-nix does not yet parse environment markers correctly.  It misses
+    # all of our requirements which have an environment marker.  Duplicate them
+    # here.
+    foolscap
+    eliot
+    pyrsistent
+  '';
+
+  # Specify where mach-nix should find packages for our Python dependencies.
+  # There are some reasonable defaults so we only need to specify certain
+  # packages where the default configuration runs into some issue.
+  providers = {
+    # Through zfec 1.5.5 the wheel has an incorrect runtime dependency
+    # declared on argparse, not available for recent versions of Python 3.
+    # Force mach-nix to use the sdist instead.  This allows us to apply a
+    # patch that removes the offending declaration.
+    zfec = "sdist";
+  };
+
+  # Define certain overrides to the way Python dependencies are built.
+  _ = {
+    # Apply the argparse declaration fix to zfec sdist.
+    zfec.patches = with pkgs; [
+      (fetchpatch {
+        name = "fix-argparse.patch";
+        url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch";
+        sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x";
+      })
+    ];
+
+    # Remove a click-default-group patch for a test suite problem which no
+    # longer applies because the project apparently no longer has a test suite
+    # in its source distribution.
+    click-default-group.patches = [];
+  };
+
+  passthru.meta.mach-nix = {
+    inherit providers _;
+  };
+}

nix/autobahn.nix

@@ -1,34 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi, isPy3k,
-  six, txaio, twisted, zope_interface, cffi, futures,
-  mock, pytest, cryptography, pynacl
-}:
-buildPythonPackage rec {
-  pname = "autobahn";
-  version = "19.8.1";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304";
-  };
-
-  propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++
-    (lib.optionals (!isPy3k) [ futures ]);
-
-  checkInputs = [ mock pytest ];
-  checkPhase = ''
-    runHook preCheck
-    USE_TWISTED=true py.test $out
-    runHook postCheck
-  '';
-
-  # Tests do no seem to be compatible yet with pytest 5.1
-  # https://github.com/crossbario/autobahn-python/issues/1235
-  doCheck = false;
-
-  meta = with lib; {
-    description = "WebSocket and WAMP in Python for Twisted and asyncio.";
-    homepage    = "https://crossbar.io/autobahn";
-    license     = licenses.mit;
-    maintainers = with maintainers; [ nand0p ];
-  };
-}

nix/cbor2.nix

@@ -1,20 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi, setuptools_scm }:
-buildPythonPackage rec {
-  pname = "cbor2";
-  version = "5.2.0";
-
-  src = fetchPypi {
-    sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3";
-    inherit pname version;
-  };
-
-  doCheck = false;
-
-  propagatedBuildInputs = [ setuptools_scm ];
-
-  meta = with lib; {
-    homepage = https://github.com/agronholm/cbor2;
-    description = "CBOR encoder/decoder";
-    license = licenses.mit;
-  };
-}

nix/collections-extended.nix

@@ -1,19 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi }:
-buildPythonPackage rec {
-  pname = "collections-extended";
-  version = "1.0.3";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw";
-  };
-
-  # Tests aren't in tarball, for 1.0.3 at least.
-  doCheck = false;
-
-  meta = with lib; {
-    homepage = https://github.com/mlenzen/collections-extended;
-    description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict";
-    license = licenses.asl20;
-  };
-}

nix/default.nix

@@ -1,7 +0,0 @@
-# This is the main entrypoint for the Tahoe-LAFS derivation.
-{ pkgs ? import <nixpkgs> { } }:
-# Add our Python packages to nixpkgs to simplify the expression for the
-# Tahoe-LAFS derivation.
-let pkgs' = pkgs.extend (import ./overlays.nix);
-# Evaluate the expression for our Tahoe-LAFS derivation.
-in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { }

nix/eliot.nix

@@ -1,31 +0,0 @@
-{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons
-, hypothesis, testtools, pytest }:
-buildPythonPackage rec {
-  pname = "eliot";
-  version = "1.7.0";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8";
-  };
-
-  postPatch = ''
-    substituteInPlace setup.py \
-      --replace "boltons >= 19.0.1" boltons
-  '';
-
-  # A seemingly random subset of the test suite fails intermittently.  After
-  # Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if
-  # the test suite continues to fail, maybe it will be more likely that we can
-  # have upstream fix it for us.
-  doCheck = false;
-
-  checkInputs = [ testtools pytest hypothesis ];
-  propagatedBuildInputs = [ zope_interface pyrsistent boltons ];
-
-  meta = with lib; {
-    homepage = https://github.com/itamarst/eliot/;
-    description = "Logging library that tells you why it happened";
-    license = licenses.asl20;
-  };
-}

nix/future.nix

@@ -1,35 +0,0 @@
-{ lib
-, buildPythonPackage
-, fetchPypi
-}:
-
-buildPythonPackage rec {
-  pname = "future";
-  version = "0.18.2";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi";
-  };
-
-  doCheck = false;
-
-  meta = {
-    description = "Clean single-source support for Python 3 and 2";
-    longDescription = ''
-      python-future is the missing compatibility layer between Python 2 and
-      Python 3. It allows you to use a single, clean Python 3.x-compatible
-      codebase to support both Python 2 and Python 3 with minimal overhead.
-
-      It provides future and past packages with backports and forward ports
-      of features from Python 3 and 2. It also comes with futurize and
-      pasteurize, customized 2to3-based scripts that helps you to convert
-      either Py2 or Py3 code easily to support both Python 2 and 3 in a
-      single clean Py3-style codebase, module by module.
-    '';
-    homepage = https://python-future.org;
-    downloadPage = https://github.com/PythonCharmers/python-future/releases;
-    license = with lib.licenses; [ mit ];
-    maintainers = with lib.maintainers; [ prikhi ];
-  };
-}

nix/overlays.nix

@@ -1,36 +0,0 @@
-self: super: {
-  python27 = super.python27.override {
-    packageOverrides = python-self: python-super: {
-      # eliot is not part of nixpkgs at all at this time.
-      eliot = python-self.pythonPackages.callPackage ./eliot.nix { };
-
-      # NixOS autobahn package has trollius as a dependency, although
-      # it is optional. Trollius is unmaintained and fails on CI.
-      autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { };
-
-      # Porting to Python 3 is greatly aided by the future package.  A
-      # slightly newer version than appears in nixos 19.09 is helpful.
-      future = python-super.pythonPackages.callPackage ./future.nix { };
-
-      # Need version of pyutil that supports Python 3. The version in 19.09
-      # is too old.
-      pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { };
-
-      # Need a newer version of Twisted, too.
-      twisted = python-super.pythonPackages.callPackage ./twisted.nix { };
-
-      # collections-extended is not part of nixpkgs at this time.
-      collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { };
-
-      # cbor2 is not part of nixpkgs at this time.
-      cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { };
-    };
-  };
-
-  python39 = super.python39.override {
-    packageOverrides = python-self: python-super: {
-      # collections-extended is not part of nixpkgs at this time.
-      collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { };
-    };
-  };
-}

nix/py3.nix

@@ -1,7 +0,0 @@
-# This is the main entrypoint for the Tahoe-LAFS derivation.
-{ pkgs ? import <nixpkgs> { } }:
-# Add our Python packages to nixpkgs to simplify the expression for the
-# Tahoe-LAFS derivation.
-let pkgs' = pkgs.extend (import ./overlays.nix);
-# Evaluate the expression for our Tahoe-LAFS derivation.
-in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { }

nix/pyutil.nix

@@ -1,48 +0,0 @@
-{ stdenv
-, buildPythonPackage
-, fetchPypi
-, setuptoolsDarcs
-, setuptoolsTrial
-, simplejson
-, twisted
-, isPyPy
-}:
-
-buildPythonPackage rec {
-  pname = "pyutil";
-  version = "3.3.0";
-
-  src = fetchPypi {
-    inherit pname version;
-    sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848";
-  };
-
-  buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []);
-  propagatedBuildInputs = [ twisted ];
-
-  # Tests fail because they try to write new code into the twisted
-  # package, apparently some kind of plugin.
-  doCheck = false;
-
-  prePatch = stdenv.lib.optionalString isPyPy ''
-    grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g"
-  '';
-
-  meta = with stdenv.lib; {
-    description = "Pyutil, a collection of mature utilities for Python programmers";
-
-    longDescription = ''
-      These are a few data structures, classes and functions which
-      we've needed over many years of Python programming and which
-      seem to be of general use to other Python programmers. Many of
-      the modules that have existed in pyutil over the years have
-      subsequently been obsoleted by new features added to the
-      Python language or its standard library, thus showing that
-      we're not alone in wanting tools like these.
-    '';
-
-    homepage = "http://allmydata.org/trac/pyutil";
-    license = licenses.gpl2Plus;
-  };
-
-}

nix/sources.json

@@ -0,0 +1,62 @@
+{
+    "mach-nix": {
+        "branch": "master",
+        "description": "Create highly reproducible python environments",
+        "homepage": "",
+        "owner": "davhau",
+        "repo": "mach-nix",
+        "rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b",
+        "sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf",
+        "type": "tarball",
+        "url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "niv": {
+        "branch": "master",
+        "description": "Easy dependency management for Nix projects",
+        "homepage": "https://github.com/nmattia/niv",
+        "owner": "nmattia",
+        "repo": "niv",
+        "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c",
+        "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm",
+        "type": "tarball",
+        "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-21.05": {
+        "branch": "nixos-21.05",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
+        "sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs-21.11": {
+        "branch": "nixos-21.11",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca",
+        "sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f",
+        "type": "tarball",
+        "url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "pypi-deps-db": {
+        "branch": "master",
+        "description": "Probably the most complete python dependency database",
+        "homepage": "",
+        "owner": "DavHau",
+        "repo": "pypi-deps-db",
+        "rev": "0f6de8bf1f186c275af862ec9667abb95aae8542",
+        "sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y",
+        "type": "tarball",
+        "url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

nix/sources.nix

@@ -0,0 +1,174 @@
+# This file has been generated by Niv.
+
+let
+
+  #
+  # The fetchers. fetch_<type> fetches specs of type <type>.
+  #
+
+  fetch_file = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchurl { inherit (spec) url sha256; name = name'; }
+      else
+        pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
+
+  fetch_tarball = pkgs: name: spec:
+    let
+      name' = sanitizeName name + "-src";
+    in
+      if spec.builtin or true then
+        builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
+      else
+        pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
+
+  fetch_git = name: spec:
+    let
+      ref =
+        if spec ? ref then spec.ref else
+          if spec ? branch then "refs/heads/${spec.branch}" else
+            if spec ? tag then "refs/tags/${spec.tag}" else
+              abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
+    in
+      builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
+
+  fetch_local = spec: spec.path;
+
+  fetch_builtin-tarball = name: throw
+    ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=tarball -a builtin=true'';
+
+  fetch_builtin-url = name: throw
+    ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
+        $ niv modify ${name} -a type=file -a builtin=true'';
+
+  #
+  # Various helpers
+  #
+
+  # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
+  sanitizeName = name:
+    (
+      concatMapStrings (s: if builtins.isList s then "-" else s)
+        (
+          builtins.split "[^[:alnum:]+._?=-]+"
+            ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
+        )
+    );
+
+  # The set of packages used when specs are fetched using non-builtins.
+  mkPkgs = sources: system:
+    let
+      sourcesNixpkgs =
+        import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
+      hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
+      hasThisAsNixpkgsPath = <nixpkgs> == ./.;
+    in
+      if builtins.hasAttr "nixpkgs" sources
+      then sourcesNixpkgs
+      else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
+        import <nixpkgs> {}
+      else
+        abort
+          ''
+            Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+            add a package called "nixpkgs" to your sources.json.
+          '';
+
+  # The actual fetching function.
+  fetch = pkgs: name: spec:
+
+    if ! builtins.hasAttr "type" spec then
+      abort "ERROR: niv spec ${name} does not have a 'type' attribute"
+    else if spec.type == "file" then fetch_file pkgs name spec
+    else if spec.type == "tarball" then fetch_tarball pkgs name spec
+    else if spec.type == "git" then fetch_git name spec
+    else if spec.type == "local" then fetch_local spec
+    else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
+    else if spec.type == "builtin-url" then fetch_builtin-url name
+    else
+      abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
+
+  # If the environment variable NIV_OVERRIDE_${name} is set, then use
+  # the path directly as opposed to the fetched source.
+  replace = name: drv:
+    let
+      saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
+      ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
+    in
+      if ersatz == "" then drv else
+        # this turns the string into an actual Nix path (for both absolute and
+        # relative paths)
+        if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
+
+  # Ports of functions for older nix versions
+
+  # a Nix version of mapAttrs if the built-in doesn't exist
+  mapAttrs = builtins.mapAttrs or (
+    f: set: with builtins;
+    listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
+  );
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
+  range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
+  stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
+
+  # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
+  stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
+  concatMapStrings = f: list: concatStrings (map f list);
+  concatStrings = builtins.concatStringsSep "";
+
+  # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
+  optionalAttrs = cond: as: if cond then as else {};
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchTarball;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl = { url, name ? null, sha256 }@attrs:
+    let
+      inherit (builtins) lessThan nixVersion fetchurl;
+    in
+      if lessThan nixVersion "1.12" then
+        fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
+      else
+        fetchurl attrs;
+
+  # Create the final "sources" from the config
+  mkSources = config:
+    mapAttrs (
+      name: spec:
+        if builtins.hasAttr "outPath" spec
+        then abort
+          "The values in sources.json should not have an 'outPath' attribute"
+        else
+          spec // { outPath = replace name (fetch config.pkgs name spec); }
+    ) config.sources;
+
+  # The "config" used by the fetchers
+  mkConfig =
+    { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
+    , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
+    , system ? builtins.currentSystem
+    , pkgs ? mkPkgs sources system
+    }: rec {
+      # The sources, i.e. the attribute set of spec name to spec
+      inherit sources;
+
+      # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
+      inherit pkgs;
+    };
+
+in
+mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

nix/tahoe-lafs.nix

@@ -1,126 +0,0 @@
-{ fetchFromGitHub, lib
-, git, python
-, twisted, foolscap, zfec
-, setuptools, setuptoolsTrial, pyasn1, zope_interface
-, service-identity, pyyaml, magic-wormhole, treq, appdirs
-, beautifulsoup4, eliot, autobahn, cryptography, netifaces
-, html5lib, pyutil, distro, configparser, klein, werkzeug, cbor2
-}:
-python.pkgs.buildPythonPackage rec {
-  # Most of the time this is not exactly the release version (eg 1.17.1).
-  # Give it a `post` component to make it look newer than the release version
-  # and we'll bump this up at the time of each release.
-  #
-  # It's difficult to read the version from Git the way the Python code does
-  # for two reasons.  First, doing so involves populating the Nix expression
-  # with values from the source.  Nix calls this "import from derivation" or
-  # "IFD" (<https://nixos.wiki/wiki/Import_From_Derivation>).  This is
-  # discouraged in most cases - including this one, I think.  Second, the
-  # Python code reads the contents of `.git` to determine its version.  `.git`
-  # is not a reproducable artifact (in the sense of "reproducable builds") so
-  # it is excluded from the source tree by default.  When it is included, the
-  # package tends to be frequently spuriously rebuilt.
-  version = "1.17.1.post1";
-  name = "tahoe-lafs-${version}";
-  src = lib.cleanSourceWith {
-    src = ../.;
-    filter = name: type:
-      let
-        basename = baseNameOf name;
-
-        split = lib.splitString ".";
-        join = builtins.concatStringsSep ".";
-        ext = join (builtins.tail (split basename));
-
-        # Build up a bunch of knowledge about what kind of file this is.
-        isTox = type == "directory" && basename == ".tox";
-        isTrialTemp = type == "directory" && basename == "_trial_temp";
-        isVersion = basename == "_version.py";
-        isBytecode = ext == "pyc" || ext == "pyo";
-        isBackup = lib.hasSuffix "~" basename;
-        isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename;
-        isSymlink = type == "symlink";
-        isGit = type == "directory" && basename == ".git";
-      in
-      # Exclude all these things
-      ! (isTox
-      || isTrialTemp
-      || isVersion
-      || isBytecode
-      || isBackup
-      || isTemporary
-      || isSymlink
-      || isGit
-      );
-  };
-
-  postPatch = ''
-    # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around
-    # that.
-    for i in $(find src/allmydata/test -type f)
-    do
-      sed -i "$i" -e"s/localhost/127.0.0.1/g"
-    done
-
-    # Some tests are flaky or fail to skip when dependencies are missing.
-    # This list is over-zealous because it's more work to disable individual
-    # tests with in a module.
-
-    # Many of these tests don't properly skip when i2p or tor dependencies are
-    # not supplied (and we are not supplying them).
-    rm src/allmydata/test/test_i2p_provider.py
-    rm src/allmydata/test/test_connections.py
-    rm src/allmydata/test/cli/test_create.py
-
-    # Generate _version.py ourselves since we can't rely on the Python code
-    # extracting the information from the .git directory we excluded.
-    cat > src/allmydata/_version.py <<EOF
-
-# This _version.py is generated from metadata by nix/tahoe-lafs.nix.
-
-__pkgname__ = "tahoe-lafs"
-real_version = "${version}"
-full_version = "${version}"
-branch = "master"
-verstr = "${version}"
-__version__ = verstr
-EOF
-'';
-
-
-  nativeBuildInputs = [
-    git
-  ];
-
-  propagatedBuildInputs = with python.pkgs; [
-    twisted foolscap zfec appdirs
-    setuptoolsTrial pyasn1 zope_interface
-    service-identity pyyaml magic-wormhole
-    eliot autobahn cryptography netifaces setuptools
-    future pyutil distro configparser collections-extended
-    klein werkzeug cbor2 treq
-  ];
-
-  checkInputs = with python.pkgs; [
-    hypothesis
-    testtools
-    fixtures
-    beautifulsoup4
-    html5lib
-    tenacity
-    prometheus_client
-  ];
-
-  checkPhase = ''
-    if ! $out/bin/tahoe --version | grep --fixed-strings "${version}"; then
-      echo "Package version:"
-      $out/bin/tahoe --version
-      echo "Did not contain expected:"
-      echo "${version}"
-      exit 1
-    else
-      echo "Version string contained expected value \"${version}.\""
-    fi
-    ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
-  '';
-}

nix/twisted.nix

@@ -1,63 +0,0 @@
-{ stdenv
-, buildPythonPackage
-, fetchPypi
-, python
-, zope_interface
-, incremental
-, automat
-, constantly
-, hyperlink
-, pyhamcrest
-, attrs
-, pyopenssl
-, service-identity
-, setuptools
-, idna
-, bcrypt
-}:
-buildPythonPackage rec {
-  pname = "Twisted";
-  version = "19.10.0";
-
-  src = fetchPypi {
-    inherit pname version;
-    extension = "tar.bz2";
-    sha256 = "7394ba7f272ae722a74f3d969dcf599bc4ef093bc392038748a490f1724a515d";
-  };
-
-  propagatedBuildInputs = [ zope_interface incremental automat constantly hyperlink pyhamcrest attrs setuptools bcrypt ];
-
-  passthru.extras.tls = [ pyopenssl service-identity idna ];
-
-  # Patch t.p._inotify to point to libc. Without this,
-  # twisted.python.runtime.platform.supportsINotify() == False
-  patchPhase = stdenv.lib.optionalString stdenv.isLinux ''
-    substituteInPlace src/twisted/python/_inotify.py --replace \
-      "ctypes.util.find_library('c')" "'${stdenv.glibc.out}/lib/libc.so.6'"
-  '';
-
-  # Generate Twisted's plug-in cache.  Twisted users must do it as well.  See
-  # http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3
-  # and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for
-  # details.
-  postFixup = ''
-    $out/bin/twistd --help > /dev/null
-  '';
-
-  checkPhase = ''
-    ${python.interpreter} -m unittest discover -s twisted/test
-  '';
-  # Tests require network
-  doCheck = false;
-
-  meta = with stdenv.lib; {
-    homepage = https://twistedmatrix.com/;
-    description = "Twisted, an event-driven networking engine written in Python";
-    longDescription = ''
-      Twisted is an event-driven networking engine written in Python
-      and licensed under the MIT license.
-    '';
-    license = licenses.mit;
-    maintainers = [ ];
-  };
-}

src/allmydata/test/mutable/test_checker.py

@@ -10,14 +10,15 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
 from foolscap.api import flushEventualQueue
 from allmydata.monitor import Monitor
 from allmydata.mutable.common import CorruptShareError
 from .util import PublishMixin, corrupt, CheckerMixin
 
-class Checker(unittest.TestCase, CheckerMixin, PublishMixin):
+class Checker(AsyncTestCase, CheckerMixin, PublishMixin):
     def setUp(self):
+        super(Checker, self).setUp()
         return self.publish_one()
 
 

src/allmydata/test/mutable/test_datahandle.py

@@ -10,11 +10,14 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import SyncTestCase
 from allmydata.mutable.publish import MutableData
+from testtools.matchers import Equals, HasLength
 
-class DataHandle(unittest.TestCase):
+
+class DataHandle(SyncTestCase):
     def setUp(self):
+        super(DataHandle, self).setUp()
         self.test_data = b"Test Data" * 50000
         self.uploadable = MutableData(self.test_data)
 
@@ -26,13 +29,13 @@ class DataHandle(unittest.TestCase):
             data = b"".join(data)
             start = i
             end = i + chunk_size
-            self.failUnlessEqual(data, self.test_data[start:end])
+            self.assertThat(data, Equals(self.test_data[start:end]))
 
 
     def test_datahandle_get_size(self):
         actual_size = len(self.test_data)
         size = self.uploadable.get_size()
-        self.failUnlessEqual(size, actual_size)
+        self.assertThat(size, Equals(actual_size))
 
 
     def test_datahandle_get_size_out_of_order(self):
@@ -40,14 +43,14 @@ class DataHandle(unittest.TestCase):
         # disturbing the location of the seek pointer.
         chunk_size = 100
         data = self.uploadable.read(chunk_size)
-        self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size])
+        self.assertThat(b"".join(data), Equals(self.test_data[:chunk_size]))
 
         # Now get the size.
         size = self.uploadable.get_size()
-        self.failUnlessEqual(size, len(self.test_data))
+        self.assertThat(self.test_data, HasLength(size))
 
         # Now get more data. We should be right where we left off.
         more_data = self.uploadable.read(chunk_size)
         start = chunk_size
         end = chunk_size * 2
-        self.failUnlessEqual(b"".join(more_data), self.test_data[start:end])
+        self.assertThat(b"".join(more_data), Equals(self.test_data[start:end]))

src/allmydata/test/mutable/test_different_encoding.py

@@ -10,11 +10,12 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
 from .util import FakeStorage, make_nodemaker
 
-class DifferentEncoding(unittest.TestCase):
+class DifferentEncoding(AsyncTestCase):
     def setUp(self):
+        super(DifferentEncoding, self).setUp()
         self._storage = s = FakeStorage()
         self.nodemaker = make_nodemaker(s)
 

src/allmydata/test/mutable/test_exceptions.py

@@ -11,12 +11,14 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import SyncTestCase
 from allmydata.mutable.common import NeedMoreDataError, UncoordinatedWriteError
 
-class Exceptions(unittest.TestCase):
+
+class Exceptions(SyncTestCase):
     def test_repr(self):
         nmde = NeedMoreDataError(100, 50, 100)
-        self.failUnless("NeedMoreDataError" in repr(nmde), repr(nmde))
+        self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde))
+        self.assertTrue("NeedMoreDataError" in repr(nmde), msg=repr(nmde))
         ucwe = UncoordinatedWriteError()
-        self.failUnless("UncoordinatedWriteError" in repr(ucwe), repr(ucwe))
+        self.assertTrue("UncoordinatedWriteError" in repr(ucwe), msg=repr(ucwe))

src/allmydata/test/mutable/test_filehandle.py

@@ -12,11 +12,13 @@ if PY2:
 
 import os
 from io import BytesIO
-from twisted.trial import unittest
+from ..common import SyncTestCase
 from allmydata.mutable.publish import MutableFileHandle
 
-class FileHandle(unittest.TestCase):
+
+class FileHandle(SyncTestCase):
     def setUp(self):
+        super(FileHandle, self).setUp()
         self.test_data = b"Test Data" * 50000
         self.sio = BytesIO(self.test_data)
         self.uploadable = MutableFileHandle(self.sio)

src/allmydata/test/mutable/test_filenode.py

@@ -12,7 +12,14 @@ if PY2:
 
 from six.moves import cStringIO as StringIO
 from twisted.internet import defer, reactor
-from twisted.trial import unittest
+from ..common import AsyncBrokenTestCase
+from testtools.matchers import (
+    Equals,
+    Contains,
+    HasLength,
+    Is,
+    IsInstance,
+)
 from allmydata import uri, client
 from allmydata.util.consumer import MemoryConsumer
 from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION, DownloadStopped
@@ -29,12 +36,13 @@ from .util import (
     make_peer,
 )
 
-class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
+class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin):
     # this used to be in Publish, but we removed the limit. Some of
     # these tests test whether the new code correctly allows files
     # larger than the limit.
     OLD_MAX_SEGMENT_SIZE = 3500000
     def setUp(self):
+        super(Filenode, self).setUp()
         self._storage = FakeStorage()
         self._peers = list(
             make_peer(self._storage, n)
@@ -48,12 +56,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
     def test_create(self):
         d = self.nodemaker.create_mutable_file()
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
-            self.failUnlessEqual(n.get_storage_index(), n._storage_index)
+            self.assertThat(n.get_storage_index(), Equals(n._storage_index))
             sb = self.nodemaker.storage_broker
             peer0 = sorted(sb.get_all_serverids())[0]
             shnums = self._storage._peers[peer0].keys()
-            self.failUnlessEqual(len(shnums), 1)
+            self.assertThat(shnums, HasLength(1))
         d.addCallback(_created)
         return d
 
@@ -61,12 +69,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
     def test_create_mdmf(self):
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
-            self.failUnlessEqual(n.get_storage_index(), n._storage_index)
+            self.assertThat(n.get_storage_index(), Equals(n._storage_index))
             sb = self.nodemaker.storage_broker
             peer0 = sorted(sb.get_all_serverids())[0]
             shnums = self._storage._peers[peer0].keys()
-            self.failUnlessEqual(len(shnums), 1)
+            self.assertThat(shnums, HasLength(1))
         d.addCallback(_created)
         return d
 
@@ -80,7 +88,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda ignored, v=v:
                 self.nodemaker.create_mutable_file(version=v))
             def _created(n):
-                self.failUnless(isinstance(n, MutableFileNode))
+                self.assertThat(n, IsInstance(MutableFileNode))
                 self._node = n
                 return n
             d.addCallback(_created)
@@ -89,19 +97,19 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda ignored:
                 self._node.download_best_version())
             d.addCallback(lambda contents:
-                self.failUnlessEqual(contents, b"Contents" * 50000))
+                self.assertThat(contents, Equals(b"Contents" * 50000)))
         return d
 
     def test_max_shares(self):
         self.nodemaker.default_encoding_parameters['n'] = 255
         d = self.nodemaker.create_mutable_file(version=SDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
-            self.failUnlessEqual(n.get_storage_index(), n._storage_index)
+            self.assertThat(n.get_storage_index(), Equals(n._storage_index))
             sb = self.nodemaker.storage_broker
             num_shares = sum([len(self._storage._peers[x].keys()) for x \
                               in sb.get_all_serverids()])
-            self.failUnlessEqual(num_shares, 255)
+            self.assertThat(num_shares, Equals(255))
             self._node = n
             return n
         d.addCallback(_created)
@@ -113,7 +121,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             self._node.download_best_version())
         # ...and check to make sure everything went okay.
         d.addCallback(lambda contents:
-            self.failUnlessEqual(b"contents" * 50000, contents))
+            self.assertThat(b"contents" * 50000, Equals(contents)))
         return d
 
     def test_max_shares_mdmf(self):
@@ -121,12 +129,12 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         self.nodemaker.default_encoding_parameters['n'] = 255
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
-            self.failUnlessEqual(n.get_storage_index(), n._storage_index)
+            self.assertThat(n.get_storage_index(), Equals(n._storage_index))
             sb = self.nodemaker.storage_broker
             num_shares = sum([len(self._storage._peers[x].keys()) for x \
                               in sb.get_all_serverids()])
-            self.failUnlessEqual(num_shares, 255)
+            self.assertThat(num_shares, Equals(255))
             self._node = n
             return n
         d.addCallback(_created)
@@ -135,20 +143,20 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         d.addCallback(lambda ignored:
             self._node.download_best_version())
         d.addCallback(lambda contents:
-            self.failUnlessEqual(contents, b"contents" * 50000))
+            self.assertThat(contents, Equals(b"contents" * 50000)))
         return d
 
     def test_mdmf_filenode_cap(self):
         # Test that an MDMF filenode, once created, returns an MDMF URI.
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
             cap = n.get_cap()
-            self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI))
+            self.assertThat(cap, IsInstance(uri.WriteableMDMFFileURI))
             rcap = n.get_readcap()
-            self.failUnless(isinstance(rcap, uri.ReadonlyMDMFFileURI))
+            self.assertThat(rcap, IsInstance(uri.ReadonlyMDMFFileURI))
             vcap = n.get_verify_cap()
-            self.failUnless(isinstance(vcap, uri.MDMFVerifierURI))
+            self.assertThat(vcap, IsInstance(uri.MDMFVerifierURI))
         d.addCallback(_created)
         return d
 
@@ -158,13 +166,13 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         # filenode given an MDMF cap.
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
             s = n.get_uri()
-            self.failUnless(s.startswith(b"URI:MDMF"))
+            self.assertTrue(s.startswith(b"URI:MDMF"))
             n2 = self.nodemaker.create_from_cap(s)
-            self.failUnless(isinstance(n2, MutableFileNode))
+            self.assertThat(n2, IsInstance(MutableFileNode))
-            self.failUnlessEqual(n.get_storage_index(), n2.get_storage_index())
+            self.assertThat(n.get_storage_index(), Equals(n2.get_storage_index()))
-            self.failUnlessEqual(n.get_uri(), n2.get_uri())
+            self.assertThat(n.get_uri(), Equals(n2.get_uri()))
         d.addCallback(_created)
         return d
 
@@ -172,13 +180,13 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
     def test_create_from_mdmf_readcap(self):
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
             s = n.get_readonly_uri()
             n2 = self.nodemaker.create_from_cap(s)
-            self.failUnless(isinstance(n2, MutableFileNode))
+            self.assertThat(n2, IsInstance(MutableFileNode))
 
             # Check that it's a readonly node
-            self.failUnless(n2.is_readonly())
+            self.assertTrue(n2.is_readonly())
         d.addCallback(_created)
         return d
 
@@ -191,10 +199,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
         def _created(n):
             self.uri = n.get_uri()
-            self.failUnlessEqual(n._protocol_version, MDMF_VERSION)
+            self.assertThat(n._protocol_version, Equals(MDMF_VERSION))
 
             n2 = self.nodemaker.create_from_cap(self.uri)
-            self.failUnlessEqual(n2._protocol_version, MDMF_VERSION)
+            self.assertThat(n2._protocol_version, Equals(MDMF_VERSION))
         d.addCallback(_created)
         return d
 
@@ -203,14 +211,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         n = MutableFileNode(None, None, {"k": 3, "n": 10}, None)
         calls = []
         def _callback(*args, **kwargs):
-            self.failUnlessEqual(args, (4,) )
+            self.assertThat(args, Equals((4,)))
-            self.failUnlessEqual(kwargs, {"foo": 5})
+            self.assertThat(kwargs, Equals({"foo": 5}))
             calls.append(1)
             return 6
         d = n._do_serialized(_callback, 4, foo=5)
         def _check_callback(res):
-            self.failUnlessEqual(res, 6)
+            self.assertThat(res, Equals(6))
-            self.failUnlessEqual(calls, [1])
+            self.assertThat(calls, Equals([1]))
         d.addCallback(_check_callback)
 
         def _errback():
@@ -227,26 +235,26 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.get_servermap(MODE_READ))
             d.addCallback(lambda smap: smap.dump(StringIO()))
             d.addCallback(lambda sio:
-                          self.failUnless("3-of-10" in sio.getvalue()))
+                          self.assertTrue("3-of-10" in sio.getvalue()))
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1")))
-            d.addCallback(lambda res: self.failUnlessIdentical(res, None))
+            d.addCallback(lambda res: self.assertThat(res, Is(None)))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1")))
             d.addCallback(lambda res: n.get_size_of_best_version())
             d.addCallback(lambda size:
-                          self.failUnlessEqual(size, len(b"contents 1")))
+                          self.assertThat(size, Equals(len(b"contents 1"))))
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2")))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2")))
             d.addCallback(lambda res: n.get_servermap(MODE_WRITE))
             d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3")))
             d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING))
             d.addCallback(lambda smap:
                           n.download_version(smap,
                                              smap.best_recoverable_version()))
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3")))
             # test a file that is large enough to overcome the
             # mapupdate-to-retrieve data caching (i.e. make the shares larger
             # than the default readsize, which is 2000 bytes). A 15kB file
@@ -254,7 +262,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.overwrite(MutableData(b"large size file" * 1000)))
             d.addCallback(lambda res: n.download_best_version())
             d.addCallback(lambda res:
-                          self.failUnlessEqual(res, b"large size file" * 1000))
+                          self.assertThat(res, Equals(b"large size file" * 1000)))
             return d
         d.addCallback(_created)
         return d
@@ -268,7 +276,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
                 n.get_servermap(MODE_READ))
             def _then(servermap):
                 dumped = servermap.dump(StringIO())
-                self.failUnlessIn("3-of-10", dumped.getvalue())
+                self.assertThat(dumped.getvalue(), Contains("3-of-10"))
             d.addCallback(_then)
             # Now overwrite the contents with some new contents. We want
             # to make them big enough to force the file to be uploaded
@@ -280,7 +288,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda ignored:
                 n.download_best_version())
             d.addCallback(lambda data:
-                self.failUnlessEqual(data, big_contents))
+                self.assertThat(data, Equals(big_contents)))
             # Overwrite the contents again with some new contents. As
             # before, they need to be big enough to force multiple
             # segments, so that we make the downloader deal with
@@ -292,7 +300,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda ignored:
                 n.download_best_version())
             d.addCallback(lambda data:
-                self.failUnlessEqual(data, bigger_contents))
+                self.assertThat(data, Equals(bigger_contents)))
             return d
         d.addCallback(_created)
         return d
@@ -323,7 +331,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         # Now we'll retrieve it into a pausing consumer.
         c = PausingConsumer()
         d = version.read(c)
-        d.addCallback(lambda ign: self.failUnlessEqual(c.size, len(data)))
+        d.addCallback(lambda ign: self.assertThat(c.size, Equals(len(data))))
 
         c2 = PausingAndStoppingConsumer()
         d.addCallback(lambda ign:
@@ -360,14 +368,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             self.uri = node.get_uri()
             # also confirm that the cap has no extension fields
             pieces = self.uri.split(b":")
-            self.failUnlessEqual(len(pieces), 4)
+            self.assertThat(pieces, HasLength(4))
 
             return node.overwrite(MutableData(b"contents1" * 100000))
         def _then(ignored):
             node = self.nodemaker.create_from_cap(self.uri)
             return node.download_best_version()
         def _downloaded(data):
-            self.failUnlessEqual(data, b"contents1" * 100000)
+            self.assertThat(data, Equals(b"contents1" * 100000))
         d.addCallback(_created)
         d.addCallback(_then)
         d.addCallback(_downloaded)
@@ -397,11 +405,11 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         d = self.nodemaker.create_mutable_file(upload1)
         def _created(n):
             d = n.download_best_version()
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1")))
             upload2 = MutableData(b"contents 2")
             d.addCallback(lambda res: n.overwrite(upload2))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2")))
             return d
         d.addCallback(_created)
         return d
@@ -415,15 +423,15 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         def _created(n):
             d = n.download_best_version()
             d.addCallback(lambda data:
-                self.failUnlessEqual(data, initial_contents))
+                self.assertThat(data, Equals(initial_contents)))
             uploadable2 = MutableData(initial_contents + b"foobarbaz")
             d.addCallback(lambda ignored:
                 n.overwrite(uploadable2))
             d.addCallback(lambda ignored:
                 n.download_best_version())
             d.addCallback(lambda data:
-                self.failUnlessEqual(data, initial_contents +
+                self.assertThat(data, Equals(initial_contents +
-                                           b"foobarbaz"))
+                                           b"foobarbaz")))
             return d
         d.addCallback(_created)
         return d
@@ -431,33 +439,33 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
     def test_create_with_initial_contents_function(self):
         data = b"initial contents"
         def _make_contents(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
             key = n.get_writekey()
-            self.failUnless(isinstance(key, bytes), key)
+            self.assertTrue(isinstance(key, bytes), key)
-            self.failUnlessEqual(len(key), 16) # AES key size
+            self.assertThat(key, HasLength(16)) # AES key size
             return MutableData(data)
         d = self.nodemaker.create_mutable_file(_make_contents)
         def _created(n):
             return n.download_best_version()
         d.addCallback(_created)
-        d.addCallback(lambda data2: self.failUnlessEqual(data2, data))
+        d.addCallback(lambda data2: self.assertThat(data2, Equals(data)))
         return d
 
 
     def test_create_mdmf_with_initial_contents_function(self):
         data = b"initial contents" * 100000
         def _make_contents(n):
-            self.failUnless(isinstance(n, MutableFileNode))
+            self.assertThat(n, IsInstance(MutableFileNode))
             key = n.get_writekey()
-            self.failUnless(isinstance(key, bytes), key)
+            self.assertTrue(isinstance(key, bytes), key)
-            self.failUnlessEqual(len(key), 16)
+            self.assertThat(key, HasLength(16))
             return MutableData(data)
         d = self.nodemaker.create_mutable_file(_make_contents,
                                                version=MDMF_VERSION)
         d.addCallback(lambda n:
             n.download_best_version())
         d.addCallback(lambda data2:
-            self.failUnlessEqual(data2, data))
+            self.assertThat(data2, Equals(data)))
         return d
 
 
@@ -476,7 +484,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         d = n.get_servermap(MODE_READ)
         d.addCallback(lambda servermap: servermap.best_recoverable_version())
         d.addCallback(lambda verinfo:
-                      self.failUnlessEqual(verinfo[0], expected_seqnum, which))
+                      self.assertThat(verinfo[0], Equals(expected_seqnum), which))
         return d
 
     def test_modify(self):
@@ -513,36 +521,36 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         def _created(n):
             d = n.modify(_modifier)
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m"))
 
             d.addCallback(lambda res: n.modify(_non_modifier))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "non"))
 
             d.addCallback(lambda res: n.modify(_none_modifier))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "none"))
 
             d.addCallback(lambda res:
                           self.shouldFail(ValueError, "error_modifier", None,
                                           n.modify, _error_modifier))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "err"))
 
 
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "big"))
 
             d.addCallback(lambda res: n.modify(_ucw_error_modifier))
-            d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2))
+            d.addCallback(lambda res: self.assertThat(calls, HasLength(2)))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res,
+            d.addCallback(lambda res: self.assertThat(res,
-                                                           b"line1line2line3"))
+                                                           Equals(b"line1line2line3")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "ucw"))
 
             def _reset_ucw_error_modifier(res):
@@ -557,10 +565,10 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             # will only be one larger than the previous test, not two (i.e. 4
             # instead of 5).
             d.addCallback(lambda res: n.modify(_ucw_error_non_modifier))
-            d.addCallback(lambda res: self.failUnlessEqual(len(calls), 2))
+            d.addCallback(lambda res: self.assertThat(calls, HasLength(2)))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res,
+            d.addCallback(lambda res: self.assertThat(res,
-                                                           b"line1line2line3"))
+                                                           Equals(b"line1line2line3")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 4, "ucw"))
             d.addCallback(lambda res: n.modify(_toobig_modifier))
             return d
@@ -596,7 +604,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
         def _created(n):
             d = n.modify(_modifier)
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "m"))
 
             d.addCallback(lambda res:
@@ -605,7 +613,7 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
                                           n.modify, _ucw_error_modifier,
                                           _backoff_stopper))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"line1line2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"line1line2")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 2, "stop"))
 
             def _reset_ucw_error_modifier(res):
@@ -615,8 +623,8 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.modify(_ucw_error_modifier,
                                                _backoff_pauser))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res,
+            d.addCallback(lambda res: self.assertThat(res,
-                                                           b"line1line2line3"))
+                                                           Equals(b"line1line2line3")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "pause"))
 
             d.addCallback(lambda res:
@@ -625,8 +633,8 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
                                           n.modify, _always_ucw_error_modifier,
                                           giveuper.delay))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res,
+            d.addCallback(lambda res: self.assertThat(res,
-                                                           b"line1line2line3"))
+                                                          Equals(b"line1line2line3")))
             d.addCallback(lambda res: self.failUnlessCurrentSeqnumIs(n, 3, "giveup"))
 
             return d
@@ -641,23 +649,23 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.get_servermap(MODE_READ))
             d.addCallback(lambda smap: smap.dump(StringIO()))
             d.addCallback(lambda sio:
-                          self.failUnless("3-of-10" in sio.getvalue()))
+                          self.assertTrue("3-of-10" in sio.getvalue()))
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 1")))
-            d.addCallback(lambda res: self.failUnlessIdentical(res, None))
+            d.addCallback(lambda res: self.assertThat(res, Is(None)))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 1")))
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2")))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 2")))
             d.addCallback(lambda res: n.get_servermap(MODE_WRITE))
             d.addCallback(lambda smap: n.upload(MutableData(b"contents 3"), smap))
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3")))
             d.addCallback(lambda res: n.get_servermap(MODE_ANYTHING))
             d.addCallback(lambda smap:
                           n.download_version(smap,
                                              smap.best_recoverable_version()))
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 3"))
+            d.addCallback(lambda res: self.assertThat(res, Equals(b"contents 3")))
             return d
         d.addCallback(_created)
         return d
@@ -673,14 +681,14 @@ class Filenode(unittest.TestCase, testutil.ShouldFailMixin):
             return n.get_servermap(MODE_READ)
         d.addCallback(_created)
         d.addCallback(lambda ignored:
-            self.failUnlessEqual(self.n.get_size(), 0))
+            self.assertThat(self.n.get_size(), Equals(0)))
         d.addCallback(lambda ignored:
             self.n.overwrite(MutableData(b"foobarbaz")))
         d.addCallback(lambda ignored:
-            self.failUnlessEqual(self.n.get_size(), 9))
+            self.assertThat(self.n.get_size(), Equals(9)))
         d.addCallback(lambda ignored:
             self.nodemaker.create_mutable_file(MutableData(b"foobarbaz")))
         d.addCallback(_created)
         d.addCallback(lambda ignored:
-            self.failUnlessEqual(self.n.get_size(), 9))
+            self.assertThat(self.n.get_size(), Equals(9)))
         return d

src/allmydata/test/mutable/test_interoperability.py

@@ -11,14 +11,15 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, base64
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import HasLength
 from allmydata import uri
 from allmydata.storage.common import storage_index_to_dir
 from allmydata.util import fileutil
 from .. import common_util as testutil
 from ..no_network import GridTestMixin
 
-class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
+class Interoperability(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin):
     sdmf_old_shares = {}
     sdmf_old_shares[0] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAQ/EX4eC/1+hGOQ/h4EiKUkqxdsfzdcPlDvd11SGWZ0VHsUclZChTzuBAU2zLTXm+cG8IFhO50ly6Ey/DB44NtMKVaVzO0nU8DE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA="
     sdmf_old_shares[1] = b"VGFob2UgbXV0YWJsZSBjb250YWluZXIgdjEKdQlEA47ESLbTdKdpLJXCpBxd5OH239tl5hvAiz1dvGdE5rIOpf8cbfxbPcwNF+Y5dM92uBVbmV6KAAAAAAAAB/wAAAAAAAAJ0AAAAAFOWSw7jSx7WXzaMpdleJYXwYsRCV82jNA5oex9m2YhXSnb2POh+vvC1LE1NAfRc9GOb2zQG84Xdsx1Jub2brEeKkyt0sRIttN0p2kslcKkHF3k4fbf22XmAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABamJprL6ecrsOoFKdrXUmWveLq8nzEGDOjFnyK9detI3noX3uyK2MwSnFdAfyN0tuAwoAAAAAAAAAFQAAAAAAAAAVAAABjwAAAo8AAAMXAAADNwAAAAAAAAM+AAAAAAAAB/wwggEgMA0GCSqGSIb3DQEBAQUAA4IBDQAwggEIAoIBAQC1IkainlJF12IBXBQdpRK1zXB7a26vuEYqRmQM09YjC6sQjCs0F2ICk8n9m/2Kw4l16eIEboB2Au9pODCE+u/dEAakEFh4qidTMn61rbGUbsLK8xzuWNW22ezzz9/nPia0HDrulXt51/FYtfnnAuD1RJGXJv/8tDllE9FL/18TzlH4WuB6Fp8FTgv7QdbZAfWJHDGFIpVCJr1XxOCsSZNFJIqGwZnD2lsChiWw5OJDbKd8otqN1hIbfHyMyfMOJ/BzRzvZXaUt4Dv5nf93EmQDWClxShRwpuX/NkZ5B2K9OFonFTbOCexm/MjMAdCBqebKKaiHFkiknUCn9eJQpZ5bAgERgV50VKj+AVTDfgTpqfO2vfo4wrufi6ZBb8QV7hllhUFBjYogQ9C96dnS7skv0s+cqFuUjwMILr5/rsbEmEMGvl0T0ytyAbtlXuowEFVj/YORNknM4yjY72YUtEPTlMpk0Cis7aIgTvu5qWMPER26PMApZuRqiwRsGIkaJIvOVOTHHjFYe3/YzdMkc7OZtqRMfQLtwVl2/zKQQV8b/a9vaT6q3mRLRd4P3esaAFe/+7sR/t+9tmB+a8kxtKM6kmaVQJMbXJZ4aoHGfeLX0m35Rcvu2Bmph7QfSDjk/eaE3q55zYSoGWShmlhlw4Kwg84sMuhmcVhLvo0LovR8bKmbdgACtTh7+7gs/l5w1lOkgbF6w7rkXLNslK7L2KYF4SPFLUcABOOLy8EETxh7h7/z9d62EiPu9CNpRrCOLxUhn+JUS+DuAAhgcAb/adrQFrhlrRNoRpvjDuxmFebA4F0qCyqWssm61AAP7FHJWQoU87gQFNsy015vnBvCBYTudJcuhMvwweODbTD8Rfh4L/X6EY5D+HgSIpSSrF2x/N1w+UO93XVIZZnRUeePDXEwhqYDE0Wua7Lx6Bnad5n91qmHAnwSEJE5YIhQM634omd6cq9Wk4seJCUIn+ucoknrpxp0IR9QMxpKSMRHRUg2K8ZegnY3YqFunRZKCfsq9ufQEKgjZN12AFqi551KPBdn4/3V5HK6xTv0P4robSsE/BvuIfByvRf/W7ZrDx+CFC4EEcsBOACOZCrkhhqd5TkYKbe9RA+vs56+9N5qZGurkxcoKviiyEncxvTuShD65DK/6x6kMDMgQv/EdZDI3x9GtHTnRBYXwDGnPJ19w+q2zC3e2XarbxTGYQIPEC5mYx0gAA0sbjf018NGfwBhl6SB54iGsa8uLvR3jHv6OSRJgwxL6j7P0Ts4Hv2EtO12P0Lv21pwi3JC1O/WviSrKCvrQD5lMHL9Uym3hwFi2zu0mqwZvxOAbGy7kfOPXkLYKOHTZLthzKj3PsdjeceWBfYIvPGKYcd6wDr36d1aXSYS4IWeApTS2AQ2lu0DUcgSefAvsA8NkgOklvJY1cjTMSg6j6cxQo48Bvl8RAWGLbr4h2S/8KwDGxwLsSv0Gop/gnFc3GzCsmL0EkEyHHWkCA8YRXCghfW80KLDV495ff7yF5oiwK56GniqowZ3RG9Jxp5MXoJQgsLV1VMQFMAmsY69yz8eoxRH3wl9L0dMyndLulhWWzNwPMQ2I0yAWdzA/pksVmwTJTFenB3MHCiWc5rEwJ3yofe6NZZnZQrYyL9r1TNnVwfTwRUiykPiLSk4x9Mi6DX7RamDAxc8u3gDVfjPsTOTagBOEGUWlGAL54KE/E6sgCQ5DEAt12chk8AxbjBFLPgV+/idrzS0lZHOL+IVBI9D0i3Bq1yZcSIqcjZB0M3IbxbPm4gLAYOWEiTUN2ecsEHHg9nt6rhgffVoqSbCCFPbpC0xf7WOC3+BQORIZECOCC7cUAciXq3xn+GuxpFE40RWRJeKAK7bBQ21X89ABIXlQFkFddZ9kRvlZ2Pnl0oeF+2pjnZu0Yc2czNfZEQF2P7BKIdLrgMgxG89snxAY8qAYTCKyQw6xTG87wkjDcpy1wzsZLP3WsOuO7cAm7b27xU0jRKq8Cw4d1hDoyRG+RdS53F8RFJzVMaNNYgxU2tfRwUvXpTRXiOheeRVvh25+YGVnjakUXjx/dSDnOw4ETHGHD+7styDkeSfc3BdSZxswzc6OehgMI+xsCxeeRym15QUm9hxvg8X7Bfz/0WulgFwgzrm11TVynZYOmvyHpiZKoqQyQyKahIrfhwuchCr7lMsZ4a+umIkNkKxCLZnI+T7jd+eGFMgKItjz3kTTxRl3IhaJG3LbPmwRUJynMxQKdMi4Uf0qy0U7+i8hIJ9m50QXc+3tw2bwDSbx22XYJ9Wf14gxx5G5SPTb1JVCbhe4fxNt91xIxCow2zk62tzbYfRe6dfmDmgYHkv2PIEtMJZK8iKLDjFfu2ZUxsKT2A5g1q17og6o9MeXeuFS3mzJXJYFQZd+3UzlFR9qwkFkby9mg5y4XSeMvRLOHPt/H/r5SpEqBE6a9MadZYt61FBV152CUEzd43ihXtrAa0XH9HdsiySBcWI1SpM3mv9rRP0DiLjMUzHw/K1D8TE2f07zW4t/9kvE11tFj/NpICixQAAAAA="
@@ -53,7 +54,7 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi
                            sharedata)
         # ...and verify that the shares are there.
         shares = self.find_uri_shares(self.sdmf_old_cap)
-        assert len(shares) == 10
+        self.assertThat(shares, HasLength(10))
 
     def test_new_downloader_can_read_old_shares(self):
         self.basedir = "mutable/Interoperability/new_downloader_can_read_old_shares"
@@ -62,5 +63,5 @@ class Interoperability(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixi
         nm = self.g.clients[0].nodemaker
         n = nm.create_from_cap(self.sdmf_old_cap)
         d = n.download_best_version()
-        d.addCallback(self.failUnlessEqual, self.sdmf_old_contents)
+        d.addCallback(self.assertEqual, self.sdmf_old_contents)
         return d

src/allmydata/test/mutable/test_multiple_encodings.py

@@ -10,7 +10,8 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import Equals
 from allmydata.interfaces import SDMF_VERSION
 from allmydata.monitor import Monitor
 from foolscap.logging import log
@@ -20,8 +21,9 @@ from allmydata.mutable.servermap import ServerMap, ServermapUpdater
 from ..common_util import DevNullDictionary
 from .util import FakeStorage, make_nodemaker
 
-class MultipleEncodings(unittest.TestCase):
+class MultipleEncodings(AsyncTestCase):
     def setUp(self):
+        super(MultipleEncodings, self).setUp()
         self.CONTENTS = b"New contents go here"
         self.uploadable = MutableData(self.CONTENTS)
         self._storage = FakeStorage()
@@ -159,6 +161,6 @@ class MultipleEncodings(unittest.TestCase):
         d.addCallback(lambda res: fn3.download_best_version())
         def _retrieved(new_contents):
             # the current specified behavior is "first version recoverable"
-            self.failUnlessEqual(new_contents, contents1)
+            self.assertThat(new_contents, Equals(contents1))
         d.addCallback(_retrieved)
         return d

src/allmydata/test/mutable/test_multiple_versions.py

@@ -10,15 +10,17 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import Equals, HasLength
 from allmydata.monitor import Monitor
 from allmydata.mutable.common import MODE_CHECK, MODE_READ
 from .util import PublishMixin, CheckerMixin
 
 
-class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin):
+class MultipleVersions(AsyncTestCase, PublishMixin, CheckerMixin):
 
     def setUp(self):
+        super(MultipleVersions, self).setUp()
         return self.publish_multiple()
 
     def test_multiple_versions(self):
@@ -26,7 +28,7 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin):
         # should get the latest one
         self._set_versions(dict([(i,2) for i in (0,2,4,6,8)]))
         d = self._fn.download_best_version()
-        d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[4]))
+        d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[4])))
         # and the checker should report problems
         d.addCallback(lambda res: self._fn.check(Monitor()))
         d.addCallback(self.check_bad, "test_multiple_versions")
@@ -35,23 +37,23 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin):
         d.addCallback(lambda res:
                       self._set_versions(dict([(i,2) for i in range(10)])))
         d.addCallback(lambda res: self._fn.download_best_version())
-        d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2]))
+        d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2])))
         # if exactly one share is at version 3, we should still get v2
         d.addCallback(lambda res:
                       self._set_versions({0:3}))
         d.addCallback(lambda res: self._fn.download_best_version())
-        d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2]))
+        d.addCallback(lambda res: self.assertThat(res, Equals(self.CONTENTS[2])))
         # but the servermap should see the unrecoverable version. This
         # depends upon the single newer share being queried early.
         d.addCallback(lambda res: self._fn.get_servermap(MODE_READ))
         def _check_smap(smap):
-            self.failUnlessEqual(len(smap.unrecoverable_versions()), 1)
+            self.assertThat(smap.unrecoverable_versions(), HasLength(1))
             newer = smap.unrecoverable_newer_versions()
-            self.failUnlessEqual(len(newer), 1)
+            self.assertThat(newer, HasLength(1))
             verinfo, health = list(newer.items())[0]
-            self.failUnlessEqual(verinfo[0], 4)
+            self.assertThat(verinfo[0], Equals(4))
-            self.failUnlessEqual(health, (1,3))
+            self.assertThat(health, Equals((1,3)))
-            self.failIf(smap.needs_merge())
+            self.assertThat(smap.needs_merge(), Equals(False))
         d.addCallback(_check_smap)
         # if we have a mix of two parallel versions (s4a and s4b), we could
         # recover either
@@ -60,13 +62,13 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin):
                                           1:4,3:4,5:4,7:4,9:4}))
         d.addCallback(lambda res: self._fn.get_servermap(MODE_READ))
         def _check_smap_mixed(smap):
-            self.failUnlessEqual(len(smap.unrecoverable_versions()), 0)
+            self.assertThat(smap.unrecoverable_versions(), HasLength(0))
             newer = smap.unrecoverable_newer_versions()
-            self.failUnlessEqual(len(newer), 0)
+            self.assertThat(newer, HasLength(0))
-            self.failUnless(smap.needs_merge())
+            self.assertTrue(smap.needs_merge())
         d.addCallback(_check_smap_mixed)
         d.addCallback(lambda res: self._fn.download_best_version())
-        d.addCallback(lambda res: self.failUnless(res == self.CONTENTS[3] or
+        d.addCallback(lambda res: self.assertTrue(res == self.CONTENTS[3] or
                                                   res == self.CONTENTS[4]))
         return d
 
@@ -86,12 +88,12 @@ class MultipleVersions(unittest.TestCase, PublishMixin, CheckerMixin):
         d = self._fn.modify(_modify)
         d.addCallback(lambda res: self._fn.download_best_version())
         expected = self.CONTENTS[2] + b" modified"
-        d.addCallback(lambda res: self.failUnlessEqual(res, expected))
+        d.addCallback(lambda res: self.assertThat(res, Equals(expected)))
         # and the servermap should indicate that the outlier was replaced too
         d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK))
         def _check_smap(smap):
-            self.failUnlessEqual(smap.highest_seqnum(), 5)
+            self.assertThat(smap.highest_seqnum(), Equals(5))
-            self.failUnlessEqual(len(smap.unrecoverable_versions()), 0)
+            self.assertThat(smap.unrecoverable_versions(), HasLength(0))
-            self.failUnlessEqual(len(smap.recoverable_versions()), 1)
+            self.assertThat(smap.recoverable_versions(), HasLength(1))
         d.addCallback(_check_smap)
         return d

src/allmydata/test/mutable/test_problems.py

@@ -11,7 +11,8 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, base64
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import HasLength
 from twisted.internet import defer
 from foolscap.logging import log
 from allmydata import uri
@@ -61,7 +62,7 @@ class FirstServerGetsDeleted(object):
             return (True, {})
         return retval
 
-class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
+class Problems(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin):
     def do_publish_surprise(self, version):
         self.basedir = "mutable/Problems/test_publish_surprise_%s" % version
         self.set_up_grid()
@@ -198,8 +199,8 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
         def _overwritten_again(smap):
             # Make sure that all shares were updated by making sure that
             # there aren't any other versions in the sharemap.
-            self.failUnlessEqual(len(smap.recoverable_versions()), 1)
+            self.assertThat(smap.recoverable_versions(), HasLength(1))
-            self.failUnlessEqual(len(smap.unrecoverable_versions()), 0)
+            self.assertThat(smap.unrecoverable_versions(), HasLength(0))
         d.addCallback(_overwritten_again)
         return d
 
@@ -240,7 +241,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
         # that ought to work
         def _got_node(n):
             d = n.download_best_version()
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1"))
+            d.addCallback(lambda res: self.assertEquals(res, b"contents 1"))
             # now break the second peer
             def _break_peer1(res):
                 self.g.break_server(self.server1.get_serverid())
@@ -248,7 +249,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2")))
             # that ought to work too
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2"))
+            d.addCallback(lambda res: self.assertEquals(res, b"contents 2"))
             def _explain_error(f):
                 print(f)
                 if f.check(NotEnoughServersError):
@@ -280,7 +281,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
         d = nm.create_mutable_file(MutableData(b"contents 1"))
         def _created(n):
             d = n.download_best_version()
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 1"))
+            d.addCallback(lambda res: self.assertEquals(res, b"contents 1"))
             # now break one of the remaining servers
             def _break_second_server(res):
                 self.g.break_server(peerids[1])
@@ -288,7 +289,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
             d.addCallback(lambda res: n.overwrite(MutableData(b"contents 2")))
             # that ought to work too
             d.addCallback(lambda res: n.download_best_version())
-            d.addCallback(lambda res: self.failUnlessEqual(res, b"contents 2"))
+            d.addCallback(lambda res: self.assertEquals(res, b"contents 2"))
             return d
         d.addCallback(_created)
         return d
@@ -419,7 +420,7 @@ class Problems(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
             return self._node.download_version(servermap, ver)
         d.addCallback(_then)
         d.addCallback(lambda data:
-            self.failUnlessEqual(data, CONTENTS))
+            self.assertEquals(data, CONTENTS))
         return d
 
     def test_1654(self):

src/allmydata/test/mutable/test_repair.py

@@ -10,7 +10,8 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import Equals, HasLength
 from allmydata.interfaces import IRepairResults, ICheckAndRepairResults
 from allmydata.monitor import Monitor
 from allmydata.mutable.common import MODE_CHECK
@@ -19,7 +20,7 @@ from allmydata.mutable.repairer import MustForceRepairError
 from ..common import ShouldFailMixin
 from .util import PublishMixin
 
-class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
+class Repair(AsyncTestCase, PublishMixin, ShouldFailMixin):
 
     def get_shares(self, s):
         all_shares = {} # maps (peerid, shnum) to share data
@@ -40,8 +41,8 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
         d.addCallback(lambda res: self._fn.check(Monitor()))
         d.addCallback(lambda check_results: self._fn.repair(check_results))
         def _check_results(rres):
-            self.failUnless(IRepairResults.providedBy(rres))
+            self.assertThat(IRepairResults.providedBy(rres), Equals(True))
-            self.failUnless(rres.get_successful())
+            self.assertThat(rres.get_successful(), Equals(True))
             # TODO: examine results
 
             self.copy_shares()
@@ -50,11 +51,11 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
             new_shares = self.old_shares[1]
             # TODO: this really shouldn't change anything. When we implement
             # a "minimal-bandwidth" repairer", change this test to assert:
-            #self.failUnlessEqual(new_shares, initial_shares)
+            #self.assertThat(new_shares, Equals(initial_shares))
 
             # all shares should be in the same place as before
-            self.failUnlessEqual(set(initial_shares.keys()),
+            self.assertThat(set(initial_shares.keys()),
-                                 set(new_shares.keys()))
+                                 Equals(set(new_shares.keys())))
             # but they should all be at a newer seqnum. The IV will be
             # different, so the roothash will be too.
             for key in initial_shares:
@@ -70,19 +71,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
                  IV1,
                  k1, N1, segsize1, datalen1,
                  o1) = unpack_header(new_shares[key])
-                self.failUnlessEqual(version0, version1)
+                self.assertThat(version0, Equals(version1))
-                self.failUnlessEqual(seqnum0+1, seqnum1)
+                self.assertThat(seqnum0+1, Equals(seqnum1))
-                self.failUnlessEqual(k0, k1)
+                self.assertThat(k0, Equals(k1))
-                self.failUnlessEqual(N0, N1)
+                self.assertThat(N0, Equals(N1))
-                self.failUnlessEqual(segsize0, segsize1)
+                self.assertThat(segsize0, Equals(segsize1))
-                self.failUnlessEqual(datalen0, datalen1)
+                self.assertThat(datalen0, Equals(datalen1))
         d.addCallback(_check_results)
         return d
 
     def failIfSharesChanged(self, ignored=None):
         old_shares = self.old_shares[-2]
         current_shares = self.old_shares[-1]
-        self.failUnlessEqual(old_shares, current_shares)
+        self.assertThat(old_shares, Equals(current_shares))
 
 
     def _test_whether_repairable(self, publisher, nshares, expected_result):
@@ -96,12 +97,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
         d.addCallback(_delete_some_shares)
         d.addCallback(lambda ign: self._fn.check(Monitor()))
         def _check(cr):
-            self.failIf(cr.is_healthy())
+            self.assertThat(cr.is_healthy(), Equals(False))
-            self.failUnlessEqual(cr.is_recoverable(), expected_result)
+            self.assertThat(cr.is_recoverable(), Equals(expected_result))
             return cr
         d.addCallback(_check)
         d.addCallback(lambda check_results: self._fn.repair(check_results))
-        d.addCallback(lambda crr: self.failUnlessEqual(crr.get_successful(), expected_result))
+        d.addCallback(lambda crr: self.assertThat(crr.get_successful(), Equals(expected_result)))
         return d
 
     def test_unrepairable_0shares(self):
@@ -136,7 +137,7 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
                         del shares[peerid][shnum]
         d.addCallback(_delete_some_shares)
         d.addCallback(lambda ign: self._fn.check_and_repair(Monitor()))
-        d.addCallback(lambda crr: self.failUnlessEqual(crr.get_repair_successful(), expected_result))
+        d.addCallback(lambda crr: self.assertThat(crr.get_repair_successful(), Equals(expected_result)))
         return d
 
     def test_unrepairable_0shares_checkandrepair(self):
@@ -181,13 +182,13 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
                       self._fn.repair(check_results, force=True))
         # this should give us 10 shares of the highest roothash
         def _check_repair_results(rres):
-            self.failUnless(rres.get_successful())
+            self.assertThat(rres.get_successful(), Equals(True))
             pass # TODO
         d.addCallback(_check_repair_results)
         d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK))
         def _check_smap(smap):
-            self.failUnlessEqual(len(smap.recoverable_versions()), 1)
+            self.assertThat(smap.recoverable_versions(), HasLength(1))
-            self.failIf(smap.unrecoverable_versions())
+            self.assertThat(smap.unrecoverable_versions(), HasLength(0))
             # now, which should have won?
             roothash_s4a = self.get_roothash_for(3)
             roothash_s4b = self.get_roothash_for(4)
@@ -196,9 +197,9 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
             else:
                 expected_contents = self.CONTENTS[3]
             new_versionid = smap.best_recoverable_version()
-            self.failUnlessEqual(new_versionid[0], 5) # seqnum 5
+            self.assertThat(new_versionid[0], Equals(5)) # seqnum 5
             d2 = self._fn.download_version(smap, new_versionid)
-            d2.addCallback(self.failUnlessEqual, expected_contents)
+            d2.addCallback(self.assertEqual, expected_contents)
             return d2
         d.addCallback(_check_smap)
         return d
@@ -216,19 +217,19 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
         d.addCallback(lambda check_results: self._fn.repair(check_results))
         # this should give us 10 shares of v3
         def _check_repair_results(rres):
-            self.failUnless(rres.get_successful())
+            self.assertThat(rres.get_successful(), Equals(True))
             pass # TODO
         d.addCallback(_check_repair_results)
         d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK))
         def _check_smap(smap):
-            self.failUnlessEqual(len(smap.recoverable_versions()), 1)
+            self.assertThat(smap.recoverable_versions(), HasLength(1))
-            self.failIf(smap.unrecoverable_versions())
+            self.assertThat(smap.unrecoverable_versions(), HasLength(0))
             # now, which should have won?
             expected_contents = self.CONTENTS[3]
             new_versionid = smap.best_recoverable_version()
-            self.failUnlessEqual(new_versionid[0], 5) # seqnum 5
+            self.assertThat(new_versionid[0], Equals(5)) # seqnum 5
             d2 = self._fn.download_version(smap, new_versionid)
-            d2.addCallback(self.failUnlessEqual, expected_contents)
+            d2.addCallback(self.assertEquals, expected_contents)
             return d2
         d.addCallback(_check_smap)
         return d
@@ -256,12 +257,12 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
         d.addCallback(_get_readcap)
         d.addCallback(lambda res: self._fn3.check_and_repair(Monitor()))
         def _check_results(crr):
-            self.failUnless(ICheckAndRepairResults.providedBy(crr))
+            self.assertThat(ICheckAndRepairResults.providedBy(crr), Equals(True))
             # we should detect the unhealthy, but skip over mutable-readcap
             # repairs until #625 is fixed
-            self.failIf(crr.get_pre_repair_results().is_healthy())
+            self.assertThat(crr.get_pre_repair_results().is_healthy(), Equals(False))
-            self.failIf(crr.get_repair_attempted())
+            self.assertThat(crr.get_repair_attempted(), Equals(False))
-            self.failIf(crr.get_post_repair_results().is_healthy())
+            self.assertThat(crr.get_post_repair_results().is_healthy(), Equals(False))
         d.addCallback(_check_results)
         return d
 
@@ -281,6 +282,6 @@ class Repair(unittest.TestCase, PublishMixin, ShouldFailMixin):
         d.addCallback(lambda ign: self._fn2.check(Monitor()))
         d.addCallback(lambda check_results: self._fn2.repair(check_results))
         def _check(crr):
-            self.failUnlessEqual(crr.get_successful(), True)
+            self.assertThat(crr.get_successful(), Equals(True))
         d.addCallback(_check)
         return d

src/allmydata/test/mutable/test_roundtrip.py

@@ -11,7 +11,8 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from six.moves import cStringIO as StringIO
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import Equals, HasLength, Contains
 from twisted.internet import defer
 
 from allmydata.util import base32, consumer
@@ -23,8 +24,9 @@ from allmydata.mutable.retrieve import Retrieve
 from .util import PublishMixin, make_storagebroker, corrupt
 from .. import common_util as testutil
 
-class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
+class Roundtrip(AsyncTestCase, testutil.ShouldFailMixin, PublishMixin):
     def setUp(self):
+        super(Roundtrip, self).setUp()
         return self.publish_one()
 
     def make_servermap(self, mode=MODE_READ, oldmap=None, sb=None):
@@ -73,11 +75,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
         def _do_retrieve(servermap):
             self._smap = servermap
             #self.dump_servermap(servermap)
-            self.failUnlessEqual(len(servermap.recoverable_versions()), 1)
+            self.assertThat(servermap.recoverable_versions(), HasLength(1))
             return self.do_download(servermap)
         d.addCallback(_do_retrieve)
         def _retrieved(new_contents):
-            self.failUnlessEqual(new_contents, self.CONTENTS)
+            self.assertThat(new_contents, Equals(self.CONTENTS))
         d.addCallback(_retrieved)
         # we should be able to re-use the same servermap, both with and
         # without updating it.
@@ -132,10 +134,10 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
         # back empty
         d = self.make_servermap(sb=sb2)
         def _check_servermap(servermap):
-            self.failUnlessEqual(servermap.best_recoverable_version(), None)
+            self.assertThat(servermap.best_recoverable_version(), Equals(None))
-            self.failIf(servermap.recoverable_versions())
+            self.assertFalse(servermap.recoverable_versions())
-            self.failIf(servermap.unrecoverable_versions())
+            self.assertFalse(servermap.unrecoverable_versions())
-            self.failIf(servermap.all_servers())
+            self.assertFalse(servermap.all_servers())
         d.addCallback(_check_servermap)
         return d
 
@@ -154,7 +156,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
             self._fn._storage_broker = self._storage_broker
             return self._fn.download_best_version()
         def _retrieved(new_contents):
-            self.failUnlessEqual(new_contents, self.CONTENTS)
+            self.assertThat(new_contents, Equals(self.CONTENTS))
         d.addCallback(_restore)
         d.addCallback(_retrieved)
         return d
@@ -178,13 +180,13 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
                 # should be noted in the servermap's list of problems.
                 if substring:
                     allproblems = [str(f) for f in servermap.get_problems()]
-                    self.failUnlessIn(substring, "".join(allproblems))
+                    self.assertThat("".join(allproblems), Contains(substring))
                 return servermap
             if should_succeed:
                 d1 = self._fn.download_version(servermap, ver,
                                                fetch_privkey)
                 d1.addCallback(lambda new_contents:
-                               self.failUnlessEqual(new_contents, self.CONTENTS))
+                               self.assertThat(new_contents, Equals(self.CONTENTS)))
             else:
                 d1 = self.shouldFail(NotEnoughSharesError,
                                      "_corrupt_all(offset=%s)" % (offset,),
@@ -207,7 +209,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
             # and the dump should mention the problems
             s = StringIO()
             dump = servermap.dump(s).getvalue()
-            self.failUnless("30 PROBLEMS" in dump, dump)
+            self.assertTrue("30 PROBLEMS" in dump, msg=dump)
         d.addCallback(_check_servermap)
         return d
 
@@ -299,8 +301,8 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
         # in NotEnoughSharesError, since each share will look invalid
         def _check(res):
             f = res[0]
-            self.failUnless(f.check(NotEnoughSharesError))
+            self.assertThat(f.check(NotEnoughSharesError), HasLength(1))
-            self.failUnless("uncoordinated write" in str(f))
+            self.assertThat("uncoordinated write" in str(f), Equals(True))
         return self._test_corrupt_all(1, "ran out of servers",
                                       corrupt_early=False,
                                       failure_checker=_check)
@@ -309,7 +311,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
     def test_corrupt_all_block_late(self):
         def _check(res):
             f = res[0]
-            self.failUnless(f.check(NotEnoughSharesError))
+            self.assertTrue(f.check(NotEnoughSharesError))
         return self._test_corrupt_all("share_data", "block hash tree failure",
                                       corrupt_early=False,
                                       failure_checker=_check)
@@ -330,9 +332,9 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
                       shnums_to_corrupt=list(range(0, N-k)))
         d.addCallback(lambda res: self.make_servermap())
         def _do_retrieve(servermap):
-            self.failUnless(servermap.get_problems())
+            self.assertTrue(servermap.get_problems())
-            self.failUnless("pubkey doesn't match fingerprint"
+            self.assertThat("pubkey doesn't match fingerprint"
-                            in str(servermap.get_problems()[0]))
+                            in str(servermap.get_problems()[0]), Equals(True))
             ver = servermap.best_recoverable_version()
             r = Retrieve(self._fn, self._storage_broker, servermap, ver)
             c = consumer.MemoryConsumer()
@@ -340,7 +342,7 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
         d.addCallback(_do_retrieve)
         d.addCallback(lambda mc: b"".join(mc.chunks))
         d.addCallback(lambda new_contents:
-                      self.failUnlessEqual(new_contents, self.CONTENTS))
+                      self.assertThat(new_contents, Equals(self.CONTENTS)))
         return d
 
 
@@ -355,11 +357,11 @@ class Roundtrip(unittest.TestCase, testutil.ShouldFailMixin, PublishMixin):
             self.make_servermap())
         def _do_retrieve(servermap):
             ver = servermap.best_recoverable_version()
-            self.failUnless(ver)
+            self.assertTrue(ver)
             return self._fn.download_best_version()
         d.addCallback(_do_retrieve)
         d.addCallback(lambda new_contents:
-            self.failUnlessEqual(new_contents, self.CONTENTS))
+            self.assertThat(new_contents, Equals(self.CONTENTS)))
         return d
 
 

src/allmydata/test/mutable/test_servermap.py

@@ -11,7 +11,8 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import Equals, NotEquals, HasLength
 from twisted.internet import defer
 from allmydata.monitor import Monitor
 from allmydata.mutable.common import \
@@ -20,8 +21,9 @@ from allmydata.mutable.publish import MutableData
 from allmydata.mutable.servermap import ServerMap, ServermapUpdater
 from .util import PublishMixin
 
-class Servermap(unittest.TestCase, PublishMixin):
+class Servermap(AsyncTestCase, PublishMixin):
     def setUp(self):
+        super(Servermap, self).setUp()
         return self.publish_one()
 
     def make_servermap(self, mode=MODE_CHECK, fn=None, sb=None,
@@ -42,17 +44,17 @@ class Servermap(unittest.TestCase, PublishMixin):
         return d
 
     def failUnlessOneRecoverable(self, sm, num_shares):
-        self.failUnlessEqual(len(sm.recoverable_versions()), 1)
+        self.assertThat(sm.recoverable_versions(), HasLength(1))
-        self.failUnlessEqual(len(sm.unrecoverable_versions()), 0)
+        self.assertThat(sm.unrecoverable_versions(), HasLength(0))
         best = sm.best_recoverable_version()
-        self.failIfEqual(best, None)
+        self.assertThat(best, NotEquals(None))
-        self.failUnlessEqual(sm.recoverable_versions(), set([best]))
+        self.assertThat(sm.recoverable_versions(), Equals(set([best])))
-        self.failUnlessEqual(len(sm.shares_available()), 1)
+        self.assertThat(sm.shares_available(), HasLength(1))
-        self.failUnlessEqual(sm.shares_available()[best], (num_shares, 3, 10))
+        self.assertThat(sm.shares_available()[best], Equals((num_shares, 3, 10)))
         shnum, servers = list(sm.make_sharemap().items())[0]
         server = list(servers)[0]
-        self.failUnlessEqual(sm.version_on_server(server, shnum), best)
+        self.assertThat(sm.version_on_server(server, shnum), Equals(best))
-        self.failUnlessEqual(sm.version_on_server(server, 666), None)
+        self.assertThat(sm.version_on_server(server, 666), Equals(None))
         return sm
 
     def test_basic(self):
@@ -117,7 +119,7 @@ class Servermap(unittest.TestCase, PublishMixin):
             v = sm.best_recoverable_version()
             vm = sm.make_versionmap()
             shares = list(vm[v])
-            self.failUnlessEqual(len(shares), 6)
+            self.assertThat(shares, HasLength(6))
             self._corrupted = set()
             # mark the first 5 shares as corrupt, then update the servermap.
             # The map should not have the marked shares it in any more, and
@@ -135,18 +137,17 @@ class Servermap(unittest.TestCase, PublishMixin):
             shares = list(vm[v])
             for (server, shnum) in self._corrupted:
                 server_shares = sm.debug_shares_on_server(server)
-                self.failIf(shnum in server_shares,
+                self.assertFalse(shnum in server_shares, "%d was in %s" % (shnum, server_shares))
-                            "%d was in %s" % (shnum, server_shares))
+            self.assertThat(shares, HasLength(5))
-            self.failUnlessEqual(len(shares), 5)
         d.addCallback(_check_map)
         return d
 
     def failUnlessNoneRecoverable(self, sm):
-        self.failUnlessEqual(len(sm.recoverable_versions()), 0)
+        self.assertThat(sm.recoverable_versions(), HasLength(0))
-        self.failUnlessEqual(len(sm.unrecoverable_versions()), 0)
+        self.assertThat(sm.unrecoverable_versions(), HasLength(0))
         best = sm.best_recoverable_version()
-        self.failUnlessEqual(best, None)
+        self.assertThat(best, Equals(None))
-        self.failUnlessEqual(len(sm.shares_available()), 0)
+        self.assertThat(sm.shares_available(), HasLength(0))
 
     def test_no_shares(self):
         self._storage._peers = {} # delete all shares
@@ -168,12 +169,12 @@ class Servermap(unittest.TestCase, PublishMixin):
         return d
 
     def failUnlessNotQuiteEnough(self, sm):
-        self.failUnlessEqual(len(sm.recoverable_versions()), 0)
+        self.assertThat(sm.recoverable_versions(), HasLength(0))
-        self.failUnlessEqual(len(sm.unrecoverable_versions()), 1)
+        self.assertThat(sm.unrecoverable_versions(), HasLength(1))
         best = sm.best_recoverable_version()
-        self.failUnlessEqual(best, None)
+        self.assertThat(best, Equals(None))
-        self.failUnlessEqual(len(sm.shares_available()), 1)
+        self.assertThat(sm.shares_available(), HasLength(1))
-        self.failUnlessEqual(list(sm.shares_available().values())[0], (2,3,10) )
+        self.assertThat(list(sm.shares_available().values())[0], Equals((2,3,10)))
         return sm
 
     def test_not_quite_enough_shares(self):
@@ -193,7 +194,7 @@ class Servermap(unittest.TestCase, PublishMixin):
         d.addCallback(lambda res: ms(mode=MODE_CHECK))
         d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm))
         d.addCallback(lambda sm:
-                      self.failUnlessEqual(len(sm.make_sharemap()), 2))
+                      self.assertThat(sm.make_sharemap(), HasLength(2)))
         d.addCallback(lambda res: ms(mode=MODE_ANYTHING))
         d.addCallback(lambda sm: self.failUnlessNotQuiteEnough(sm))
         d.addCallback(lambda res: ms(mode=MODE_WRITE))
@@ -216,7 +217,7 @@ class Servermap(unittest.TestCase, PublishMixin):
         # Calling make_servermap also updates the servermap in the mode
         # that we specify, so we just need to see what it says.
         def _check_servermap(sm):
-            self.failUnlessEqual(len(sm.recoverable_versions()), 1)
+            self.assertThat(sm.recoverable_versions(), HasLength(1))
         d.addCallback(_check_servermap)
         return d
 
@@ -229,10 +230,10 @@ class Servermap(unittest.TestCase, PublishMixin):
             self.make_servermap(mode=MODE_WRITE, update_range=(1, 2)))
         def _check_servermap(sm):
             # 10 shares
-            self.failUnlessEqual(len(sm.update_data), 10)
+            self.assertThat(sm.update_data, HasLength(10))
             # one version
             for data in sm.update_data.values():
-                self.failUnlessEqual(len(data), 1)
+                self.assertThat(data, HasLength(1))
         d.addCallback(_check_servermap)
         return d
 
@@ -244,5 +245,5 @@ class Servermap(unittest.TestCase, PublishMixin):
         d.addCallback(lambda ignored:
             self.make_servermap(mode=MODE_CHECK))
         d.addCallback(lambda servermap:
-            self.failUnlessEqual(len(servermap.recoverable_versions()), 1))
+            self.assertThat(servermap.recoverable_versions(), HasLength(1)))
         return d

src/allmydata/test/mutable/test_update.py

@@ -11,7 +11,12 @@ if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import re
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import (
+    Equals,
+    IsInstance,
+    GreaterThan,
+)
 from twisted.internet import defer
 from allmydata.interfaces import MDMF_VERSION
 from allmydata.mutable.filenode import MutableFileNode
@@ -25,7 +30,7 @@ from .. import common_util as testutil
 # this up.
 SEGSIZE = 128*1024
 
-class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
+class Update(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin):
     def setUp(self):
         GridTestMixin.setUp(self)
         self.basedir = self.mktemp()
@@ -35,14 +40,14 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
         # self.data should be at least three segments long.
         td = b"testdata "
         self.data = td*(int(3*SEGSIZE//len(td))+10) # currently about 400kB
-        assert len(self.data) > 3*SEGSIZE
+        self.assertThat(len(self.data), GreaterThan(3*SEGSIZE))
         self.small_data = b"test data" * 10 # 90 B; SDMF
 
 
     def do_upload_sdmf(self):
         d = self.nm.create_mutable_file(MutableData(self.small_data))
         def _then(n):
-            assert isinstance(n, MutableFileNode)
+            self.assertThat(n, IsInstance(MutableFileNode))
             self.sdmf_node = n
         d.addCallback(_then)
         return d
@@ -51,7 +56,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
         d = self.nm.create_mutable_file(MutableData(self.data),
                                         version=MDMF_VERSION)
         def _then(n):
-            assert isinstance(n, MutableFileNode)
+            self.assertThat(n, IsInstance(MutableFileNode))
             self.mdmf_node = n
         d.addCallback(_then)
         return d
@@ -185,7 +190,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
                                                len(self.data)))
             d.addCallback(lambda ign: self.mdmf_node.download_best_version())
             d.addCallback(lambda results:
-                          self.failUnlessEqual(results, new_data))
+                          self.assertThat(results, Equals(new_data)))
             return d
         d0.addCallback(_run)
         return d0
@@ -201,7 +206,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
                                                len(self.small_data)))
             d.addCallback(lambda ign: self.sdmf_node.download_best_version())
             d.addCallback(lambda results:
-                          self.failUnlessEqual(results, new_data))
+                          self.assertThat(results, Equals(new_data)))
             return d
         d0.addCallback(_run)
         return d0
@@ -221,7 +226,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
                                                replace_offset))
             d.addCallback(lambda ign: self.mdmf_node.download_best_version())
             d.addCallback(lambda results:
-                          self.failUnlessEqual(results, new_data))
+                          self.assertThat(results, Equals(new_data)))
             return d
         d0.addCallback(_run)
         return d0
@@ -242,7 +247,7 @@ class Update(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
                                                replace_offset))
             d.addCallback(lambda ignored: self.mdmf_node.download_best_version())
             d.addCallback(lambda results:
-                          self.failUnlessEqual(results, new_data))
+                          self.assertThat(results, Equals(new_data)))
             return d
         d0.addCallback(_run)
         return d0

src/allmydata/test/mutable/test_version.py

@@ -14,7 +14,13 @@ import os
 from six.moves import cStringIO as StringIO
 
 from twisted.internet import defer
-from twisted.trial import unittest
+from ..common import AsyncTestCase
+from testtools.matchers import (
+    Equals,
+    IsInstance,
+    HasLength,
+    Contains,
+)
 
 from allmydata import uri
 from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION
@@ -29,7 +35,7 @@ from ..no_network import GridTestMixin
 from .util import PublishMixin
 from .. import common_util as testutil
 
-class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
+class Version(GridTestMixin, AsyncTestCase, testutil.ShouldFailMixin, \
               PublishMixin):
     def setUp(self):
         GridTestMixin.setUp(self)
@@ -47,8 +53,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d = self.nm.create_mutable_file(MutableData(data),
                                         version=MDMF_VERSION)
         def _then(n):
-            assert isinstance(n, MutableFileNode)
+            self.assertThat(n, IsInstance(MutableFileNode))
-            assert n._protocol_version == MDMF_VERSION
+            self.assertThat(n._protocol_version, Equals(MDMF_VERSION))
             self.mdmf_node = n
             return n
         d.addCallback(_then)
@@ -59,8 +65,8 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             data = self.small_data
         d = self.nm.create_mutable_file(MutableData(data))
         def _then(n):
-            assert isinstance(n, MutableFileNode)
+            self.assertThat(n, IsInstance(MutableFileNode))
-            assert n._protocol_version == SDMF_VERSION
+            self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
             self.sdmf_node = n
             return n
         d.addCallback(_then)
@@ -69,9 +75,9 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
     def do_upload_empty_sdmf(self):
         d = self.nm.create_mutable_file(MutableData(b""))
         def _then(n):
-            assert isinstance(n, MutableFileNode)
+            self.assertThat(n, IsInstance(MutableFileNode))
             self.sdmf_zero_length_node = n
-            assert n._protocol_version == SDMF_VERSION
+            self.assertThat(n._protocol_version, Equals(SDMF_VERSION))
             return n
         d.addCallback(_then)
         return d
@@ -95,7 +101,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             debug.find_shares(fso)
             sharefiles = fso.stdout.getvalue().splitlines()
             expected = self.nm.default_encoding_parameters["n"]
-            self.failUnlessEqual(len(sharefiles), expected)
+            self.assertThat(sharefiles, HasLength(expected))
 
             do = debug.DumpOptions()
             do["filename"] = sharefiles[0]
@@ -103,17 +109,17 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             debug.dump_share(do)
             output = do.stdout.getvalue()
             lines = set(output.splitlines())
-            self.failUnless("Mutable slot found:" in lines, output)
+            self.assertTrue("Mutable slot found:" in lines, output)
-            self.failUnless(" share_type: MDMF" in lines, output)
+            self.assertTrue(" share_type: MDMF" in lines, output)
-            self.failUnless(" num_extra_leases: 0" in lines, output)
+            self.assertTrue(" num_extra_leases: 0" in lines, output)
-            self.failUnless(" MDMF contents:" in lines, output)
+            self.assertTrue(" MDMF contents:" in lines, output)
-            self.failUnless("  seqnum: 1" in lines, output)
+            self.assertTrue("  seqnum: 1" in lines, output)
-            self.failUnless("  required_shares: 3" in lines, output)
+            self.assertTrue("  required_shares: 3" in lines, output)
-            self.failUnless("  total_shares: 10" in lines, output)
+            self.assertTrue("  total_shares: 10" in lines, output)
-            self.failUnless("  segsize: 131073" in lines, output)
+            self.assertTrue("  segsize: 131073" in lines, output)
-            self.failUnless("  datalen: %d" % len(self.data) in lines, output)
+            self.assertTrue("  datalen: %d" % len(self.data) in lines, output)
             vcap = str(n.get_verify_cap().to_string(), "utf-8")
-            self.failUnless("  verify-cap: %s" % vcap in lines, output)
+            self.assertTrue("  verify-cap: %s" % vcap in lines, output)
             cso = debug.CatalogSharesOptions()
             cso.nodedirs = fso.nodedirs
             cso.stdout = StringIO()
@@ -122,13 +128,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             shares = cso.stdout.getvalue().splitlines()
             oneshare = shares[0] # all shares should be MDMF
             self.failIf(oneshare.startswith("UNKNOWN"), oneshare)
-            self.failUnless(oneshare.startswith("MDMF"), oneshare)
+            self.assertTrue(oneshare.startswith("MDMF"), oneshare)
             fields = oneshare.split()
-            self.failUnlessEqual(fields[0], "MDMF")
+            self.assertThat(fields[0], Equals("MDMF"))
-            self.failUnlessEqual(fields[1].encode("ascii"), storage_index)
+            self.assertThat(fields[1].encode("ascii"), Equals(storage_index))
-            self.failUnlessEqual(fields[2], "3/10")
+            self.assertThat(fields[2], Equals("3/10"))
-            self.failUnlessEqual(fields[3], "%d" % len(self.data))
+            self.assertThat(fields[3], Equals("%d" % len(self.data)))
-            self.failUnless(fields[4].startswith("#1:"), fields[3])
+            self.assertTrue(fields[4].startswith("#1:"), fields[3])
             # the rest of fields[4] is the roothash, which depends upon
             # encryption salts and is not constant. fields[5] is the
             # remaining time on the longest lease, which is timing dependent.
@@ -140,11 +146,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d = self.do_upload()
         d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version())
         d.addCallback(lambda bv:
-            self.failUnlessEqual(bv.get_sequence_number(), 1))
+            self.assertThat(bv.get_sequence_number(), Equals(1)))
         d.addCallback(lambda ignored:
             self.sdmf_node.get_best_readable_version())
         d.addCallback(lambda bv:
-            self.failUnlessEqual(bv.get_sequence_number(), 1))
+            self.assertThat(bv.get_sequence_number(), Equals(1)))
         # Now update. The sequence number in both cases should be 1 in
         # both cases.
         def _do_update(ignored):
@@ -158,11 +164,11 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ignored:
             self.mdmf_node.get_best_readable_version())
         d.addCallback(lambda bv:
-            self.failUnlessEqual(bv.get_sequence_number(), 2))
+            self.assertThat(bv.get_sequence_number(), Equals(2)))
         d.addCallback(lambda ignored:
             self.sdmf_node.get_best_readable_version())
         d.addCallback(lambda bv:
-            self.failUnlessEqual(bv.get_sequence_number(), 2))
+            self.assertThat(bv.get_sequence_number(), Equals(2)))
         return d
 
 
@@ -175,10 +181,10 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         def _then(ign):
             mdmf_uri = self.mdmf_node.get_uri()
             cap = uri.from_string(mdmf_uri)
-            self.failUnless(isinstance(cap, uri.WriteableMDMFFileURI))
+            self.assertTrue(isinstance(cap, uri.WriteableMDMFFileURI))
             readonly_mdmf_uri = self.mdmf_node.get_readonly_uri()
             cap = uri.from_string(readonly_mdmf_uri)
-            self.failUnless(isinstance(cap, uri.ReadonlyMDMFFileURI))
+            self.assertTrue(isinstance(cap, uri.ReadonlyMDMFFileURI))
         d.addCallback(_then)
         return d
 
@@ -189,16 +195,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ign: self.mdmf_node.get_best_mutable_version())
         def _check_mdmf(bv):
             n = self.mdmf_node
-            self.failUnlessEqual(bv.get_writekey(), n.get_writekey())
+            self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
-            self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index())
+            self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
-            self.failIf(bv.is_readonly())
+            self.assertFalse(bv.is_readonly())
         d.addCallback(_check_mdmf)
         d.addCallback(lambda ign: self.sdmf_node.get_best_mutable_version())
         def _check_sdmf(bv):
             n = self.sdmf_node
-            self.failUnlessEqual(bv.get_writekey(), n.get_writekey())
+            self.assertThat(bv.get_writekey(), Equals(n.get_writekey()))
-            self.failUnlessEqual(bv.get_storage_index(), n.get_storage_index())
+            self.assertThat(bv.get_storage_index(), Equals(n.get_storage_index()))
-            self.failIf(bv.is_readonly())
+            self.assertFalse(bv.is_readonly())
         d.addCallback(_check_sdmf)
         return d
 
@@ -206,21 +212,21 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
     def test_get_readonly_version(self):
         d = self.do_upload()
         d.addCallback(lambda ign: self.mdmf_node.get_best_readable_version())
-        d.addCallback(lambda bv: self.failUnless(bv.is_readonly()))
+        d.addCallback(lambda bv: self.assertTrue(bv.is_readonly()))
 
         # Attempting to get a mutable version of a mutable file from a
         # filenode initialized with a readcap should return a readonly
         # version of that same node.
         d.addCallback(lambda ign: self.mdmf_node.get_readonly())
         d.addCallback(lambda ro: ro.get_best_mutable_version())
-        d.addCallback(lambda v: self.failUnless(v.is_readonly()))
+        d.addCallback(lambda v: self.assertTrue(v.is_readonly()))
 
         d.addCallback(lambda ign: self.sdmf_node.get_best_readable_version())
-        d.addCallback(lambda bv: self.failUnless(bv.is_readonly()))
+        d.addCallback(lambda bv: self.assertTrue(bv.is_readonly()))
 
         d.addCallback(lambda ign: self.sdmf_node.get_readonly())
         d.addCallback(lambda ro: ro.get_best_mutable_version())
-        d.addCallback(lambda v: self.failUnless(v.is_readonly()))
+        d.addCallback(lambda v: self.assertTrue(v.is_readonly()))
         return d
 
 
@@ -232,13 +238,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ignored:
             self.mdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessEqual(data, b"foo bar baz" * 100000))
+            self.assertThat(data, Equals(b"foo bar baz" * 100000)))
         d.addCallback(lambda ignored:
             self.sdmf_node.overwrite(new_small_data))
         d.addCallback(lambda ignored:
             self.sdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessEqual(data, b"foo bar baz" * 10))
+            self.assertThat(data, Equals(b"foo bar baz" * 10)))
         return d
 
 
@@ -250,13 +256,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ignored:
             self.mdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessIn(b"modified", data))
+            self.assertThat(data, Contains(b"modified")))
         d.addCallback(lambda ignored:
             self.sdmf_node.modify(modifier))
         d.addCallback(lambda ignored:
             self.sdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessIn(b"modified", data))
+            self.assertThat(data, Contains(b"modified")))
         return d
 
 
@@ -271,13 +277,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ignored:
             self.mdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessIn(b"modified", data))
+            self.assertThat(data, Contains(b"modified")))
         d.addCallback(lambda ignored:
             self.sdmf_node.modify(modifier))
         d.addCallback(lambda ignored:
             self.sdmf_node.download_best_version())
         d.addCallback(lambda data:
-            self.failUnlessIn(b"modified", data))
+            self.assertThat(data, Contains(b"modified")))
         return d
 
 
@@ -308,13 +314,13 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
         d.addCallback(lambda ignored:
             self._fn.download_version(self.servermap, self.version1))
         d.addCallback(lambda results:
-            self.failUnlessEqual(self.CONTENTS[self.version1_index],
+            self.assertThat(self.CONTENTS[self.version1_index],
-                                 results))
+                                 Equals(results)))
         d.addCallback(lambda ignored:
             self._fn.download_version(self.servermap, self.version2))
         d.addCallback(lambda results:
-            self.failUnlessEqual(self.CONTENTS[self.version2_index],
+            self.assertThat(self.CONTENTS[self.version2_index],
-                                 results))
+                                 Equals(results)))
         return d
 
 
@@ -344,7 +350,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             for i in range(0, len(expected), step):
                 d2.addCallback(lambda ignored, i=i: version.read(c, i, step))
             d2.addCallback(lambda ignored:
-                self.failUnlessEqual(expected, b"".join(c.chunks)))
+                self.assertThat(expected, Equals(b"".join(c.chunks))))
             return d2
         d.addCallback(_read_data)
         return d
@@ -447,16 +453,16 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             d2 = defer.succeed(None)
             d2.addCallback(lambda ignored: version.read(c))
             d2.addCallback(lambda ignored:
-                self.failUnlessEqual(expected, b"".join(c.chunks)))
+                self.assertThat(expected, Equals(b"".join(c.chunks))))
 
             d2.addCallback(lambda ignored: version.read(c2, offset=0,
                                                         size=len(expected)))
             d2.addCallback(lambda ignored:
-                self.failUnlessEqual(expected, b"".join(c2.chunks)))
+                self.assertThat(expected, Equals(b"".join(c2.chunks))))
             return d2
         d.addCallback(_read_data)
         d.addCallback(lambda ignored: node.download_best_version())
-        d.addCallback(lambda data: self.failUnlessEqual(expected, data))
+        d.addCallback(lambda data: self.assertThat(expected, Equals(data)))
         return d
 
     def test_read_and_download_mdmf(self):

tests.nix

@@ -0,0 +1,87 @@
+let
+  sources = import nix/sources.nix;
+in
+# See default.nix for documentation about parameters.
+{ pkgsVersion ? "nixpkgs-21.11"
+, pkgs ? import sources.${pkgsVersion} { }
+, pypiData ? sources.pypi-deps-db
+, pythonVersion ? "python37"
+, mach-nix ? import sources.mach-nix {
+    inherit pkgs pypiData;
+    python = pythonVersion;
+  }
+}@args:
+let
+  # We would like to know the test requirements but mach-nix does not directly
+  # expose this information to us.  However, it is perfectly capable of
+  # determining it if we ask right...  This is probably not meant to be a
+  # public mach-nix API but we pinned mach-nix so we can deal with mach-nix
+  # upgrade breakage in our own time.
+  mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" {
+    inherit pkgs;
+    lib = pkgs.lib;
+  };
+  tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test;
+
+  # Get the Tahoe-LAFS package itself.  This does not include test
+  # requirements and we don't ask for test requirements so that we can just
+  # re-use the normal package if it is already built.
+  tahoe-lafs = import ./. args;
+
+  # If we want to get tahoe-lafs into a Python environment with a bunch of
+  # *other* Python modules and let them interact in the usual way then we have
+  # to ask mach-nix for tahoe-lafs and those other Python modules in the same
+  # way - i.e., using `requirements`.  The other tempting mechanism,
+  # `packagesExtra`, inserts an extra layer of Python environment and prevents
+  # normal interaction between Python modules (as well as usually producing
+  # file collisions in the packages that are both runtime and test
+  # dependencies).  To get the tahoe-lafs we just built into the environment,
+  # put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs
+  # from nixpkgs.
+  overridesPre = [(self: super: { inherit tahoe-lafs; })];
+  providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; };
+
+  # Make the Python environment in which we can run the tests.
+  python-env = mach-nix.mkPython {
+    # Get the packaging fixes we already know we need from putting together
+    # the runtime package.
+    inherit (tahoe-lafs.meta.mach-nix) _;
+    # Share the runtime package's provider configuration - combined with our
+    # own that causes the right tahoe-lafs to be picked up.
+    inherit providers overridesPre;
+    requirements = ''
+      # Here we pull in the Tahoe-LAFS package itself.
+      tahoe-lafs
+
+      # Unfortunately mach-nix misses all of the Python dependencies of the
+      # tahoe-lafs satisfied from nixpkgs.  Drag them in here.  This gives a
+      # bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will
+      # fix this soon.
+      #
+      # https://github.com/DavHau/mach-nix/issues/123
+      # https://github.com/DavHau/mach-nix/pull/386
+      ${tahoe-lafs.requirements}
+
+      # And then all of the test-only dependencies.
+      ${builtins.concatStringsSep "\n" tests_require}
+
+      # txi2p-tahoe is another dependency with an environment marker that
+      # mach-nix doesn't automatically pick up.
+      txi2p-tahoe
+    '';
+  };
+in
+# Make a derivation that runs the unit test suite.
+pkgs.runCommand "tahoe-lafs-tests" { } ''
+  ${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
+
+  # It's not cool to put the whole _trial_temp into $out because it has weird
+  # files in it we don't want in the store.  Plus, even all of the less weird
+  # files are mostly just trash that's not meaningful if the test suite passes
+  # (which is the only way we get $out anyway).
+  #
+  # The build log itself is typically available from `nix-store --read-log` so
+  # we don't need to record that either.
+  echo "passed" >$out
+
+''