Add a flake with some packages and apps and an overlay

default.nix

@@ -18,32 +18,11 @@ in
   pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
                                 # niv-managed sources data
 
-, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
+, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself
+  overlays = [ (import ./nix/overlay.nix) ];
+}
 
 , pythonVersion ? "python310" # a string choosing the python derivation from
                               # nixpkgs to target
-
-, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
-                                # the dependencies of which the resulting
-                                # package will also depend on.  Include all of the
-                                # runtime extras by default because the incremental
-                                # cost of including them is a lot smaller than the
-                                # cost of re-building the whole thing to add them.
-
 }:
-with (pkgs.${pythonVersion}.override {
+pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ])
-  packageOverrides = import ./nix/python-overrides.nix;
-}).pkgs;
-callPackage ./nix/tahoe-lafs.nix {
-  # Select whichever package extras were requested.
-  inherit extrasNames;
-
-  # Define the location of the Tahoe-LAFS source to be packaged (the same
-  # directory as contains this file).  Clean up as many of the non-source
-  # files (eg the `.git` directory, `~` backup files, nix's own `result`
-  # symlink, etc) as possible to avoid needing to re-build when files that
-  # make no difference to the package have changed.
-  tahoe-lafs-src = pkgs.lib.cleanSource ./.;
-
-  doCheck = false;
-}

flake.lock

@@ -0,0 +1,80 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1687709756,
+        "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs-22_11": {
+      "locked": {
+        "lastModified": 1688392541,
+        "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1688486599,
+        "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs-22_11"
+        ],
+        "nixpkgs-22_11": "nixpkgs-22_11",
+        "nixpkgs-unstable": "nixpkgs-unstable"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,162 @@
+{
+  description = "Tahoe-LAFS, free and open decentralized data store";
+
+  inputs = {
+    # Two alternate nixpkgs pins.  Ideally these could be selected easily from
+    # the command line but there seems to be no syntax/support for that.
+    # However, these at least cause certain revisions to be pinned in our lock
+    # file where you *can* dig them out - and the CI configuration does.
+    "nixpkgs-22_11" = {
+      url = github:NixOS/nixpkgs?ref=nixos-22.11;
+    };
+    "nixpkgs-unstable" = {
+      url = github:NixOS/nixpkgs;
+    };
+
+    # Point the default nixpkgs at one of those.  This avoids having getting a
+    # _third_ package set involved and gives a way to provide what should be a
+    # working experience by default (that is, if nixpkgs doesn't get
+    # overridden).
+    nixpkgs.follows = "nixpkgs-22_11";
+
+    # Also get flake-utils for simplified multi-system definitions.
+    flake-utils = {
+      url = github:numtide/flake-utils;
+    };
+  };
+
+  outputs = { self, nixpkgs, flake-utils, ... }:
+    {
+      # Expose an overlay which adds our version of Tahoe-LAFS to the Python
+      # package sets we specify, as well as all of the correct versions of its
+      # dependencies.
+      #
+      # We will also use this to define some other outputs since it gives us
+      # the most succinct way to get a working Tahoe-LAFS package.
+      overlays.default = import ./nix/overlay.nix;
+
+    } // (flake-utils.lib.eachDefaultSystem (system: let
+
+      # First get the package set for this system architecture.
+      pkgs = import nixpkgs {
+        inherit system;
+        # And include our Tahoe-LAFS package in that package set.
+        overlays = [ self.overlays.default ];
+      };
+
+      # Find out what Python versions we're working with.
+      pythonVersions = builtins.attrNames (
+        pkgs.lib.attrsets.filterAttrs
+          # Match attribute names that look like a Python derivation - CPython
+          # or PyPy.  We take care to avoid things like "python-foo" and
+          # "python3Full-unittest" though.  We only want things like "pypy38"
+          # or "python311".
+          (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name)
+          pkgs
+      );
+
+      # An element of pythonVersions which we'll use for the default package.
+      defaultPyVersion = "python310";
+
+      # Retrieve the actual Python package for each configured version.  We
+      # already applied our overlay to pkgs so our packages will already be
+      # available.
+      pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions;
+
+      # string -> string
+      #
+      # Construct the Tahoe-LAFS package name for the given Python runtime.
+      packageName = pyVersion: "${pyVersion}-tahoe-lafs";
+
+      # string -> string
+      #
+      # Construct the unit test application name for the given Python runtime.
+      unitTestName = pyVersion: "${pyVersion}-unittest";
+
+      # Create a derivation that includes a Python runtime, Tahoe-LAFS, and
+      # all of its dependencies.
+      makeRuntimeEnv = pyVersion: {
+        ${packageName pyVersion} = makeRuntimeEnv' pyVersion;
+      };
+
+      makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps;
+        [ tahoe-lafs ] ++
+        tahoe-lafs.passthru.extras.i2p ++
+        tahoe-lafs.passthru.extras.tor
+      )).overrideAttrs (old: {
+        name = packageName pyVersion;
+      });
+
+      # Create a derivation that includes a Python runtime, Tahoe-LAFS, and
+      # all of its dependencies.
+      makeTestEnv = pyVersion: {
+        ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps;
+          [ tahoe-lafs ] ++
+          tahoe-lafs.passthru.extras.i2p ++
+          tahoe-lafs.passthru.extras.tor ++
+          tahoe-lafs.passthru.extras.unittest
+        )).overrideAttrs (old: {
+          name = packageName pyVersion;
+        });
+      };
+    in {
+      # Define the flake's package outputs.  We'll define one version of the
+      # package for each version of Python we could find.  We'll also point
+      # the flake's "default" package at one of these somewhat arbitrarily.
+      # The package consists of a Python environment with Tahoe-LAFS available
+      # to it.
+      packages = with pkgs.lib;
+        foldr mergeAttrs {} ([
+          { default = self.packages.${system}.${packageName defaultPyVersion}; }
+        ] ++ (builtins.map makeRuntimeEnv pythonVersions));
+
+      # Define the flake's app outputs.  We'll define a version of an app for
+      # running the test suite for each version of Python we could find.
+      # We'll also define a version of an app for running the "tahoe"
+      # command-line entrypoint for each version of Python we could find.
+      apps =
+        let
+          # We avoid writeShellApplication here because it has ghc as a
+          # dependency but ghc has Python as a dependency and our Python
+          # package override triggers a rebuild of ghc which takes a looong
+          # time.
+          writeScript = name: text:
+            let script = pkgs.writeShellScript name text;
+            in "${script}";
+
+          # A helper function to define the runtime entrypoint for a certain
+          # Python runtime.
+          makeTahoeApp = pyVersion: {
+            "tahoe-${pyVersion}" = {
+              type = "app";
+              program =
+                writeScript "tahoe"
+                  ''
+                    ${makeRuntimeEnv' pyVersion}/bin/tahoe "$@"
+                  '';
+            };
+          };
+
+          # A helper function to define the unit test entrypoint for a certain
+          # Python runtime.
+          makeUnitTestsApp = pyVersion: {
+            "${unitTestName pyVersion}" = {
+              type = "app";
+              program =
+                writeScript "unit-tests"
+                  ''
+                    export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
+                    ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@"
+                  '';
+            };
+          };
+        in
+          with pkgs.lib;
+          foldr mergeAttrs
+            { default = self.apps.${system}."tahoe-python3"; }
+            (
+              (builtins.map makeUnitTestsApp pythonVersions) ++
+              (builtins.map makeTahoeApp pythonVersions)
+            );
+    }));
+}

nix/overlay.nix

@@ -0,0 +1,10 @@
+# This overlay adds Tahoe-LAFS and all of its properly-configured Python
+# package dependencies to a Python package set.  Downstream consumers can
+# apply it to their own nixpkgs derivation to produce a Tahoe-LAFS package.
+final: prev: {
+  # Add our overrides such that they will be applied to any Python derivation
+  # in nixpkgs.
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (import ./python-overrides.nix)
+  ];
+}

nix/python-overrides.nix

@@ -9,14 +9,39 @@ let
   # Disable a Python package's test suite.
   dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; });
 
+  # string -> any -> derivation -> derivation
+  #
+  # If the overrideable function for the given derivation accepts an argument
+  # with the given name, override it with the given value.
+  #
+  # Since we try to work with multiple versions of nixpkgs, sometimes we need
+  # to override a parameter that exists in one version but not others.  This
+  # makes it a bit easier to do so.
+  overrideIfPresent = name: value: drv:
+    if (drv.override.__functionArgs ? ${name})
+    then drv.override { "${name}" = value; }
+    else drv;
+
   # Disable building a Python package's documentation.
-  dontBuildDocs = alsoDisable: drv: (drv.override ({
+  dontBuildDocs = drv: (
-    sphinxHook = null;
+    overrideIfPresent "sphinxHook" null (
-  } // alsoDisable)).overrideAttrs ({ outputs, ... }: {
+      overrideIfPresent "sphinx-rtd-theme" null
+        drv
+    )
+  ).overrideAttrs ({ outputs, ... }: {
     outputs = builtins.filter (x: "doc" != x) outputs;
   });
 
 in {
+  tahoe-lafs = self.callPackage ./tahoe-lafs.nix {
+    # Define the location of the Tahoe-LAFS source to be packaged (the same
+    # directory as contains this file).  Clean up as many of the non-source
+    # files (eg the `.git` directory, `~` backup files, nix's own `result`
+    # symlink, etc) as possible to avoid needing to re-build when files that
+    # make no difference to the package have changed.
+    tahoe-lafs-src = self.lib.cleanSource ../.;
+  };
+
   # Some dependencies aren't packaged in nixpkgs so supply our own packages.
   pycddl = self.callPackage ./pycddl.nix { };
   txi2p = self.callPackage ./txi2p.nix { };
@@ -33,11 +58,10 @@ in {
   # Update the version of pyopenssl.
   pyopenssl = self.callPackage ./pyopenssl.nix {
     pyopenssl =
-      # Building the docs requires sphinx which brings in a dependency on babel,
+      # Building the docs requires sphinx which brings in a dependency on
-      # the test suite of which fails.
+      # babel, the test suite of which fails.  Avoid infinite recursion here
-      onPyPy (dontBuildDocs { sphinx-rtd-theme = null; })
+      # by taking pyopenssl from the `super` package set.
-        # Avoid infinite recursion.
+      onPyPy dontBuildDocs super.pyopenssl;
-        super.pyopenssl;
   };
 
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
@@ -74,7 +98,7 @@ in {
   six = onPyPy dontCheck super.six;
 
   # Likewise for beautifulsoup4.
-  beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
+  beautifulsoup4 = onPyPy dontBuildDocs super.beautifulsoup4;
 
   # The autobahn test suite pulls in a vast number of dependencies for
   # functionality we don't care about.  It might be nice to *selectively*

nix/tahoe-lafs.nix

@@ -1,24 +1,16 @@
+let
+  pname = "tahoe-lafs";
+  version = "1.18.0.post1";
+in
 { lib
 , pythonPackages
 , buildPythonPackage
 , tahoe-lafs-src
-, extrasNames
-
-# control how the test suite is run
-, doCheck
 }:
-let
+buildPythonPackage rec {
-  pname = "tahoe-lafs";
+  inherit pname version;
-  version = "1.18.0.post1";
+  src = tahoe-lafs-src;
-
+  propagatedBuildInputs = with pythonPackages; [
-  pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum  ++ deps.${extra}) [] extras;
-
-  pythonExtraDependencies = with pythonPackages; {
-    tor = [ txtorcon ];
-    i2p = [ txi2p ];
-  };
-
-  pythonPackageDependencies = with pythonPackages; [
     attrs
     autobahn
     cbor2
@@ -41,35 +33,37 @@ let
     six
     treq
     twisted
-    # Get the dependencies for the Twisted extras we depend on, too.
-    twisted.passthru.optional-dependencies.tls
-    twisted.passthru.optional-dependencies.conch
     werkzeug
     zfec
     zope_interface
-  ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
+  ] ++
+  # Get the dependencies for the Twisted extras we depend on, too.
+  twisted.passthru.optional-dependencies.tls ++
+  twisted.passthru.optional-dependencies.conch;
 
-  unitTestDependencies = with pythonPackages; [
+  # The test suite lives elsewhere.
-    beautifulsoup4
+  doCheck = false;
-    fixtures
-    hypothesis
-    mock
-    prometheus-client
-    testtools
-  ];
 
-in
+  passthru = {
-buildPythonPackage {
+    extras = with pythonPackages; {
-  inherit pname version;
+      tor = [ txtorcon ];
-  src = tahoe-lafs-src;
+      i2p = [ txi2p ];
-  propagatedBuildInputs = pythonPackageDependencies;
+      unittest = [
-
+        beautifulsoup4
-  inherit doCheck;
+        fixtures
-  checkInputs = unitTestDependencies;
+        hypothesis
-  checkPhase = ''
+        mock
-    export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
+        prometheus-client
-    python -m twisted.trial -j $NIX_BUILD_CORES allmydata
+        testtools
-  '';
+      ];
+      integrationtest = [
+        pytest
+        pytest-twisted
+        paramiko
+        pytest-timeout
+      ];
+    };
+  };
 
   meta = with lib; {
     homepage = "https://tahoe-lafs.org/";

nix/tests.nix

@@ -1,4 +0,0 @@
-# Build the package with the test suite enabled.
-args@{...}: (import ../. args).override {
-  doCheck = true;
-}