From 92474375350022fc12521b060160beeba777e3db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:43:23 -0400 Subject: [PATCH] Add a flake with some packages and apps and an overlay --- default.nix | 29 +------ flake.lock | 80 +++++++++++++++++++ flake.nix | 162 +++++++++++++++++++++++++++++++++++++++ nix/overlay.nix | 10 +++ nix/python-overrides.nix | 42 +++++++--- nix/tahoe-lafs.nix | 74 ++++++++---------- nix/tests.nix | 4 - 7 files changed, 323 insertions(+), 78 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/overlay.nix delete mode 100644 nix/tests.nix diff --git a/default.nix b/default.nix index d616f63b8..196db4030 100644 --- a/default.nix +++ b/default.nix @@ -18,32 +18,11 @@ in pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the # niv-managed sources data -, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself +, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself + overlays = [ (import ./nix/overlay.nix) ]; +} , pythonVersion ? "python310" # a string choosing the python derivation from # nixpkgs to target - -, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, - # the dependencies of which the resulting - # package will also depend on. Include all of the - # runtime extras by default because the incremental - # cost of including them is a lot smaller than the - # cost of re-building the whole thing to add them. - }: -with (pkgs.${pythonVersion}.override { - packageOverrides = import ./nix/python-overrides.nix; -}).pkgs; -callPackage ./nix/tahoe-lafs.nix { - # Select whichever package extras were requested. - inherit extrasNames; - - # Define the location of the Tahoe-LAFS source to be packaged (the same - # directory as contains this file). Clean up as many of the non-source - # files (eg the `.git` directory, `~` backup files, nix's own `result` - # symlink, etc) as possible to avoid needing to re-build when files that - # make no difference to the package have changed. - tahoe-lafs-src = pkgs.lib.cleanSource ./.; - - doCheck = false; -} +pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ]) diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..0fd3c90c9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,80 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1687709756, + "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs-22_11": { + "locked": { + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1688486599, + "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs-22_11" + ], + "nixpkgs-22_11": "nixpkgs-22_11", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..89a4a6f14 --- /dev/null +++ b/flake.nix @@ -0,0 +1,162 @@ +{ + description = "Tahoe-LAFS, free and open decentralized data store"; + + inputs = { + # Two alternate nixpkgs pins. Ideally these could be selected easily from + # the command line but there seems to be no syntax/support for that. + # However, these at least cause certain revisions to be pinned in our lock + # file where you *can* dig them out - and the CI configuration does. + "nixpkgs-22_11" = { + url = github:NixOS/nixpkgs?ref=nixos-22.11; + }; + "nixpkgs-unstable" = { + url = github:NixOS/nixpkgs; + }; + + # Point the default nixpkgs at one of those. This avoids having getting a + # _third_ package set involved and gives a way to provide what should be a + # working experience by default (that is, if nixpkgs doesn't get + # overridden). + nixpkgs.follows = "nixpkgs-22_11"; + + # Also get flake-utils for simplified multi-system definitions. + flake-utils = { + url = github:numtide/flake-utils; + }; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + { + # Expose an overlay which adds our version of Tahoe-LAFS to the Python + # package sets we specify, as well as all of the correct versions of its + # dependencies. + # + # We will also use this to define some other outputs since it gives us + # the most succinct way to get a working Tahoe-LAFS package. + overlays.default = import ./nix/overlay.nix; + + } // (flake-utils.lib.eachDefaultSystem (system: let + + # First get the package set for this system architecture. + pkgs = import nixpkgs { + inherit system; + # And include our Tahoe-LAFS package in that package set. + overlays = [ self.overlays.default ]; + }; + + # Find out what Python versions we're working with. + pythonVersions = builtins.attrNames ( + pkgs.lib.attrsets.filterAttrs + # Match attribute names that look like a Python derivation - CPython + # or PyPy. We take care to avoid things like "python-foo" and + # "python3Full-unittest" though. We only want things like "pypy38" + # or "python311". + (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name) + pkgs + ); + + # An element of pythonVersions which we'll use for the default package. + defaultPyVersion = "python310"; + + # Retrieve the actual Python package for each configured version. We + # already applied our overlay to pkgs so our packages will already be + # available. + pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions; + + # string -> string + # + # Construct the Tahoe-LAFS package name for the given Python runtime. + packageName = pyVersion: "${pyVersion}-tahoe-lafs"; + + # string -> string + # + # Construct the unit test application name for the given Python runtime. + unitTestName = pyVersion: "${pyVersion}-unittest"; + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeRuntimeEnv = pyVersion: { + ${packageName pyVersion} = makeRuntimeEnv' pyVersion; + }; + + makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeTestEnv = pyVersion: { + ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor ++ + tahoe-lafs.passthru.extras.unittest + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + }; + in { + # Define the flake's package outputs. We'll define one version of the + # package for each version of Python we could find. We'll also point + # the flake's "default" package at one of these somewhat arbitrarily. + # The package consists of a Python environment with Tahoe-LAFS available + # to it. + packages = with pkgs.lib; + foldr mergeAttrs {} ([ + { default = self.packages.${system}.${packageName defaultPyVersion}; } + ] ++ (builtins.map makeRuntimeEnv pythonVersions)); + + # Define the flake's app outputs. We'll define a version of an app for + # running the test suite for each version of Python we could find. + # We'll also define a version of an app for running the "tahoe" + # command-line entrypoint for each version of Python we could find. + apps = + let + # We avoid writeShellApplication here because it has ghc as a + # dependency but ghc has Python as a dependency and our Python + # package override triggers a rebuild of ghc which takes a looong + # time. + writeScript = name: text: + let script = pkgs.writeShellScript name text; + in "${script}"; + + # A helper function to define the runtime entrypoint for a certain + # Python runtime. + makeTahoeApp = pyVersion: { + "tahoe-${pyVersion}" = { + type = "app"; + program = + writeScript "tahoe" + '' + ${makeRuntimeEnv' pyVersion}/bin/tahoe "$@" + ''; + }; + }; + + # A helper function to define the unit test entrypoint for a certain + # Python runtime. + makeUnitTestsApp = pyVersion: { + "${unitTestName pyVersion}" = { + type = "app"; + program = + writeScript "unit-tests" + '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ''; + }; + }; + in + with pkgs.lib; + foldr mergeAttrs + { default = self.apps.${system}."tahoe-python3"; } + ( + (builtins.map makeUnitTestsApp pythonVersions) ++ + (builtins.map makeTahoeApp pythonVersions) + ); + })); +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 000000000..41f0e3086 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,10 @@ +# This overlay adds Tahoe-LAFS and all of its properly-configured Python +# package dependencies to a Python package set. Downstream consumers can +# apply it to their own nixpkgs derivation to produce a Tahoe-LAFS package. +final: prev: { + # Add our overrides such that they will be applied to any Python derivation + # in nixpkgs. + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (import ./python-overrides.nix) + ]; +} diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 0ed415691..a333808be 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -9,14 +9,39 @@ let # Disable a Python package's test suite. dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; }); + # string -> any -> derivation -> derivation + # + # If the overrideable function for the given derivation accepts an argument + # with the given name, override it with the given value. + # + # Since we try to work with multiple versions of nixpkgs, sometimes we need + # to override a parameter that exists in one version but not others. This + # makes it a bit easier to do so. + overrideIfPresent = name: value: drv: + if (drv.override.__functionArgs ? ${name}) + then drv.override { "${name}" = value; } + else drv; + # Disable building a Python package's documentation. - dontBuildDocs = alsoDisable: drv: (drv.override ({ - sphinxHook = null; - } // alsoDisable)).overrideAttrs ({ outputs, ... }: { + dontBuildDocs = drv: ( + overrideIfPresent "sphinxHook" null ( + overrideIfPresent "sphinx-rtd-theme" null + drv + ) + ).overrideAttrs ({ outputs, ... }: { outputs = builtins.filter (x: "doc" != x) outputs; }); in { + tahoe-lafs = self.callPackage ./tahoe-lafs.nix { + # Define the location of the Tahoe-LAFS source to be packaged (the same + # directory as contains this file). Clean up as many of the non-source + # files (eg the `.git` directory, `~` backup files, nix's own `result` + # symlink, etc) as possible to avoid needing to re-build when files that + # make no difference to the package have changed. + tahoe-lafs-src = self.lib.cleanSource ../.; + }; + # Some dependencies aren't packaged in nixpkgs so supply our own packages. pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; @@ -33,11 +58,10 @@ in { # Update the version of pyopenssl. pyopenssl = self.callPackage ./pyopenssl.nix { pyopenssl = - # Building the docs requires sphinx which brings in a dependency on babel, - # the test suite of which fails. - onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) - # Avoid infinite recursion. - super.pyopenssl; + # Building the docs requires sphinx which brings in a dependency on + # babel, the test suite of which fails. Avoid infinite recursion here + # by taking pyopenssl from the `super` package set. + onPyPy dontBuildDocs super.pyopenssl; }; # collections-extended is currently broken for Python 3.11 in nixpkgs but @@ -74,7 +98,7 @@ in { six = onPyPy dontCheck super.six; # Likewise for beautifulsoup4. - beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; + beautifulsoup4 = onPyPy dontBuildDocs super.beautifulsoup4; # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index bf3ea83d3..f7037e1ae 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,24 +1,16 @@ +let + pname = "tahoe-lafs"; + version = "1.18.0.post1"; +in { lib , pythonPackages , buildPythonPackage , tahoe-lafs-src -, extrasNames - -# control how the test suite is run -, doCheck }: -let - pname = "tahoe-lafs"; - version = "1.18.0.post1"; - - pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras; - - pythonExtraDependencies = with pythonPackages; { - tor = [ txtorcon ]; - i2p = [ txi2p ]; - }; - - pythonPackageDependencies = with pythonPackages; [ +buildPythonPackage rec { + inherit pname version; + src = tahoe-lafs-src; + propagatedBuildInputs = with pythonPackages; [ attrs autobahn cbor2 @@ -41,35 +33,37 @@ let six treq twisted - # Get the dependencies for the Twisted extras we depend on, too. - twisted.passthru.optional-dependencies.tls - twisted.passthru.optional-dependencies.conch werkzeug zfec zope_interface - ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; + ] ++ + # Get the dependencies for the Twisted extras we depend on, too. + twisted.passthru.optional-dependencies.tls ++ + twisted.passthru.optional-dependencies.conch; - unitTestDependencies = with pythonPackages; [ - beautifulsoup4 - fixtures - hypothesis - mock - prometheus-client - testtools - ]; + # The test suite lives elsewhere. + doCheck = false; -in -buildPythonPackage { - inherit pname version; - src = tahoe-lafs-src; - propagatedBuildInputs = pythonPackageDependencies; - - inherit doCheck; - checkInputs = unitTestDependencies; - checkPhase = '' - export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - python -m twisted.trial -j $NIX_BUILD_CORES allmydata - ''; + passthru = { + extras = with pythonPackages; { + tor = [ txtorcon ]; + i2p = [ txi2p ]; + unittest = [ + beautifulsoup4 + fixtures + hypothesis + mock + prometheus-client + testtools + ]; + integrationtest = [ + pytest + pytest-twisted + paramiko + pytest-timeout + ]; + }; + }; meta = with lib; { homepage = "https://tahoe-lafs.org/"; diff --git a/nix/tests.nix b/nix/tests.nix deleted file mode 100644 index 42ca9f882..000000000 --- a/nix/tests.nix +++ /dev/null @@ -1,4 +0,0 @@ -# Build the package with the test suite enabled. -args@{...}: (import ../. args).override { - doCheck = true; -}