From 900b4a3c989f0ef707b0ab39b72063aa3f991a7c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:55:41 -0400 Subject: [PATCH 1/6] Package a version of collections-extended compatible with Python 3.11 --- nix/collections-extended.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 nix/collections-extended.nix diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix new file mode 100644 index 000000000..05254fc1b --- /dev/null +++ b/nix/collections-extended.nix @@ -0,0 +1,12 @@ +# Package a version that's compatible with Python 3.11. This can go away once +# https://github.com/mlenzen/collections-extended/pull/199 is merged and +# included in a version of nixpkgs we depend on. +{ fetchFromGitHub, collections-extended }: +collections-extended.overrideAttrs (old: { + src = fetchFromGitHub { + owner = "mlenzen"; + repo = "collections-extended"; + rev = "8b93390636d58d28012b8e9d22334ee64ca37d73"; + hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0="; + }; +}) From 41d5538921e07d291e3f88f6ffaf60d8c2e68daa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:56:05 -0400 Subject: [PATCH 2/6] Fix `maturin build` when using PyPy for the pycddl package --- nix/pycddl.nix | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nix/pycddl.nix b/nix/pycddl.nix index 703f00595..4c68830d4 100644 --- a/nix/pycddl.nix +++ b/nix/pycddl.nix @@ -27,7 +27,7 @@ # # 8. run `nix-build`. it should succeed. if it does not, seek assistance. # -{ lib, fetchPypi, buildPythonPackage, rustPlatform }: +{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }: buildPythonPackage rec { pname = "pycddl"; version = "0.4.0"; @@ -38,6 +38,12 @@ buildPythonPackage rec { sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY="; }; + # Without this, when building for PyPy, `maturin build` seems to fail to + # find the interpreter at all and then fails early in the build process with + # an error saying "unsupported Python interpreter". We can easily point + # directly at the relevant interpreter, so do that. + maturinBuildFlags = [ "--interpreter" python.executable ]; + nativeBuildInputs = with rustPlatform; [ maturinBuildHook cargoSetupHook From dd8f6d408d27908de924ce400c7c717536095f0b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:56:50 -0400 Subject: [PATCH 3/6] Remove the non-unit test dependencies from the unit test inputs --- nix/tahoe-lafs.nix | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 5986db420..2e1c4aa39 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -48,19 +48,15 @@ let zope_interface ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; - pythonCheckDependencies = with pythonPackages; [ + unitTestDependencies = with pythonPackages; [ beautifulsoup4 fixtures hypothesis mock - paramiko prometheus-client - pytest - pytest-timeout - pytest-twisted testtools - towncrier ]; + in buildPythonPackage { inherit pname version; @@ -68,7 +64,7 @@ buildPythonPackage { propagatedBuildInputs = pythonPackageDependencies; inherit doCheck; - checkInputs = pythonCheckDependencies; + checkInputs = unitTestDependencies; checkPhase = '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata From 35b921b11d1fdf23d8039a7cf4b526eb56474995 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 08:57:21 -0400 Subject: [PATCH 4/6] Put Python package overrides in one place, and add a lot more of them These packaging changes fix issues against CPython 3.11 or PyPy. --- default.nix | 6 +- nix/python-overrides.nix | 133 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 nix/python-overrides.nix diff --git a/default.nix b/default.nix index b87a6730a..d616f63b8 100644 --- a/default.nix +++ b/default.nix @@ -32,11 +32,7 @@ in }: with (pkgs.${pythonVersion}.override { - packageOverrides = self: super: { - # Some dependencies aren't packaged in nixpkgs so supply our own packages. - pycddl = self.callPackage ./nix/pycddl.nix { }; - txi2p = self.callPackage ./nix/txi2p.nix { }; - }; + packageOverrides = import ./nix/python-overrides.nix; }).pkgs; callPackage ./nix/tahoe-lafs.nix { # Select whichever package extras were requested. diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix new file mode 100644 index 000000000..87c42ad58 --- /dev/null +++ b/nix/python-overrides.nix @@ -0,0 +1,133 @@ +# Override various Python packages to create a package set that works for +# Tahoe-LAFS on CPython and PyPy. +self: super: +let + + # Run a function on a derivation if and only if we're building for PyPy. + onPyPy = f: drv: if super.isPyPy then f drv else drv; + + # Disable a Python package's test suite. + dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; }); + + # Disable building a Python package's documentation. + dontBuildDocs = alsoDisable: drv: (drv.override ({ + sphinxHook = null; + } // alsoDisable)).overrideAttrs ({ outputs, ... }: { + outputs = builtins.filter (x: "doc" != x) outputs; + }); + +in { + # Some dependencies aren't packaged in nixpkgs so supply our own packages. + pycddl = self.callPackage ./pycddl.nix { }; + txi2p = self.callPackage ./txi2p.nix { }; + + # collections-extended is currently broken for Python 3.11 in nixpkgs but + # we know where a working version lives. + collections-extended = self.callPackage ./collections-extended.nix { + inherit (super) collections-extended; + }; + + # greenlet is incompatible with PyPy but PyPy has a builtin equivalent. + # Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67. + greenlet = onPyPy (drv: null) super.greenlet; + + # tornado and tk pull in a huge dependency trees for functionality we don't + # care about, also tkinter doesn't work on PyPy. + matplotlib = super.matplotlib.override { tornado = null; enableTk = false; }; + + tqdm = super.tqdm.override { + # ibid. + tkinter = null; + # pandas is only required by the part of the test suite covering + # integration with pandas that we don't care about. pandas is a huge + # dependency. + pandas = null; + }; + + # The treq test suite depends on httpbin. httpbin pulls in babel (flask -> + # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow). babel fails + # its test suite and arrow segfaults. + treq = onPyPy dontCheck super.treq; + + # the six test suite fails on PyPy because it depends on dbm which the + # nixpkgs PyPy build appears to be missing. Maybe fixed in nixpkgs in + # a5f8184fb816a4fd5ae87136838c9981e0d22c67. + six = onPyPy dontCheck super.six; + + # Building the docs requires sphinx which brings in a dependency on babel, + # the test suite of which fails. + pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl; + + # Likewise for beautifulsoup4. + beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; + + # The autobahn test suite pulls in a vast number of dependencies for + # functionality we don't care about. It might be nice to *selectively* + # disable just some of it but this is easier. + autobahn = onPyPy dontCheck super.autobahn; + + # and python-dotenv tests pulls in a lot of dependencies, including jedi, + # which does not work on PyPy. + python-dotenv = onPyPy dontCheck super.python-dotenv; + + # Upstream package unaccountably includes a sqlalchemy dependency ... but + # the project has no such dependency. Fixed in nixpkgs in + # da10e809fff70fbe1d86303b133b779f09f56503. + aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; }; + + # By default, the sphinx docs are built, which pulls in a lot of + # dependencies - including jedi, which does not work on PyPy. + hypothesis = + (let h = super.hypothesis; + in + if (h.override.__functionArgs.enableDocumentation or false) + then h.override { enableDocumentation = false; } + else h).overrideAttrs ({ nativeBuildInputs, ... }: { + # The nixpkgs expression is missing the tzdata check input. + nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ]; + }); + + # flaky's test suite depends on nose and nose appears to have Python 3 + # incompatibilities (it includes `print` statements, for example). + flaky = onPyPy dontCheck super.flaky; + + # Replace the deprecated way of running the test suite with the modern way. + # This also drops a bunch of unnecessary build-time dependencies, some of + # which are broken on PyPy. Fixed in nixpkgs in + # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. + zfec = (super.zfec.override { + setuptoolsTrial = null; + }).overrideAttrs (old: { + checkPhase = "trial zfec"; + }); + + # collections-extended is packaged with poetry-core. poetry-core test suite + # uses virtualenv and virtualenv test suite fails on PyPy. + poetry-core = onPyPy dontCheck super.poetry-core; + + # The test suite fails with some rather irrelevant (to us) string comparison + # failure on PyPy. Probably a PyPy bug but doesn't seem like we should + # care. + rich = onPyPy dontCheck super.rich; + + # The pyutil test suite fails in some ... test ... for some deprecation + # functionality we don't care about. + pyutil = onPyPy dontCheck super.pyutil; + + # testCall1 fails fairly inscrutibly on PyPy. Perhaps someone can fix that, + # or we could at least just skip that one test. Probably better to fix it + # since we actually depend directly and significantly on Foolscap. + foolscap = onPyPy dontCheck super.foolscap; + + # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246 + psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: { + # Upstream already disables some tests but there are even more that have + # build impurities that come from build system hardware configuration. + # Skip them too. + pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray; + disabledTests = disabledTests ++ [ "sensors_temperatures" ]; + }); + + # CircleCI build systems don't have enough memory to run this test suite. + lz4 = dontCheck super.lz4; +} From a173df4561fb3fad1898f32421553962f7659ff1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 21 Mar 2023 09:29:12 -0400 Subject: [PATCH 5/6] news fragment --- newsfragments/3991.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3991.minor diff --git a/newsfragments/3991.minor b/newsfragments/3991.minor new file mode 100644 index 000000000..e69de29bb From 226da2fb2afa5961f8580619002905e3aeec580d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 26 Mar 2023 11:49:17 -0400 Subject: [PATCH 6/6] Add missing pyyaml dependency It worked without this because we got the pyyaml dependency transitively but we should declare it directly since it is a direct dependency. --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 2e1c4aa39..bf3ea83d3 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -34,6 +34,7 @@ let magic-wormhole netifaces psutil + pyyaml pycddl pyrsistent pyutil