Merge pull request #1272 from exarkun/3991.build-with-nix-for-pypy

Build with Nix for Python 3.11 and PyPy 3.9

Fixes: ticket:3991

default.nix

@@ -32,11 +32,7 @@ in
 
 }:
 with (pkgs.${pythonVersion}.override {
-  packageOverrides = self: super: {
-    # Some dependencies aren't packaged in nixpkgs so supply our own packages.
-    pycddl = self.callPackage ./nix/pycddl.nix { };
-    txi2p = self.callPackage ./nix/txi2p.nix { };
-  };
+  packageOverrides = import ./nix/python-overrides.nix;
 }).pkgs;
 callPackage ./nix/tahoe-lafs.nix {
   # Select whichever package extras were requested.

nix/collections-extended.nix

@@ -0,0 +1,12 @@
+# Package a version that's compatible with Python 3.11.  This can go away once
+# https://github.com/mlenzen/collections-extended/pull/199 is merged and
+# included in a version of nixpkgs we depend on.
+{ fetchFromGitHub, collections-extended }:
+collections-extended.overrideAttrs (old: {
+  src = fetchFromGitHub {
+    owner = "mlenzen";
+    repo = "collections-extended";
+    rev = "8b93390636d58d28012b8e9d22334ee64ca37d73";
+    hash = "sha256-e7RCpNsqyS1d3q0E+uaE4UOEQziueYsRkKEvy3gCHt0=";
+  };
+})

nix/pycddl.nix

@@ -27,7 +27,7 @@
 #
 # 8. run `nix-build`.  it should succeed.  if it does not, seek assistance.
 #
-{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
+{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }:
 buildPythonPackage rec {
   pname = "pycddl";
   version = "0.4.0";
@@ -38,6 +38,12 @@ buildPythonPackage rec {
     sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
   };
 
+  # Without this, when building for PyPy, `maturin build` seems to fail to
+  # find the interpreter at all and then fails early in the build process with
+  # an error saying "unsupported Python interpreter".  We can easily point
+  # directly at the relevant interpreter, so do that.
+  maturinBuildFlags = [ "--interpreter" python.executable ];
+
   nativeBuildInputs = with rustPlatform; [
     maturinBuildHook
     cargoSetupHook

nix/python-overrides.nix

@@ -0,0 +1,133 @@
+# Override various Python packages to create a package set that works for
+# Tahoe-LAFS on CPython and PyPy.
+self: super:
+let
+
+  # Run a function on a derivation if and only if we're building for PyPy.
+  onPyPy = f: drv: if super.isPyPy then f drv else drv;
+
+  # Disable a Python package's test suite.
+  dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; });
+
+  # Disable building a Python package's documentation.
+  dontBuildDocs = alsoDisable: drv: (drv.override ({
+    sphinxHook = null;
+  } // alsoDisable)).overrideAttrs ({ outputs, ... }: {
+    outputs = builtins.filter (x: "doc" != x) outputs;
+  });
+
+in {
+  # Some dependencies aren't packaged in nixpkgs so supply our own packages.
+  pycddl = self.callPackage ./pycddl.nix { };
+  txi2p = self.callPackage ./txi2p.nix { };
+
+  # collections-extended is currently broken for Python 3.11 in nixpkgs but
+  # we know where a working version lives.
+  collections-extended = self.callPackage ./collections-extended.nix {
+    inherit (super) collections-extended;
+  };
+
+  # greenlet is incompatible with PyPy but PyPy has a builtin equivalent.
+  # Fixed in nixpkgs in a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  greenlet = onPyPy (drv: null) super.greenlet;
+
+  # tornado and tk pull in a huge dependency trees for functionality we don't
+  # care about, also tkinter doesn't work on PyPy.
+  matplotlib = super.matplotlib.override { tornado = null; enableTk = false; };
+
+  tqdm = super.tqdm.override {
+    # ibid.
+    tkinter = null;
+    # pandas is only required by the part of the test suite covering
+    # integration with pandas that we don't care about.  pandas is a huge
+    # dependency.
+    pandas = null;
+  };
+
+  # The treq test suite depends on httpbin.  httpbin pulls in babel (flask ->
+  # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow).  babel fails
+  # its test suite and arrow segfaults.
+  treq = onPyPy dontCheck super.treq;
+
+  # the six test suite fails on PyPy because it depends on dbm which the
+  # nixpkgs PyPy build appears to be missing.  Maybe fixed in nixpkgs in
+  # a5f8184fb816a4fd5ae87136838c9981e0d22c67.
+  six = onPyPy dontCheck super.six;
+
+  # Building the docs requires sphinx which brings in a dependency on babel,
+  # the test suite of which fails.
+  pyopenssl = onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) super.pyopenssl;
+
+  # Likewise for beautifulsoup4.
+  beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4;
+
+  # The autobahn test suite pulls in a vast number of dependencies for
+  # functionality we don't care about.  It might be nice to *selectively*
+  # disable just some of it but this is easier.
+  autobahn = onPyPy dontCheck super.autobahn;
+
+  # and python-dotenv tests pulls in a lot of dependencies, including jedi,
+  # which does not work on PyPy.
+  python-dotenv = onPyPy dontCheck super.python-dotenv;
+
+  # Upstream package unaccountably includes a sqlalchemy dependency ... but
+  # the project has no such dependency.  Fixed in nixpkgs in
+  # da10e809fff70fbe1d86303b133b779f09f56503.
+  aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; };
+
+  # By default, the sphinx docs are built, which pulls in a lot of
+  # dependencies - including jedi, which does not work on PyPy.
+  hypothesis =
+    (let h = super.hypothesis;
+     in
+       if (h.override.__functionArgs.enableDocumentation or false)
+       then h.override { enableDocumentation = false; }
+       else h).overrideAttrs ({ nativeBuildInputs, ... }: {
+         # The nixpkgs expression is missing the tzdata check input.
+         nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ];
+       });
+
+  # flaky's test suite depends on nose and nose appears to have Python 3
+  # incompatibilities (it includes `print` statements, for example).
+  flaky = onPyPy dontCheck super.flaky;
+
+  # Replace the deprecated way of running the test suite with the modern way.
+  # This also drops a bunch of unnecessary build-time dependencies, some of
+  # which are broken on PyPy.  Fixed in nixpkgs in
+  # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea.
+  zfec = (super.zfec.override {
+    setuptoolsTrial = null;
+  }).overrideAttrs (old: {
+    checkPhase = "trial zfec";
+  });
+
+  # collections-extended is packaged with poetry-core.  poetry-core test suite
+  # uses virtualenv and virtualenv test suite fails on PyPy.
+  poetry-core = onPyPy dontCheck super.poetry-core;
+
+  # The test suite fails with some rather irrelevant (to us) string comparison
+  # failure on PyPy.  Probably a PyPy bug but doesn't seem like we should
+  # care.
+  rich = onPyPy dontCheck super.rich;
+
+  # The pyutil test suite fails in some ... test ... for some deprecation
+  # functionality we don't care about.
+  pyutil = onPyPy dontCheck super.pyutil;
+
+  # testCall1 fails fairly inscrutibly on PyPy.  Perhaps someone can fix that,
+  # or we could at least just skip that one test.  Probably better to fix it
+  # since we actually depend directly and significantly on Foolscap.
+  foolscap = onPyPy dontCheck super.foolscap;
+
+  # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246
+  psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: {
+    # Upstream already disables some tests but there are even more that have
+    # build impurities that come from build system hardware configuration.
+    # Skip them too.
+    pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray;
+    disabledTests = disabledTests ++ [ "sensors_temperatures" ];
+  });
+
+  # CircleCI build systems don't have enough memory to run this test suite.
+  lz4 = dontCheck super.lz4;
+}

nix/tahoe-lafs.nix

@@ -34,6 +34,7 @@ let
     magic-wormhole
     netifaces
     psutil
+    pyyaml
     pycddl
     pyrsistent
     pyutil
@@ -48,19 +49,15 @@ let
     zope_interface
   ] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
 
-  pythonCheckDependencies = with pythonPackages; [
+  unitTestDependencies = with pythonPackages; [
     beautifulsoup4
     fixtures
     hypothesis
     mock
-    paramiko
     prometheus-client
-    pytest
-    pytest-timeout
-    pytest-twisted
     testtools
-    towncrier
   ];
+
 in
 buildPythonPackage {
   inherit pname version;
@@ -68,7 +65,7 @@ buildPythonPackage {
   propagatedBuildInputs = pythonPackageDependencies;
 
   inherit doCheck;
-  checkInputs = pythonCheckDependencies;
+  checkInputs = unitTestDependencies;
   checkPhase = ''
     export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
     python -m twisted.trial -j $NIX_BUILD_CORES allmydata