more python package tweaks

also point nixpkgs-unstable at HEAD of a PR with a cryptography upgrade
I tried just overriding the upgrade into place but it results in infinite
recursion, I suppose because cryptography is a dependency of some of the build
tools and needs extra handling that I don't feel like figuring out for this
short-term hack.  someday the upgrade will land in nixpkgs master and we can
switch back.

flake.lock

@@ -34,17 +34,34 @@
         "type": "github"
       }
     },
-    "nixpkgs-unstable": {
+    "nixpkgs-23_05": {
       "locked": {
-        "lastModified": 1688486599,
-        "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=",
+        "lastModified": 1688722168,
+        "narHash": "sha256-UDqeQd2neUXICpHAZSS965kGCJsfHkrOFS/vl80I7d8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8",
+        "rev": "28d812a63a0f0d6c1170aac16f5219e506c44b79",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
+        "ref": "release-23.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-unstable": {
+      "locked": {
+        "lastModified": 1687349636,
+        "narHash": "sha256-wpWWNoKJ6z8Nt9egpeiKzsCgkkDO2SO4g6ab9SxgvpM=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "49b7c90c06e557e7473ef467f40d98e7c368d29f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "pull/238965/head",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -56,6 +73,7 @@
           "nixpkgs-22_11"
         ],
         "nixpkgs-22_11": "nixpkgs-22_11",
+        "nixpkgs-23_05": "nixpkgs-23_05",
         "nixpkgs-unstable": "nixpkgs-unstable"
       }
     },

flake.nix

@@ -9,8 +9,11 @@
     "nixpkgs-22_11" = {
       url = github:NixOS/nixpkgs?ref=nixos-22.11;
     };
+    "nixpkgs-23_05" = {
+      url = github:NixOS/nixpkgs?ref=release-23.05;
+    };
     "nixpkgs-unstable" = {
-      url = github:NixOS/nixpkgs;
+      url = github:NixOS/nixpkgs?ref=pull/238965/head;
     };
 
     # Point the default nixpkgs at one of those.  This avoids having getting a
@@ -100,6 +103,8 @@
         name = packageName pyVersion;
       });
     in {
+      legacyPackages = pkgs;
+
       # Define the flake's package outputs.  We'll define one version of the
       # package for each version of Python we could find.  We'll also point
       # the flake's "default" package at one of these somewhat arbitrarily.
@@ -108,7 +113,9 @@
       packages = with pkgs.lib;
         foldr mergeAttrs {} ([
           { default = self.packages.${system}.${packageName defaultPyVersion}; }
-        ] ++ (builtins.map makeRuntimeEnv pythonVersions));
+        ] ++ (builtins.map makeRuntimeEnv pythonVersions)
+        ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions)
+        );
 
       # Define the flake's app outputs.  We'll define a version of an app for
       # running the test suite for each version of Python we could find.

nix/python-overrides.nix

@@ -55,14 +55,22 @@ in {
     inherit (super) txtorcon;
   };
 
-  # Update the version of pyopenssl.
-  pyopenssl = self.callPackage ./pyopenssl.nix {
-    pyopenssl =
-      # Building the docs requires sphinx which brings in a dependency on
-      # babel, the test suite of which fails.  Avoid infinite recursion here
-      # by taking pyopenssl from the `super` package set.
-      onPyPy dontBuildDocs super.pyopenssl;
-  };
+  # With our customized package set a Twisted unit test fails.  Patch the
+  # Twisted test suite to skip that test.
+  twisted = super.twisted.overrideAttrs (old: {
+    patches = (old.patches or []) ++ [ ./twisted.patch ];
+  });
+
+  # Update the version of pyopenssl - and since we're doing that anyway, we
+  # don't need the docs.  Unfortunately this triggers a lot of rebuilding of
+  # dependent packages.
+  pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix {
+    inherit (super) pyopenssl;
+  });
+
+  # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade
+  # that we did breaks service-identity ... so get a newer version that works.
+  service-identity = self.callPackage ./service-identity.nix { };
 
   # collections-extended is currently broken for Python 3.11 in nixpkgs but
   # we know where a working version lives.
@@ -76,16 +84,19 @@ in {
 
   # tornado and tk pull in a huge dependency trees for functionality we don't
   # care about, also tkinter doesn't work on PyPy.
-  matplotlib = super.matplotlib.override { tornado = null; enableTk = false; };
+  matplotlib = onPyPy (matplotlib: matplotlib.override {
+    tornado = null;
+    enableTk = false;
+  }) super.matplotlib;
 
-  tqdm = super.tqdm.override {
+  tqdm = onPyPy (tqdm: tqdm.override {
     # ibid.
     tkinter = null;
     # pandas is only required by the part of the test suite covering
     # integration with pandas that we don't care about.  pandas is a huge
     # dependency.
     pandas = null;
-  };
+  }) super.tqdm;
 
   # The treq test suite depends on httpbin.  httpbin pulls in babel (flask ->
   # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow).  babel fails
@@ -103,57 +114,20 @@ in {
   # The autobahn test suite pulls in a vast number of dependencies for
   # functionality we don't care about.  It might be nice to *selectively*
   # disable just some of it but this is easier.
-  autobahn = onPyPy dontCheck (super.autobahn.override {
-    base58 = null;
-    click = null;
-    ecdsa = null;
-    eth-abi = null;
-    jinja2 = null;
-    hkdf = null;
-    mnemonic = null;
-    py-ecc = null;
-    py-eth-sig-utils = null;
-    py-multihash = null;
-    rlp = null;
-    spake2 = null;
-    yapf = null;
-    eth-account = null;
-  });
+  autobahn = dontCheck super.autobahn;
 
   # and python-dotenv tests pulls in a lot of dependencies, including jedi,
   # which does not work on PyPy.
   python-dotenv = onPyPy dontCheck super.python-dotenv;
 
-  # Upstream package unaccountably includes a sqlalchemy dependency ... but
-  # the project has no such dependency.  Fixed in nixpkgs in
-  # da10e809fff70fbe1d86303b133b779f09f56503.
-  aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; };
-
   # By default, the sphinx docs are built, which pulls in a lot of
   # dependencies - including jedi, which does not work on PyPy.
-  hypothesis =
-    (let h = super.hypothesis;
-     in
-       if (h.override.__functionArgs.enableDocumentation or false)
-       then h.override { enableDocumentation = false; }
-       else h).overrideAttrs ({ nativeBuildInputs, ... }: {
-         # The nixpkgs expression is missing the tzdata check input.
-         nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ];
-       });
+  hypothesis = onPyPy dontBuildDocs super.hypothesis;
 
   # flaky's test suite depends on nose and nose appears to have Python 3
   # incompatibilities (it includes `print` statements, for example).
   flaky = onPyPy dontCheck super.flaky;
 
-  # Replace the deprecated way of running the test suite with the modern way.
-  # This also drops a bunch of unnecessary build-time dependencies, some of
-  # which are broken on PyPy.  Fixed in nixpkgs in
-  # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea.
-  zfec = (overrideIfPresent "setuptoolsTrial" null super.zfec).overrideAttrs (
-    old: {
-      checkPhase = "trial zfec";
-    });
-
   # collections-extended is packaged with poetry-core.  poetry-core test suite
   # uses virtualenv and virtualenv test suite fails on PyPy.
   poetry-core = onPyPy dontCheck super.poetry-core;
@@ -172,15 +146,6 @@ in {
   # since we actually depend directly and significantly on Foolscap.
   foolscap = onPyPy dontCheck super.foolscap;
 
-  # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246
-  psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: {
-    # Upstream already disables some tests but there are even more that have
-    # build impurities that come from build system hardware configuration.
-    # Skip them too.
-    pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray;
-    disabledTests = disabledTests ++ [ "sensors_temperatures" ];
-  });
-
   # CircleCI build systems don't have enough memory to run this test suite.
-  lz4 = dontCheck super.lz4;
+  lz4 = onPyPy dontCheck super.lz4;
 }

nix/service-identity.nix

@@ -0,0 +1,61 @@
+{ lib
+, attrs
+, buildPythonPackage
+, cryptography
+, fetchFromGitHub
+, hatch-fancy-pypi-readme
+, hatch-vcs
+, hatchling
+, idna
+, pyasn1
+, pyasn1-modules
+, pytestCheckHook
+, pythonOlder
+, setuptools
+}:
+
+buildPythonPackage rec {
+  pname = "service-identity";
+  version = "23.1.0";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "pyca";
+    repo = pname;
+    rev = "refs/tags/${version}";
+    hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo=";
+  };
+
+  nativeBuildInputs = [
+    hatch-fancy-pypi-readme
+    hatch-vcs
+    hatchling
+    setuptools
+  ];
+
+  propagatedBuildInputs = [
+    attrs
+    cryptography
+    idna
+    pyasn1
+    pyasn1-modules
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  pythonImportsCheck = [
+    "service_identity"
+  ];
+
+  meta = with lib; {
+    description = "Service identity verification for pyOpenSSL";
+    homepage = "https://service-identity.readthedocs.io";
+    changelog = "https://github.com/pyca/service-identity/releases/tag/${version}";
+    license = licenses.mit;
+    maintainers = with maintainers; [ fab ];
+  };
+}

nix/tahoe-lafs.nix

@@ -46,8 +46,12 @@ buildPythonPackage rec {
 
   passthru = {
     extras = with pythonPackages; {
-      tor = [ txtorcon ];
-      i2p = [ txi2p ];
+      tor = [
+        txtorcon
+      ];
+      i2p = [
+        txi2p
+      ];
       unittest = [
         beautifulsoup4
         fixtures

nix/twisted.patch

@@ -0,0 +1,12 @@
+diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py
+index c650fd8aa6..a1754fd533 100644
+--- a/src/twisted/internet/test/test_endpoints.py
++++ b/src/twisted/internet/test/test_endpoints.py
+@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase):
+         connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint)
+         self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com")
+ 
++    @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails")
+     def test_tls(self):
+         """
+         When passed a string endpoint description beginning with C{tls:},