From 19aa0cd1c2b1f73615233f46645230bb5db50893 Mon Sep 17 00:00:00 2001 From: Benoit Donneaux Date: Thu, 5 Dec 2024 11:09:04 +0100 Subject: [PATCH 01/15] Avoid Cachix until we recover the onwership on that cache Co-authored-by: Florian Sesser Signed-off-by: Benoit Donneaux --- .circleci/config.yml | 16 +++------------- flake.nix | 8 -------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6377ece0b..dfa95456d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -602,7 +602,7 @@ jobs: # can use to override the default nixpkgs input. NIXPKGS=$(nixpkgs_flake_reference <>) - cache_if_able nix run \ + nix run \ --override-input nixpkgs "$NIXPKGS" \ .#<>-unittest -- \ --jobs $UNITTEST_CORES \ @@ -743,10 +743,6 @@ executors: - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.16.1" environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us - # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push - # to. - CACHIX_NAME: "tahoe-lafs-opensource" # Let us use features marked "experimental". For example, most/all of # the `nix ` forms. NIX_CONFIG: "experimental-features = nix-command flakes" @@ -770,7 +766,6 @@ commands: steps: - "run": - # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | # Get some build environment dependencies and let them float on a @@ -778,12 +773,7 @@ commands: # package build (only in CI environment setup) so the fact that # they float shouldn't hurt reproducibility. NIXPKGS="nixpkgs/nixos-23.05" - nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp - - # Activate our cachix cache for "binary substitution". This sets - # up configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" + nix profile install $NIXPKGS#bash $NIXPKGS#jp - "checkout" @@ -800,7 +790,7 @@ commands: command: | source .circleci/lib.sh NIXPKGS=$(nixpkgs_flake_reference <>) - cache_if_able nix build \ + nix build \ --verbose \ --print-build-logs \ --cores "$DEPENDENCY_CORES" \ diff --git a/flake.nix b/flake.nix index bde792db3..44000c9ab 100644 --- a/flake.nix +++ b/flake.nix @@ -1,14 +1,6 @@ { description = "Tahoe-LAFS, free and open decentralized data store"; - nixConfig = { - # Supply configuration for the build cache updated by our CI system. This - # should allow most users to avoid having to build a large number of - # packages (otherwise necessary due to our Python package overrides). - substituters = ["https://tahoe-lafs-opensource.cachix.org"]; - trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="]; - }; - inputs = { # A couple possible nixpkgs pins. Ideally these could be selected easily # from the command line but there seems to be no syntax/support for that. From 3187e00735cb7146d956a6644bb304d8f57bb895 Mon Sep 17 00:00:00 2001 From: Benoit Donneaux Date: Thu, 5 Dec 2024 11:22:00 +0100 Subject: [PATCH 02/15] Update nixpkgs to 24.11, drop test for python39 but add python312 Co-authored-by: Florian Sesser Signed-off-by: Benoit Donneaux --- .circleci/config.yml | 11 ++++---- flake.lock | 60 ++++++++++---------------------------------- flake.nix | 21 +++------------- 3 files changed, 23 insertions(+), 69 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dfa95456d..bf18f1f89 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,14 +93,15 @@ workflows: {} - "nixos": - name: "<>" - nixpkgs: "nixpkgs-unstable" + name: "<>-<>" matrix: parameters: + nixpkgs: + - "nixpkgs-24_11" pythonVersion: - - "python39" - "python310" - "python311" + - "python312" # Eventually, test against PyPy 3.8 #- "pypy27-buster": @@ -741,7 +742,7 @@ executors: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.16.1" + image: "nixos/nix:2.25.3" environment: # Let us use features marked "experimental". For example, most/all of # the `nix ` forms. @@ -772,7 +773,7 @@ commands: # certain release branch. These aren't involved in the actual # package build (only in CI environment setup) so the fact that # they float shouldn't hurt reproducibility. - NIXPKGS="nixpkgs/nixos-23.05" + NIXPKGS="nixpkgs/nixos-24.11" nix profile install $NIXPKGS#bash $NIXPKGS#jp - "checkout" diff --git a/flake.lock b/flake.lock index b7b74a0e4..cd383b229 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -21,11 +21,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1687709756, - "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -34,50 +34,18 @@ "type": "github" } }, - "nixpkgs-22_11": { + "nixpkgs-24_11": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1733261153, + "narHash": "sha256-eq51hyiaIwtWo19fPEeE0Zr2s83DYMKJoukNLgGGpek=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "b681065d0919f7eb5309a93cea2cfa84dec9aa88", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-23_05": { - "locked": { - "lastModified": 1689885880, - "narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-unstable": { - "locked": { - "lastModified": 1689791806, - "narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "pull/244135/head", + "ref": "nixos-24.11", "repo": "nixpkgs", "type": "github" } @@ -87,11 +55,9 @@ "flake-compat": "flake-compat", "flake-utils": "flake-utils", "nixpkgs": [ - "nixpkgs-unstable" + "nixpkgs-24_11" ], - "nixpkgs-22_11": "nixpkgs-22_11", - "nixpkgs-23_05": "nixpkgs-23_05", - "nixpkgs-unstable": "nixpkgs-unstable" + "nixpkgs-24_11": "nixpkgs-24_11" } }, "systems": { diff --git a/flake.nix b/flake.nix index 44000c9ab..8a2d3c326 100644 --- a/flake.nix +++ b/flake.nix @@ -12,25 +12,12 @@ # requirements. We could decide in the future that supporting multiple # releases of NixOS at a time is worthwhile and then pins like these will # help us test each of those releases. - "nixpkgs-22_11" = { - url = github:NixOS/nixpkgs?ref=nixos-22.11; - }; - "nixpkgs-23_05" = { - url = github:NixOS/nixpkgs?ref=nixos-23.05; + "nixpkgs-24_11" = { + url = github:NixOS/nixpkgs?ref=nixos-24.11; }; - # We depend on a very new python-cryptography which is not yet available - # from any release branch of nixpkgs. However, it is contained in a PR - # currently up for review. Point our nixpkgs at that for now. - "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs?ref=pull/244135/head; - }; - - # Point the default nixpkgs at one of those. This avoids having getting a - # _third_ package set involved and gives a way to provide what should be a - # working experience by default (that is, if nixpkgs doesn't get - # overridden). - nixpkgs.follows = "nixpkgs-unstable"; + # Point the default nixpkgs at one of those. + nixpkgs.follows = "nixpkgs-24_11"; # Also get flake-utils for simplified multi-system definitions. flake-utils = { From cf8909a2b53da91a6dfe93385974031dd8bfc916 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Tue, 5 Nov 2024 17:43:45 +0000 Subject: [PATCH 03/15] Update PyOpenSSL to the second-latest version The latest version 404s ? --- nix/pyopenssl.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix index b8966fad1..100be4784 100644 --- a/nix/pyopenssl.nix +++ b/nix/pyopenssl.nix @@ -1,10 +1,10 @@ { pyopenssl, fetchPypi, isPyPy }: pyopenssl.overrideAttrs (old: rec { pname = "pyOpenSSL"; - version = "23.2.0"; + version = "24.1.0"; name = "${pname}-${version}"; src = fetchPypi { inherit pname version; - sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w="; + sha256 = "yr7Uv6pd+fGhbA72Sgy2Uxi1zQd6ftp9aXATHKL0Gm8="; }; }) From c43fab3a04c44e4d7bc142feb049953718e6f592 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 17:43:57 +0000 Subject: [PATCH 04/15] Drop our vendored pycddl, upstream has our fixes now --- nix/pycddl.nix | 57 ---------------------------------------- nix/python-overrides.nix | 1 - 2 files changed, 58 deletions(-) delete mode 100644 nix/pycddl.nix diff --git a/nix/pycddl.nix b/nix/pycddl.nix deleted file mode 100644 index 8b214a91b..000000000 --- a/nix/pycddl.nix +++ /dev/null @@ -1,57 +0,0 @@ -# package https://gitlab.com/tahoe-lafs/pycddl -# -# also in the process of being pushed upstream -# https://github.com/NixOS/nixpkgs/pull/221220 -# -# we should switch to the upstream package when it is available from our -# minimum version of nixpkgs. -# -# if you need to update this package to a new pycddl release then -# -# 1. change value given to `buildPythonPackage` for `version` to match the new -# release -# -# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` -# -# 3. run `nix-build` -# -# 4. there will be an error about a hash mismatch. change the value given to -# `fetchPypi` for `sha256` to the "actual" hash value report. -# -# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`. -# -# 6. run `nix-build` -# -# 7. there will be an error about a hash mismatch. change the value given to -# `cargoDeps` for `hash` to the "actual" hash value report. -# -# 8. run `nix-build`. it should succeed. if it does not, seek assistance. -# -{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }: -buildPythonPackage rec { - pname = "pycddl"; - version = "0.6.1"; - format = "pyproject"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256-63fe8UJXEH6t4l7ujV8JDvlGb7q3kL6fHHATFdklzFc="; - }; - - # Without this, when building for PyPy, `maturin build` seems to fail to - # find the interpreter at all and then fails early in the build process with - # an error saying "unsupported Python interpreter". We can easily point - # directly at the relevant interpreter, so do that. - maturinBuildFlags = [ "--interpreter" python.executable ]; - - nativeBuildInputs = with rustPlatform; [ - maturinBuildHook - cargoSetupHook - ]; - - cargoDeps = rustPlatform.fetchCargoTarball { - inherit src; - name = "${pname}-${version}"; - hash = "sha256-ssDEKRd3Y9/10oXBZHCxvlRkl9KMh3pGYbCkM4rXThQ="; - }; -} diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 006c2682d..2e42ef3d4 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -43,7 +43,6 @@ in { }; # Some dependencies aren't packaged in nixpkgs so supply our own packages. - pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; # Some packages are of somewhat too-old versions - update them. From 76726f24f2e1e0837e51b8d722adb2cedfb6bd5b Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 18:21:08 +0000 Subject: [PATCH 05/15] Use the nixpkgs-provided klein --- nix/klein.nix | 9 --------- nix/python-overrides.nix | 5 +---- 2 files changed, 1 insertion(+), 13 deletions(-) delete mode 100644 nix/klein.nix diff --git a/nix/klein.nix b/nix/klein.nix deleted file mode 100644 index be4426465..000000000 --- a/nix/klein.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ klein, fetchPypi }: -klein.overrideAttrs (old: rec { - pname = "klein"; - version = "23.5.0"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc="; - }; -}) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 2e42ef3d4..4621fd321 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -46,11 +46,8 @@ in { txi2p = self.callPackage ./txi2p.nix { }; # Some packages are of somewhat too-old versions - update them. - klein = self.callPackage ./klein.nix { - # Avoid infinite recursion. - inherit (super) klein; - }; txtorcon = self.callPackage ./txtorcon.nix { + # Avoid infinite recursion. inherit (super) txtorcon; }; From 916a68439ac921dd21fd88d3dd8ca778e527d5d6 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 18:54:52 +0000 Subject: [PATCH 06/15] Use the nixpkgs-provided pyopenssl --- nix/pyopenssl.nix | 10 ------- nix/python-overrides.nix | 7 ----- nix/service-identity.nix | 61 ---------------------------------------- 3 files changed, 78 deletions(-) delete mode 100644 nix/pyopenssl.nix delete mode 100644 nix/service-identity.nix diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix deleted file mode 100644 index 100be4784..000000000 --- a/nix/pyopenssl.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ pyopenssl, fetchPypi, isPyPy }: -pyopenssl.overrideAttrs (old: rec { - pname = "pyOpenSSL"; - version = "24.1.0"; - name = "${pname}-${version}"; - src = fetchPypi { - inherit pname version; - sha256 = "yr7Uv6pd+fGhbA72Sgy2Uxi1zQd6ftp9aXATHKL0Gm8="; - }; -}) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 4621fd321..3faa4aa99 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -58,13 +58,6 @@ in { patches = (old.patches or []) ++ [ ./twisted.patch ]; }); - # Update the version of pyopenssl - and since we're doing that anyway, we - # don't need the docs. Unfortunately this triggers a lot of rebuilding of - # dependent packages. - pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix { - inherit (super) pyopenssl; - }); - # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade # that we did breaks service-identity ... so get a newer version that works. service-identity = self.callPackage ./service-identity.nix { }; diff --git a/nix/service-identity.nix b/nix/service-identity.nix deleted file mode 100644 index fef68b16e..000000000 --- a/nix/service-identity.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib -, attrs -, buildPythonPackage -, cryptography -, fetchFromGitHub -, hatch-fancy-pypi-readme -, hatch-vcs -, hatchling -, idna -, pyasn1 -, pyasn1-modules -, pytestCheckHook -, pythonOlder -, setuptools -}: - -buildPythonPackage rec { - pname = "service-identity"; - version = "23.1.0"; - format = "pyproject"; - - disabled = pythonOlder "3.8"; - - src = fetchFromGitHub { - owner = "pyca"; - repo = pname; - rev = "refs/tags/${version}"; - hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo="; - }; - - nativeBuildInputs = [ - hatch-fancy-pypi-readme - hatch-vcs - hatchling - setuptools - ]; - - propagatedBuildInputs = [ - attrs - cryptography - idna - pyasn1 - pyasn1-modules - ]; - - nativeCheckInputs = [ - pytestCheckHook - ]; - - pythonImportsCheck = [ - "service_identity" - ]; - - meta = with lib; { - description = "Service identity verification for pyOpenSSL"; - homepage = "https://service-identity.readthedocs.io"; - changelog = "https://github.com/pyca/service-identity/releases/tag/${version}"; - license = licenses.mit; - maintainers = with maintainers; [ fab ]; - }; -} From e01453e0fdbaf8ea8dd12583aa2395a7ca57292c Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 19:00:18 +0000 Subject: [PATCH 07/15] Use the nixpkgs-provided twisted --- nix/python-overrides.nix | 11 ----------- nix/twisted.patch | 12 ------------ 2 files changed, 23 deletions(-) delete mode 100644 nix/twisted.patch diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 3faa4aa99..22bac1ca1 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -51,17 +51,6 @@ in { inherit (super) txtorcon; }; - # With our customized package set a Twisted unit test fails. Patch the - # Twisted test suite to skip that test. - # Filed upstream at https://github.com/twisted/twisted/issues/11877 - twisted = super.twisted.overrideAttrs (old: { - patches = (old.patches or []) ++ [ ./twisted.patch ]; - }); - - # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade - # that we did breaks service-identity ... so get a newer version that works. - service-identity = self.callPackage ./service-identity.nix { }; - # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { diff --git a/nix/twisted.patch b/nix/twisted.patch deleted file mode 100644 index 1b6846c8e..000000000 --- a/nix/twisted.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py -index c650fd8aa6..a1754fd533 100644 ---- a/src/twisted/internet/test/test_endpoints.py -+++ b/src/twisted/internet/test/test_endpoints.py -@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase): - connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint) - self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com") - -+ @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails") - def test_tls(self): - """ - When passed a string endpoint description beginning with C{tls:}, From 37373133b910b956cc27728747ceceb192880813 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 19:07:34 +0000 Subject: [PATCH 08/15] Use nixpkgs-provided txtorcon --- nix/python-overrides.nix | 7 +------ nix/txtorcon.nix | 9 --------- 2 files changed, 1 insertion(+), 15 deletions(-) delete mode 100644 nix/txtorcon.nix diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 22bac1ca1..48a8e4515 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -45,15 +45,10 @@ in { # Some dependencies aren't packaged in nixpkgs so supply our own packages. txi2p = self.callPackage ./txi2p.nix { }; - # Some packages are of somewhat too-old versions - update them. - txtorcon = self.callPackage ./txtorcon.nix { - # Avoid infinite recursion. - inherit (super) txtorcon; - }; - # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { + # Avoid infinite recursion. inherit (super) collections-extended; }; diff --git a/nix/txtorcon.nix b/nix/txtorcon.nix deleted file mode 100644 index 552c03fd0..000000000 --- a/nix/txtorcon.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ txtorcon, fetchPypi }: -txtorcon.overrideAttrs (old: rec { - pname = "txtorcon"; - version = "23.5.0"; - src = fetchPypi { - inherit pname version; - hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw="; - }; -}) From b37d6022cb3630bd69f0c374b9f0e210629cae3f Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 19:08:16 +0000 Subject: [PATCH 09/15] Use nixpkgs-provided txi2p-tahoe --- nix/python-overrides.nix | 3 --- nix/txi2p.nix | 39 --------------------------------------- 2 files changed, 42 deletions(-) delete mode 100644 nix/txi2p.nix diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 48a8e4515..1094e08a3 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -42,9 +42,6 @@ in { tahoe-lafs-src = self.lib.cleanSource ../.; }; - # Some dependencies aren't packaged in nixpkgs so supply our own packages. - txi2p = self.callPackage ./txi2p.nix { }; - # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { diff --git a/nix/txi2p.nix b/nix/txi2p.nix deleted file mode 100644 index 3464b7b3d..000000000 --- a/nix/txi2p.nix +++ /dev/null @@ -1,39 +0,0 @@ -# package https://github.com/tahoe-lafs/txi2p -# -# if you need to update this package to a new txi2p release then -# -# 1. change value given to `buildPythonPackage` for `version` to match the new -# release -# -# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` -# -# 3. run `nix-build` -# -# 4. there will be an error about a hash mismatch. change the value given to -# `fetchPypi` for `sha256` to the "actual" hash value report. -# -# 5. if there are new runtime dependencies then add them to the argument list -# at the top. if there are new test dependencies add them to the -# `checkInputs` list. -# -# 6. run `nix-build`. it should succeed. if it does not, seek assistance. -# -{ fetchPypi -, buildPythonPackage -, parsley -, twisted -, unittestCheckHook -}: -buildPythonPackage rec { - pname = "txi2p-tahoe"; - version = "0.3.7"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY="; - }; - - propagatedBuildInputs = [ twisted parsley ]; - checkInputs = [ unittestCheckHook ]; - pythonImportsCheck = [ "parsley" "ometa"]; -} From af8e84fc768be94748c9737de622b1c9090bf155 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 19:15:07 +0000 Subject: [PATCH 10/15] Spell txi2p correctly --- nix/tahoe-lafs.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 273fa3a76..fa538767d 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -50,7 +50,7 @@ buildPythonPackage rec { txtorcon ]; i2p = [ - txi2p + txi2p-tahoe ]; unittest = [ beautifulsoup4 From cf82b0b3fadcf9eec313a3e159eb67aef3e57b2f Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 4 Dec 2024 20:04:32 +0000 Subject: [PATCH 11/15] Patch _version.py Courtesy of the ZKAPAuthorizer nix expressions --- nix/tahoe-lafs.nix | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index fa538767d..7d4a631af 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -70,6 +70,24 @@ buildPythonPackage rec { }; }; + postPatch = + let + versionFileContents = version: '' + # This _version.py is generated by tahoe-lafs.nix. + # TODO: We can have more metadata after we switch to flakes. + # Then the `self` input will have a `sourceInfo` attribute telling + __pkgname__ = "tahoe-lafs" + real_version = "${version}" + full_version = "${version}" + branch = "" + verstr = "${version}" + __version__ = verstr + ''; + in + '' + cp ${builtins.toFile "_version.py" (versionFileContents version)} src/allmydata/_version.py + ''; + meta = with lib; { homepage = "https://tahoe-lafs.org/"; description = "secure, decentralized, fault-tolerant file store"; From b5d54c21844041def3992b501b16233a32de4a77 Mon Sep 17 00:00:00 2001 From: Benoit Donneaux Date: Thu, 5 Dec 2024 12:26:59 +0100 Subject: [PATCH 12/15] Add news fragment about fixing the Nix CI Co-authored-by: Florian Sesser Signed-off-by: Benoit Donneaux --- newsfragments/4134.minor | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 newsfragments/4134.minor diff --git a/newsfragments/4134.minor b/newsfragments/4134.minor new file mode 100644 index 000000000..46fe1d3d3 --- /dev/null +++ b/newsfragments/4134.minor @@ -0,0 +1,4 @@ +Avoid private cache from Cachix until we can restore it. +Update nixpkgs to 24.11 wich is well cached for now. +Stop packaging and testing on nixpkgs/python39 (too old). +Start packaging and testing on nixpkgs/python312 instead. From de355ec634bb5165a1cb3a9bdaa0b10136b651c6 Mon Sep 17 00:00:00 2001 From: Benoit Donneaux Date: Fri, 6 Dec 2024 00:04:28 +0100 Subject: [PATCH 13/15] Remove nixpkgs override and the lib we no longer use Signed-off-by: Benoit Donneaux --- .circleci/config.yml | 30 +++------ .circleci/lib.sh | 148 ------------------------------------------- 2 files changed, 10 insertions(+), 168 deletions(-) delete mode 100644 .circleci/lib.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index bf18f1f89..21d08cff6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -596,15 +596,12 @@ jobs: buildSteps: - "run": name: "Unit Test" + environment: + # Once dependencies are built, we can allow some more concurrency for our own + # test suite. + UNITTEST_CORES: 8 command: | - source .circleci/lib.sh - - # Translate the nixpkgs selection into a flake reference we - # can use to override the default nixpkgs input. - NIXPKGS=$(nixpkgs_flake_reference <>) - nix run \ - --override-input nixpkgs "$NIXPKGS" \ .#<>-unittest -- \ --jobs $UNITTEST_CORES \ allmydata @@ -766,16 +763,6 @@ commands: type: "steps" steps: - - "run": - name: "Install Basic Dependencies" - command: | - # Get some build environment dependencies and let them float on a - # certain release branch. These aren't involved in the actual - # package build (only in CI environment setup) so the fact that - # they float shouldn't hurt reproducibility. - NIXPKGS="nixpkgs/nixos-24.11" - nix profile install $NIXPKGS#bash $NIXPKGS#jp - - "checkout" - "run": @@ -788,14 +775,17 @@ commands: - "run": name: "Build Package" + environment: + # CircleCI build environment looks like it has a zillion and a half cores. + # Don't let Nix autodetect this high core count because it blows up memory + # usage and fails the test run. Pick a number of cores that suits the build + # environment we're paying for (the free one!). + DEPENDENCY_CORES: 3 command: | - source .circleci/lib.sh - NIXPKGS=$(nixpkgs_flake_reference <>) nix build \ --verbose \ --print-build-logs \ --cores "$DEPENDENCY_CORES" \ - --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs - steps: "<>" diff --git a/.circleci/lib.sh b/.circleci/lib.sh deleted file mode 100644 index a53c33dce..000000000 --- a/.circleci/lib.sh +++ /dev/null @@ -1,148 +0,0 @@ -# CircleCI build environment looks like it has a zillion and a half cores. -# Don't let Nix autodetect this high core count because it blows up memory -# usage and fails the test run. Pick a number of cores that suits the build -# environment we're paying for (the free one!). -DEPENDENCY_CORES=3 - -# Once dependencies are built, we can allow some more concurrency for our own -# test suite. -UNITTEST_CORES=8 - -# Run a command, enabling cache writes to cachix if possible. The command is -# accepted as a variable number of positional arguments (like argv). -function cache_if_able() { - # Dump some info about our build environment. - describe_build - - if is_cache_writeable; then - # If the cache is available we'll use it. This lets fork owners set - # up their own caching if they want. - echo "Cachix credentials present; will attempt to write to cache." - - # The `cachix watch-exec ...` does our cache population. When it sees - # something added to the store (I guess) it pushes it to the named - # cache. - cachix watch-exec "${CACHIX_NAME}" -- "$@" - else - if is_cache_required; then - echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." - return 1 - else - echo "Cachix credentials missing; will not attempt cache writes." - "$@" - fi - fi -} - -function is_cache_writeable() { - # We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo - # jobs will get this from CircleCI configuration but jobs from forks may - # not. - [ -v CACHIX_AUTH_TOKEN ] -} - -function is_cache_required() { - # If we're building in tahoe-lafs/tahoe-lafs then we must use the cache. - # If we're building anything from a fork then we're allowed to not have - # the credentials. - is_upstream -} - -# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs -# repository itself (and so we expect to have cache credentials available), -# failure otherwise. -# -# See circleci.txt for notes about how this determination is made. -function is_upstream() { - # CIRCLE_PROJECT_USERNAME is set to the org the build is happening for. - # If a PR targets a fork of the repo then this is set to something other - # than "tahoe-lafs". - [ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] && - - # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and - # "pull/NNNN" for pull requests from forks. - # - # CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full - # URLs of the PR pages which share an underlying branch, with one of - # them ended with that same "pull/NNNN" for PRs from forks. - ! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS" -} - -# Return success if splitting $3 on $2 results in an array with any element -# that ends with $1, failure otherwise. -function any_element_endswith() { - suffix=$1 - shift - - sep=$1 - shift - - haystack=$1 - shift - - IFS="${sep}" read -r -a elements <<< "$haystack" - for elem in "${elements[@]}"; do - if endswith "$suffix" "$elem"; then - return 0 - fi - done - return 1 -} - -# Return success if $2 ends with $1, failure otherwise. -function endswith() { - suffix=$1 - shift - - haystack=$1 - shift - - case "$haystack" in - *${suffix}) - return 0 - ;; - - *) - return 1 - ;; - esac -} - -function describe_build() { - echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}" - echo "Building branch: ${CIRCLE_BRANCH}" - if is_upstream; then - echo "Upstream build." - else - echo "Non-upstream build." - fi - if is_cache_required; then - echo "Cache is required." - else - echo "Cache not required." - fi - if is_cache_writeable; then - echo "Cache is writeable." - else - echo "Cache not writeable." - fi -} - -# Inspect the flake input metadata for an input of a given name and return the -# revision at which that input is pinned. If the input does not exist then -# return garbage (probably "null"). -read_input_revision() { - input_name=$1 - shift - - nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev' -} - -# Return a flake reference that refers to a certain revision of nixpkgs. The -# certain revision is the revision to which the specified input is pinned. -nixpkgs_flake_reference() { - input_name=$1 - shift - - echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)" -} From 086f93432c138a987b72f66f84a8c885c0047622 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 11 Dec 2024 19:49:35 +0000 Subject: [PATCH 14/15] Adapt to Tahoe-LAFS' new build system --- flake.nix | 1 - nix/tahoe-lafs.nix | 21 +++------------------ 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/flake.nix b/flake.nix index 8a2d3c326..f1cbb459f 100644 --- a/flake.nix +++ b/flake.nix @@ -201,7 +201,6 @@ in writeScript "unit-tests" '' - ${python} setup.py update_version export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci export PYTHONPATH=$PWD/src ${python} -m twisted.trial "$@" diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 7d4a631af..f4734358a 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -9,6 +9,7 @@ in }: buildPythonPackage rec { inherit pname version; + pyproject = true; src = tahoe-lafs-src; propagatedBuildInputs = with pythonPackages; [ attrs @@ -22,6 +23,8 @@ buildPythonPackage rec { filelock foolscap future + hatchling + hatch-vcs klein magic-wormhole netifaces @@ -70,24 +73,6 @@ buildPythonPackage rec { }; }; - postPatch = - let - versionFileContents = version: '' - # This _version.py is generated by tahoe-lafs.nix. - # TODO: We can have more metadata after we switch to flakes. - # Then the `self` input will have a `sourceInfo` attribute telling - __pkgname__ = "tahoe-lafs" - real_version = "${version}" - full_version = "${version}" - branch = "" - verstr = "${version}" - __version__ = verstr - ''; - in - '' - cp ${builtins.toFile "_version.py" (versionFileContents version)} src/allmydata/_version.py - ''; - meta = with lib; { homepage = "https://tahoe-lafs.org/"; description = "secure, decentralized, fault-tolerant file store"; From 4de350890441d7cf570abd376900cbade5b6ce66 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 12 Dec 2024 17:05:22 +0000 Subject: [PATCH 15/15] Make hatchling write us a _version.py when running the tests Co-authored-by: Benoit Donneaux --- flake.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index f1cbb459f..cc851d984 100644 --- a/flake.nix +++ b/flake.nix @@ -132,7 +132,8 @@ [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor ++ - tahoe-lafs.passthru.extras.unittest + tahoe-lafs.passthru.extras.unittest ++ + [ hatchling hatch-vcs ] )).overrideAttrs (old: { # See the similar override in makeRuntimeEnv'. name = packageName pyVersion; @@ -198,9 +199,11 @@ program = let python = "${makeTestEnv pyVersion}/bin/python"; + hatchling = "${makeTestEnv pyVersion}/bin/hatchling"; in writeScript "unit-tests" '' + ${hatchling} build --hooks-only # Write _version.py export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci export PYTHONPATH=$PWD/src ${python} -m twisted.trial "$@"