diff --git a/.circleci/config.yml b/.circleci/config.yml index 7312c2973..51e3976c7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,14 +87,15 @@ workflows: {} - "nixos": - name: "nixos-<>" - nixpkgs: "nixpkgs-unstable" + name: "<>-<>" matrix: parameters: + nixpkgs: + - "nixpkgs-24_11" pythonVersion: - - "python39" - "python310" - "python311" + - "python312" # Eventually, test against PyPy 3.8 #- "pypy27-buster": @@ -555,15 +556,12 @@ jobs: buildSteps: - "run": name: "Unit Test" + environment: + # Once dependencies are built, we can allow some more concurrency for our own + # test suite. + UNITTEST_CORES: 8 command: | - source .circleci/lib.sh - - # Translate the nixpkgs selection into a flake reference we - # can use to override the default nixpkgs input. - NIXPKGS=$(nixpkgs_flake_reference <>) - - cache_if_able nix run \ - --override-input nixpkgs "$NIXPKGS" \ + nix run \ .#<>-unittest -- \ --jobs $UNITTEST_CORES \ allmydata @@ -693,15 +691,8 @@ executors: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.16.1" + image: "nixos/nix:2.25.3" environment: - # currently, all NixOS builds are broken; ignore them - ALLOWED_FAILURE: "yes" - - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us - # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push - # to. - CACHIX_NAME: "tahoe-lafs-opensource" # Let us use features marked "experimental". For example, most/all of # the `nix ` forms. NIX_CONFIG: "experimental-features = nix-command flakes" @@ -724,34 +715,21 @@ commands: type: "steps" steps: - - "run": - # Get cachix for Nix-friendly caching. - name: "Install Basic Dependencies" - command: | - # Get some build environment dependencies and let them float on a - # certain release branch. These aren't involved in the actual - # package build (only in CI environment setup) so the fact that - # they float shouldn't hurt reproducibility. - NIXPKGS="nixpkgs/nixos-23.05" - nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp - - # Activate our cachix cache for "binary substitution". This sets - # up configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" - - "checkout" - "run": name: "Build Package" + environment: + # CircleCI build environment looks like it has a zillion and a half cores. + # Don't let Nix autodetect this high core count because it blows up memory + # usage and fails the test run. Pick a number of cores that suits the build + # environment we're paying for (the free one!). + DEPENDENCY_CORES: 3 command: | - source .circleci/lib.sh - NIXPKGS=$(nixpkgs_flake_reference <>) - cache_if_able nix build \ + nix build \ --verbose \ --print-build-logs \ --cores "$DEPENDENCY_CORES" \ - --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs - steps: "<>" diff --git a/.circleci/lib.sh b/.circleci/lib.sh deleted file mode 100644 index a53c33dce..000000000 --- a/.circleci/lib.sh +++ /dev/null @@ -1,148 +0,0 @@ -# CircleCI build environment looks like it has a zillion and a half cores. -# Don't let Nix autodetect this high core count because it blows up memory -# usage and fails the test run. Pick a number of cores that suits the build -# environment we're paying for (the free one!). -DEPENDENCY_CORES=3 - -# Once dependencies are built, we can allow some more concurrency for our own -# test suite. -UNITTEST_CORES=8 - -# Run a command, enabling cache writes to cachix if possible. The command is -# accepted as a variable number of positional arguments (like argv). -function cache_if_able() { - # Dump some info about our build environment. - describe_build - - if is_cache_writeable; then - # If the cache is available we'll use it. This lets fork owners set - # up their own caching if they want. - echo "Cachix credentials present; will attempt to write to cache." - - # The `cachix watch-exec ...` does our cache population. When it sees - # something added to the store (I guess) it pushes it to the named - # cache. - cachix watch-exec "${CACHIX_NAME}" -- "$@" - else - if is_cache_required; then - echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." - return 1 - else - echo "Cachix credentials missing; will not attempt cache writes." - "$@" - fi - fi -} - -function is_cache_writeable() { - # We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo - # jobs will get this from CircleCI configuration but jobs from forks may - # not. - [ -v CACHIX_AUTH_TOKEN ] -} - -function is_cache_required() { - # If we're building in tahoe-lafs/tahoe-lafs then we must use the cache. - # If we're building anything from a fork then we're allowed to not have - # the credentials. - is_upstream -} - -# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs -# repository itself (and so we expect to have cache credentials available), -# failure otherwise. -# -# See circleci.txt for notes about how this determination is made. -function is_upstream() { - # CIRCLE_PROJECT_USERNAME is set to the org the build is happening for. - # If a PR targets a fork of the repo then this is set to something other - # than "tahoe-lafs". - [ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] && - - # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and - # "pull/NNNN" for pull requests from forks. - # - # CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full - # URLs of the PR pages which share an underlying branch, with one of - # them ended with that same "pull/NNNN" for PRs from forks. - ! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS" -} - -# Return success if splitting $3 on $2 results in an array with any element -# that ends with $1, failure otherwise. -function any_element_endswith() { - suffix=$1 - shift - - sep=$1 - shift - - haystack=$1 - shift - - IFS="${sep}" read -r -a elements <<< "$haystack" - for elem in "${elements[@]}"; do - if endswith "$suffix" "$elem"; then - return 0 - fi - done - return 1 -} - -# Return success if $2 ends with $1, failure otherwise. -function endswith() { - suffix=$1 - shift - - haystack=$1 - shift - - case "$haystack" in - *${suffix}) - return 0 - ;; - - *) - return 1 - ;; - esac -} - -function describe_build() { - echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}" - echo "Building branch: ${CIRCLE_BRANCH}" - if is_upstream; then - echo "Upstream build." - else - echo "Non-upstream build." - fi - if is_cache_required; then - echo "Cache is required." - else - echo "Cache not required." - fi - if is_cache_writeable; then - echo "Cache is writeable." - else - echo "Cache not writeable." - fi -} - -# Inspect the flake input metadata for an input of a given name and return the -# revision at which that input is pinned. If the input does not exist then -# return garbage (probably "null"). -read_input_revision() { - input_name=$1 - shift - - nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev' -} - -# Return a flake reference that refers to a certain revision of nixpkgs. The -# certain revision is the revision to which the specified input is pinned. -nixpkgs_flake_reference() { - input_name=$1 - shift - - echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)" -} diff --git a/flake.lock b/flake.lock index b7b74a0e4..cd383b229 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -21,11 +21,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1687709756, - "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -34,50 +34,18 @@ "type": "github" } }, - "nixpkgs-22_11": { + "nixpkgs-24_11": { "locked": { - "lastModified": 1688392541, - "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "lastModified": 1733261153, + "narHash": "sha256-eq51hyiaIwtWo19fPEeE0Zr2s83DYMKJoukNLgGGpek=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "rev": "b681065d0919f7eb5309a93cea2cfa84dec9aa88", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-23_05": { - "locked": { - "lastModified": 1689885880, - "narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-23.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-unstable": { - "locked": { - "lastModified": 1689791806, - "narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "pull/244135/head", + "ref": "nixos-24.11", "repo": "nixpkgs", "type": "github" } @@ -87,11 +55,9 @@ "flake-compat": "flake-compat", "flake-utils": "flake-utils", "nixpkgs": [ - "nixpkgs-unstable" + "nixpkgs-24_11" ], - "nixpkgs-22_11": "nixpkgs-22_11", - "nixpkgs-23_05": "nixpkgs-23_05", - "nixpkgs-unstable": "nixpkgs-unstable" + "nixpkgs-24_11": "nixpkgs-24_11" } }, "systems": { diff --git a/flake.nix b/flake.nix index bde792db3..cc851d984 100644 --- a/flake.nix +++ b/flake.nix @@ -1,14 +1,6 @@ { description = "Tahoe-LAFS, free and open decentralized data store"; - nixConfig = { - # Supply configuration for the build cache updated by our CI system. This - # should allow most users to avoid having to build a large number of - # packages (otherwise necessary due to our Python package overrides). - substituters = ["https://tahoe-lafs-opensource.cachix.org"]; - trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="]; - }; - inputs = { # A couple possible nixpkgs pins. Ideally these could be selected easily # from the command line but there seems to be no syntax/support for that. @@ -20,25 +12,12 @@ # requirements. We could decide in the future that supporting multiple # releases of NixOS at a time is worthwhile and then pins like these will # help us test each of those releases. - "nixpkgs-22_11" = { - url = github:NixOS/nixpkgs?ref=nixos-22.11; - }; - "nixpkgs-23_05" = { - url = github:NixOS/nixpkgs?ref=nixos-23.05; + "nixpkgs-24_11" = { + url = github:NixOS/nixpkgs?ref=nixos-24.11; }; - # We depend on a very new python-cryptography which is not yet available - # from any release branch of nixpkgs. However, it is contained in a PR - # currently up for review. Point our nixpkgs at that for now. - "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs?ref=pull/244135/head; - }; - - # Point the default nixpkgs at one of those. This avoids having getting a - # _third_ package set involved and gives a way to provide what should be a - # working experience by default (that is, if nixpkgs doesn't get - # overridden). - nixpkgs.follows = "nixpkgs-unstable"; + # Point the default nixpkgs at one of those. + nixpkgs.follows = "nixpkgs-24_11"; # Also get flake-utils for simplified multi-system definitions. flake-utils = { @@ -153,7 +132,8 @@ [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor ++ - tahoe-lafs.passthru.extras.unittest + tahoe-lafs.passthru.extras.unittest ++ + [ hatchling hatch-vcs ] )).overrideAttrs (old: { # See the similar override in makeRuntimeEnv'. name = packageName pyVersion; @@ -219,10 +199,11 @@ program = let python = "${makeTestEnv pyVersion}/bin/python"; + hatchling = "${makeTestEnv pyVersion}/bin/hatchling"; in writeScript "unit-tests" '' - ${python} setup.py update_version + ${hatchling} build --hooks-only # Write _version.py export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci export PYTHONPATH=$PWD/src ${python} -m twisted.trial "$@" diff --git a/newsfragments/4134.minor b/newsfragments/4134.minor new file mode 100644 index 000000000..46fe1d3d3 --- /dev/null +++ b/newsfragments/4134.minor @@ -0,0 +1,4 @@ +Avoid private cache from Cachix until we can restore it. +Update nixpkgs to 24.11 wich is well cached for now. +Stop packaging and testing on nixpkgs/python39 (too old). +Start packaging and testing on nixpkgs/python312 instead. diff --git a/nix/klein.nix b/nix/klein.nix deleted file mode 100644 index be4426465..000000000 --- a/nix/klein.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ klein, fetchPypi }: -klein.overrideAttrs (old: rec { - pname = "klein"; - version = "23.5.0"; - src = fetchPypi { - inherit pname version; - sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc="; - }; -}) diff --git a/nix/pycddl.nix b/nix/pycddl.nix deleted file mode 100644 index 8b214a91b..000000000 --- a/nix/pycddl.nix +++ /dev/null @@ -1,57 +0,0 @@ -# package https://gitlab.com/tahoe-lafs/pycddl -# -# also in the process of being pushed upstream -# https://github.com/NixOS/nixpkgs/pull/221220 -# -# we should switch to the upstream package when it is available from our -# minimum version of nixpkgs. -# -# if you need to update this package to a new pycddl release then -# -# 1. change value given to `buildPythonPackage` for `version` to match the new -# release -# -# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` -# -# 3. run `nix-build` -# -# 4. there will be an error about a hash mismatch. change the value given to -# `fetchPypi` for `sha256` to the "actual" hash value report. -# -# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`. -# -# 6. run `nix-build` -# -# 7. there will be an error about a hash mismatch. change the value given to -# `cargoDeps` for `hash` to the "actual" hash value report. -# -# 8. run `nix-build`. it should succeed. if it does not, seek assistance. -# -{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }: -buildPythonPackage rec { - pname = "pycddl"; - version = "0.6.1"; - format = "pyproject"; - - src = fetchPypi { - inherit pname version; - sha256 = "sha256-63fe8UJXEH6t4l7ujV8JDvlGb7q3kL6fHHATFdklzFc="; - }; - - # Without this, when building for PyPy, `maturin build` seems to fail to - # find the interpreter at all and then fails early in the build process with - # an error saying "unsupported Python interpreter". We can easily point - # directly at the relevant interpreter, so do that. - maturinBuildFlags = [ "--interpreter" python.executable ]; - - nativeBuildInputs = with rustPlatform; [ - maturinBuildHook - cargoSetupHook - ]; - - cargoDeps = rustPlatform.fetchCargoTarball { - inherit src; - name = "${pname}-${version}"; - hash = "sha256-ssDEKRd3Y9/10oXBZHCxvlRkl9KMh3pGYbCkM4rXThQ="; - }; -} diff --git a/nix/pyopenssl.nix b/nix/pyopenssl.nix deleted file mode 100644 index b8966fad1..000000000 --- a/nix/pyopenssl.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ pyopenssl, fetchPypi, isPyPy }: -pyopenssl.overrideAttrs (old: rec { - pname = "pyOpenSSL"; - version = "23.2.0"; - name = "${pname}-${version}"; - src = fetchPypi { - inherit pname version; - sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w="; - }; -}) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 006c2682d..1094e08a3 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -42,40 +42,10 @@ in { tahoe-lafs-src = self.lib.cleanSource ../.; }; - # Some dependencies aren't packaged in nixpkgs so supply our own packages. - pycddl = self.callPackage ./pycddl.nix { }; - txi2p = self.callPackage ./txi2p.nix { }; - - # Some packages are of somewhat too-old versions - update them. - klein = self.callPackage ./klein.nix { - # Avoid infinite recursion. - inherit (super) klein; - }; - txtorcon = self.callPackage ./txtorcon.nix { - inherit (super) txtorcon; - }; - - # With our customized package set a Twisted unit test fails. Patch the - # Twisted test suite to skip that test. - # Filed upstream at https://github.com/twisted/twisted/issues/11877 - twisted = super.twisted.overrideAttrs (old: { - patches = (old.patches or []) ++ [ ./twisted.patch ]; - }); - - # Update the version of pyopenssl - and since we're doing that anyway, we - # don't need the docs. Unfortunately this triggers a lot of rebuilding of - # dependent packages. - pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix { - inherit (super) pyopenssl; - }); - - # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade - # that we did breaks service-identity ... so get a newer version that works. - service-identity = self.callPackage ./service-identity.nix { }; - # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. collections-extended = self.callPackage ./collections-extended.nix { + # Avoid infinite recursion. inherit (super) collections-extended; }; diff --git a/nix/service-identity.nix b/nix/service-identity.nix deleted file mode 100644 index fef68b16e..000000000 --- a/nix/service-identity.nix +++ /dev/null @@ -1,61 +0,0 @@ -{ lib -, attrs -, buildPythonPackage -, cryptography -, fetchFromGitHub -, hatch-fancy-pypi-readme -, hatch-vcs -, hatchling -, idna -, pyasn1 -, pyasn1-modules -, pytestCheckHook -, pythonOlder -, setuptools -}: - -buildPythonPackage rec { - pname = "service-identity"; - version = "23.1.0"; - format = "pyproject"; - - disabled = pythonOlder "3.8"; - - src = fetchFromGitHub { - owner = "pyca"; - repo = pname; - rev = "refs/tags/${version}"; - hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo="; - }; - - nativeBuildInputs = [ - hatch-fancy-pypi-readme - hatch-vcs - hatchling - setuptools - ]; - - propagatedBuildInputs = [ - attrs - cryptography - idna - pyasn1 - pyasn1-modules - ]; - - nativeCheckInputs = [ - pytestCheckHook - ]; - - pythonImportsCheck = [ - "service_identity" - ]; - - meta = with lib; { - description = "Service identity verification for pyOpenSSL"; - homepage = "https://service-identity.readthedocs.io"; - changelog = "https://github.com/pyca/service-identity/releases/tag/${version}"; - license = licenses.mit; - maintainers = with maintainers; [ fab ]; - }; -} diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 273fa3a76..f4734358a 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -9,6 +9,7 @@ in }: buildPythonPackage rec { inherit pname version; + pyproject = true; src = tahoe-lafs-src; propagatedBuildInputs = with pythonPackages; [ attrs @@ -22,6 +23,8 @@ buildPythonPackage rec { filelock foolscap future + hatchling + hatch-vcs klein magic-wormhole netifaces @@ -50,7 +53,7 @@ buildPythonPackage rec { txtorcon ]; i2p = [ - txi2p + txi2p-tahoe ]; unittest = [ beautifulsoup4 diff --git a/nix/twisted.patch b/nix/twisted.patch deleted file mode 100644 index 1b6846c8e..000000000 --- a/nix/twisted.patch +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py -index c650fd8aa6..a1754fd533 100644 ---- a/src/twisted/internet/test/test_endpoints.py -+++ b/src/twisted/internet/test/test_endpoints.py -@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase): - connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint) - self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com") - -+ @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails") - def test_tls(self): - """ - When passed a string endpoint description beginning with C{tls:}, diff --git a/nix/txi2p.nix b/nix/txi2p.nix deleted file mode 100644 index 3464b7b3d..000000000 --- a/nix/txi2p.nix +++ /dev/null @@ -1,39 +0,0 @@ -# package https://github.com/tahoe-lafs/txi2p -# -# if you need to update this package to a new txi2p release then -# -# 1. change value given to `buildPythonPackage` for `version` to match the new -# release -# -# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash` -# -# 3. run `nix-build` -# -# 4. there will be an error about a hash mismatch. change the value given to -# `fetchPypi` for `sha256` to the "actual" hash value report. -# -# 5. if there are new runtime dependencies then add them to the argument list -# at the top. if there are new test dependencies add them to the -# `checkInputs` list. -# -# 6. run `nix-build`. it should succeed. if it does not, seek assistance. -# -{ fetchPypi -, buildPythonPackage -, parsley -, twisted -, unittestCheckHook -}: -buildPythonPackage rec { - pname = "txi2p-tahoe"; - version = "0.3.7"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY="; - }; - - propagatedBuildInputs = [ twisted parsley ]; - checkInputs = [ unittestCheckHook ]; - pythonImportsCheck = [ "parsley" "ometa"]; -} diff --git a/nix/txtorcon.nix b/nix/txtorcon.nix deleted file mode 100644 index 552c03fd0..000000000 --- a/nix/txtorcon.nix +++ /dev/null @@ -1,9 +0,0 @@ -{ txtorcon, fetchPypi }: -txtorcon.overrideAttrs (old: rec { - pname = "txtorcon"; - version = "23.5.0"; - src = fetchPypi { - inherit pname version; - hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw="; - }; -})