ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-18 20:47:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'leastauthority/master' into 4131.remove-python-3.8-only

Browse Source
This commit is contained in:
Benoit Donneaux 2024-12-13 11:38:22 +01:00
commit a74a82d538
14 changed files with 47 additions and 490 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
.circleci/config.yml
View File

@ -90,14 +90,15 @@ workflows:
{}
- "nixos":
name: "nixos-<<matrix.pythonVersion>>"
nixpkgs: "nixpkgs-unstable"
name: "<<matrix.nixpkgs>>-<<matrix.pythonVersion>>"
matrix:
parameters:
nixpkgs:
- "nixpkgs-24_11"
pythonVersion:
- "python39"
- "python310"
- "python311"
- "python312"
# Other assorted tasks and configurations
- "codechecks":
@ -570,15 +571,12 @@ jobs:
buildSteps:
- "run":
name: "Unit Test"
environment:
# Once dependencies are built, we can allow some more concurrency for our own
# test suite.
UNITTEST_CORES: 8
command: |
source .circleci/lib.sh
# Translate the nixpkgs selection into a flake reference we
# can use to override the default nixpkgs input.
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix run \
--override-input nixpkgs "$NIXPKGS" \
nix run \
.#<<parameters.pythonVersion>>-unittest -- \
--jobs $UNITTEST_CORES \
allmydata
@ -716,15 +714,8 @@ executors:
docker:
# Run in a highly Nix-capable environment.
- <<: *DOCKERHUB_AUTH
image: "nixos/nix:2.16.1"
image: "nixos/nix:2.25.3"
environment:
# currently, all NixOS builds are broken; ignore them
ALLOWED_FAILURE: "yes"
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
# to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push
# to.
CACHIX_NAME: "tahoe-lafs-opensource"
# Let us use features marked "experimental". For example, most/all of
# the `nix <subcommand>` forms.
NIX_CONFIG: "experimental-features = nix-command flakes"
@ -747,34 +738,21 @@ commands:
type: "steps"
steps:
- "run":
# Get cachix for Nix-friendly caching.
name: "Install Basic Dependencies"
command: |
# Get some build environment dependencies and let them float on a
# certain release branch. These aren't involved in the actual
# package build (only in CI environment setup) so the fact that
# they float shouldn't hurt reproducibility.
NIXPKGS="nixpkgs/nixos-23.05"
nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp
# Activate our cachix cache for "binary substitution". This sets
# up configuration tht lets Nix download something from the cache
# instead of building it locally, if possible.
cachix use "${CACHIX_NAME}"
- "checkout"
- "run":
name: "Build Package"
environment:
# CircleCI build environment looks like it has a zillion and a half cores.
# Don't let Nix autodetect this high core count because it blows up memory
# usage and fails the test run. Pick a number of cores that suits the build
# environment we're paying for (the free one!).
DEPENDENCY_CORES: 3
command: |
source .circleci/lib.sh
NIXPKGS=$(nixpkgs_flake_reference <<parameters.nixpkgs>>)
cache_if_able nix build \
nix build \
--verbose \
--print-build-logs \
--cores "$DEPENDENCY_CORES" \
--override-input nixpkgs "$NIXPKGS" \
.#<<parameters.pythonVersion>>-tahoe-lafs
- steps: "<<parameters.buildSteps>>"

148
.circleci/lib.sh
View File

@ -1,148 +0,0 @@
# CircleCI build environment looks like it has a zillion and a half cores.
# Don't let Nix autodetect this high core count because it blows up memory
# usage and fails the test run. Pick a number of cores that suits the build
# environment we're paying for (the free one!).
DEPENDENCY_CORES=3
# Once dependencies are built, we can allow some more concurrency for our own
# test suite.
UNITTEST_CORES=8
# Run a command, enabling cache writes to cachix if possible. The command is
# accepted as a variable number of positional arguments (like argv).
function cache_if_able() {
# Dump some info about our build environment.
describe_build
if is_cache_writeable; then
# If the cache is available we'll use it. This lets fork owners set
# up their own caching if they want.
echo "Cachix credentials present; will attempt to write to cache."
# The `cachix watch-exec ...` does our cache population. When it sees
# something added to the store (I guess) it pushes it to the named
# cache.
cachix watch-exec "${CACHIX_NAME}" -- "$@"
else
if is_cache_required; then
echo "Required credentials (CACHIX_AUTH_TOKEN) are missing."
return 1
else
echo "Cachix credentials missing; will not attempt cache writes."
"$@"
fi
fi
}
function is_cache_writeable() {
# We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo
# jobs will get this from CircleCI configuration but jobs from forks may
# not.
[ -v CACHIX_AUTH_TOKEN ]
}
function is_cache_required() {
# If we're building in tahoe-lafs/tahoe-lafs then we must use the cache.
# If we're building anything from a fork then we're allowed to not have
# the credentials.
is_upstream
}
# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs
# repository itself (and so we expect to have cache credentials available),
# failure otherwise.
#
# See circleci.txt for notes about how this determination is made.
function is_upstream() {
# CIRCLE_PROJECT_USERNAME is set to the org the build is happening for.
# If a PR targets a fork of the repo then this is set to something other
# than "tahoe-lafs".
[ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] &&
# CIRCLE_BRANCH is set to the real branch name for in-repo PRs and
# "pull/NNNN" for pull requests from forks.
#
# CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full
# URLs of the PR pages which share an underlying branch, with one of
# them ended with that same "pull/NNNN" for PRs from forks.
! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS"
}
# Return success if splitting $3 on $2 results in an array with any element
# that ends with $1, failure otherwise.
function any_element_endswith() {
suffix=$1
shift
sep=$1
shift
haystack=$1
shift
IFS="${sep}" read -r -a elements <<< "$haystack"
for elem in "${elements[@]}"; do
if endswith "$suffix" "$elem"; then
return 0
fi
done
return 1
}
# Return success if $2 ends with $1, failure otherwise.
function endswith() {
suffix=$1
shift
haystack=$1
shift
case "$haystack" in
*${suffix})
return 0
;;
*)
return 1
;;
esac
}
function describe_build() {
echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}"
echo "Building branch: ${CIRCLE_BRANCH}"
if is_upstream; then
echo "Upstream build."
else
echo "Non-upstream build."
fi
if is_cache_required; then
echo "Cache is required."
else
echo "Cache not required."
fi
if is_cache_writeable; then
echo "Cache is writeable."
else
echo "Cache not writeable."
fi
}
# Inspect the flake input metadata for an input of a given name and return the
# revision at which that input is pinned. If the input does not exist then
# return garbage (probably "null").
read_input_revision() {
input_name=$1
shift
nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev'
}
# Return a flake reference that refers to a certain revision of nixpkgs. The
# certain revision is the revision to which the specified input is pinned.
nixpkgs_flake_reference() {
input_name=$1
shift
echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)"
}

60
flake.lock generated
View File

@ -3,11 +3,11 @@
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1673956053,
"narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
"lastModified": 1733328505,
"narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
"rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec",
"type": "github"
},
"original": {
@ -21,11 +21,11 @@
"systems": "systems"
},
"locked": {
"lastModified": 1687709756,
"narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=",
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
@ -34,50 +34,18 @@
"type": "github"
}
},
"nixpkgs-22_11": {
"nixpkgs-24_11": {
"locked": {
"lastModified": 1688392541,
"narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=",
"lastModified": 1733261153,
"narHash": "sha256-eq51hyiaIwtWo19fPEeE0Zr2s83DYMKJoukNLgGGpek=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b",
"rev": "b681065d0919f7eb5309a93cea2cfa84dec9aa88",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-22.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-23_05": {
"locked": {
"lastModified": 1689885880,
"narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-unstable": {
"locked": {
"lastModified": 1689791806,
"narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "pull/244135/head",
"ref": "nixos-24.11",
"repo": "nixpkgs",
"type": "github"
}
@ -87,11 +55,9 @@
"flake-compat": "flake-compat",
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs-unstable"
"nixpkgs-24_11"
],
"nixpkgs-22_11": "nixpkgs-22_11",
"nixpkgs-23_05": "nixpkgs-23_05",
"nixpkgs-unstable": "nixpkgs-unstable"
"nixpkgs-24_11": "nixpkgs-24_11"
}
},
"systems": {

35
flake.nix
View File

@ -1,14 +1,6 @@
{
description = "Tahoe-LAFS, free and open decentralized data store";
nixConfig = {
# Supply configuration for the build cache updated by our CI system. This
# should allow most users to avoid having to build a large number of
# packages (otherwise necessary due to our Python package overrides).
substituters = ["https://tahoe-lafs-opensource.cachix.org"];
trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="];
};
inputs = {
# A couple possible nixpkgs pins. Ideally these could be selected easily
# from the command line but there seems to be no syntax/support for that.
@ -20,25 +12,12 @@
# requirements. We could decide in the future that supporting multiple
# releases of NixOS at a time is worthwhile and then pins like these will
# help us test each of those releases.
"nixpkgs-22_11" = {
url = github:NixOS/nixpkgs?ref=nixos-22.11;
};
"nixpkgs-23_05" = {
url = github:NixOS/nixpkgs?ref=nixos-23.05;
"nixpkgs-24_11" = {
url = github:NixOS/nixpkgs?ref=nixos-24.11;
};
# We depend on a very new python-cryptography which is not yet available
# from any release branch of nixpkgs. However, it is contained in a PR
# currently up for review. Point our nixpkgs at that for now.
"nixpkgs-unstable" = {
url = github:NixOS/nixpkgs?ref=pull/244135/head;
};
# Point the default nixpkgs at one of those. This avoids having getting a
# _third_ package set involved and gives a way to provide what should be a
# working experience by default (that is, if nixpkgs doesn't get
# overridden).
nixpkgs.follows = "nixpkgs-unstable";
# Point the default nixpkgs at one of those.
nixpkgs.follows = "nixpkgs-24_11";
# Also get flake-utils for simplified multi-system definitions.
flake-utils = {
@ -153,7 +132,8 @@
[ tahoe-lafs ] ++
tahoe-lafs.passthru.extras.i2p ++
tahoe-lafs.passthru.extras.tor ++
tahoe-lafs.passthru.extras.unittest
tahoe-lafs.passthru.extras.unittest ++
[ hatchling hatch-vcs ]
)).overrideAttrs (old: {
# See the similar override in makeRuntimeEnv'.
name = packageName pyVersion;
@ -219,10 +199,11 @@
program =
let
python = "${makeTestEnv pyVersion}/bin/python";
hatchling = "${makeTestEnv pyVersion}/bin/hatchling";
in
writeScript "unit-tests"
''
${python} setup.py update_version
${hatchling} build --hooks-only # Write _version.py
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
export PYTHONPATH=$PWD/src
${python} -m twisted.trial "$@"

4
newsfragments/4134.minor Normal file
View File

@ -0,0 +1,4 @@
Avoid private cache from Cachix until we can restore it.
Update nixpkgs to 24.11 wich is well cached for now.
Stop packaging and testing on nixpkgs/python39 (too old).
Start packaging and testing on nixpkgs/python312 instead.

9
nix/klein.nix
View File

@ -1,9 +0,0 @@
{ klein, fetchPypi }:
klein.overrideAttrs (old: rec {
pname = "klein";
version = "23.5.0";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-kGkSt6tBDZp/NRICg5w81zoqwHe9AHHIYcMfDu92Aoc=";
};
})

57
nix/pycddl.nix
View File

@ -1,57 +0,0 @@
# package https://gitlab.com/tahoe-lafs/pycddl
#
# also in the process of being pushed upstream
# https://github.com/NixOS/nixpkgs/pull/221220
#
# we should switch to the upstream package when it is available from our
# minimum version of nixpkgs.
#
# if you need to update this package to a new pycddl release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`.
#
# 6. run `nix-build`
#
# 7. there will be an error about a hash mismatch. change the value given to
# `cargoDeps` for `hash` to the "actual" hash value report.
#
# 8. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ lib, fetchPypi, python, buildPythonPackage, rustPlatform }:
buildPythonPackage rec {
pname = "pycddl";
version = "0.6.1";
format = "pyproject";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-63fe8UJXEH6t4l7ujV8JDvlGb7q3kL6fHHATFdklzFc=";
};
# Without this, when building for PyPy, `maturin build` seems to fail to
# find the interpreter at all and then fails early in the build process with
# an error saying "unsupported Python interpreter". We can easily point
# directly at the relevant interpreter, so do that.
maturinBuildFlags = [ "--interpreter" python.executable ];
nativeBuildInputs = with rustPlatform; [
maturinBuildHook
cargoSetupHook
];
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-ssDEKRd3Y9/10oXBZHCxvlRkl9KMh3pGYbCkM4rXThQ=";
};
}

10
nix/pyopenssl.nix
View File

@ -1,10 +0,0 @@
{ pyopenssl, fetchPypi, isPyPy }:
pyopenssl.overrideAttrs (old: rec {
pname = "pyOpenSSL";
version = "23.2.0";
name = "${pname}-${version}";
src = fetchPypi {
inherit pname version;
sha256 = "J2+TH1WkUufeppxxc+mE6ypEB85BPJGKo0tV+C+bi6w=";
};
})

32
nix/python-overrides.nix
View File

@ -42,40 +42,10 @@ in {
tahoe-lafs-src = self.lib.cleanSource ../.;
};
# Some dependencies aren't packaged in nixpkgs so supply our own packages.
pycddl = self.callPackage ./pycddl.nix { };
txi2p = self.callPackage ./txi2p.nix { };
# Some packages are of somewhat too-old versions - update them.
klein = self.callPackage ./klein.nix {
# Avoid infinite recursion.
inherit (super) klein;
};
txtorcon = self.callPackage ./txtorcon.nix {
inherit (super) txtorcon;
};
# With our customized package set a Twisted unit test fails. Patch the
# Twisted test suite to skip that test.
# Filed upstream at https://github.com/twisted/twisted/issues/11877
twisted = super.twisted.overrideAttrs (old: {
patches = (old.patches or []) ++ [ ./twisted.patch ];
});
# Update the version of pyopenssl - and since we're doing that anyway, we
# don't need the docs. Unfortunately this triggers a lot of rebuilding of
# dependent packages.
pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix {
inherit (super) pyopenssl;
});
# The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade
# that we did breaks service-identity ... so get a newer version that works.
service-identity = self.callPackage ./service-identity.nix { };
# collections-extended is currently broken for Python 3.11 in nixpkgs but
# we know where a working version lives.
collections-extended = self.callPackage ./collections-extended.nix {
# Avoid infinite recursion.
inherit (super) collections-extended;
};

61
nix/service-identity.nix
View File

@ -1,61 +0,0 @@
{ lib
, attrs
, buildPythonPackage
, cryptography
, fetchFromGitHub
, hatch-fancy-pypi-readme
, hatch-vcs
, hatchling
, idna
, pyasn1
, pyasn1-modules
, pytestCheckHook
, pythonOlder
, setuptools
}:
buildPythonPackage rec {
pname = "service-identity";
version = "23.1.0";
format = "pyproject";
disabled = pythonOlder "3.8";
src = fetchFromGitHub {
owner = "pyca";
repo = pname;
rev = "refs/tags/${version}";
hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo=";
};
nativeBuildInputs = [
hatch-fancy-pypi-readme
hatch-vcs
hatchling
setuptools
];
propagatedBuildInputs = [
attrs
cryptography
idna
pyasn1
pyasn1-modules
];
nativeCheckInputs = [
pytestCheckHook
];
pythonImportsCheck = [
"service_identity"
];
meta = with lib; {
description = "Service identity verification for pyOpenSSL";
homepage = "https://service-identity.readthedocs.io";
changelog = "https://github.com/pyca/service-identity/releases/tag/${version}";
license = licenses.mit;
maintainers = with maintainers; [ fab ];
};
}

5
nix/tahoe-lafs.nix
View File

@ -9,6 +9,7 @@ in
}:
buildPythonPackage rec {
inherit pname version;
pyproject = true;
src = tahoe-lafs-src;
propagatedBuildInputs = with pythonPackages; [
attrs
@ -22,6 +23,8 @@ buildPythonPackage rec {
filelock
foolscap
future
hatchling
hatch-vcs
klein
magic-wormhole
netifaces
@ -50,7 +53,7 @@ buildPythonPackage rec {
txtorcon
];
i2p = [
txi2p
txi2p-tahoe
];
unittest = [
beautifulsoup4

12
nix/twisted.patch
View File

@ -1,12 +0,0 @@
diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py
index c650fd8aa6..a1754fd533 100644
--- a/src/twisted/internet/test/test_endpoints.py
+++ b/src/twisted/internet/test/test_endpoints.py
@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase):
connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint)
self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com")
+ @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails")
def test_tls(self):
"""
When passed a string endpoint description beginning with C{tls:},

39
nix/txi2p.nix
View File

@ -1,39 +0,0 @@
# package https://github.com/tahoe-lafs/txi2p
#
# if you need to update this package to a new txi2p release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. if there are new runtime dependencies then add them to the argument list
# at the top. if there are new test dependencies add them to the
# `checkInputs` list.
#
# 6. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ fetchPypi
, buildPythonPackage
, parsley
, twisted
, unittestCheckHook
}:
buildPythonPackage rec {
pname = "txi2p-tahoe";
version = "0.3.7";
src = fetchPypi {
inherit pname version;
hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY=";
};
propagatedBuildInputs = [ twisted parsley ];
checkInputs = [ unittestCheckHook ];
pythonImportsCheck = [ "parsley" "ometa"];
}

9
nix/txtorcon.nix
View File

@ -1,9 +0,0 @@
{ txtorcon, fetchPypi }:
txtorcon.overrideAttrs (old: rec {
pname = "txtorcon";
version = "23.5.0";
src = fetchPypi {
inherit pname version;
hash = "sha256-k/2Aqd1QX2mNCGT+k9uLapwRRLX+uRUwggtw7YmCZRw=";
};
})