ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-08 11:24:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1267 from exarkun/replace-mach-nix

Browse Source 
Fixes: ticket:3987
This commit is contained in:
Jean-Paul Calderone 2023-03-15 18:08:48 -04:00 committed by GitHub
commit e033f25d0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 225 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
.circleci/config.yml
View File

@ -71,12 +71,12 @@ workflows:
{}
- "nixos":
name: "NixOS 21.05"
nixpkgs: "21.05"
name: "NixOS 22.11"
nixpkgs: "22.11"
- "nixos":
name: "NixOS 21.11"
nixpkgs: "21.11"
name: "NixOS unstable"
nixpkgs: "unstable"
# Eventually, test against PyPy 3.8
#- "pypy27-buster":
@ -412,33 +412,18 @@ jobs:
--run 'python setup.py update_version'
- "run":
name: "Build"
name: "Test"
command: |
# CircleCI build environment looks like it has a zillion and a
# half cores. Don't let Nix autodetect this high core count
# because it blows up memory usage and fails the test run. Pick a
# number of cores that suites the build environment we're paying
# for (the free one!).
#
# Also, let it run more than one job at a time because we have to
# build a couple simple little dependencies that don't take
# advantage of multiple cores and we get a little speedup by doing
# them in parallel.
source .circleci/lib.sh
cache_if_able nix-build \
--cores 3 \
--max-jobs 2 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
- "run":
name: "Test"
command: |
# Let it go somewhat wild for the test suite itself
source .circleci/lib.sh
cache_if_able nix-build \
--cores 8 \
--argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
tests.nix
nix/tests.nix
typechecks:
docker:

103
default.nix
View File

@ -1,8 +1,7 @@
let
# sources.nix contains information about which versions of some of our
# dependencies we should use. since we use it to pin nixpkgs and the PyPI
# package database, roughly all the rest of our dependencies are *also*
# pinned - indirectly.
# dependencies we should use. since we use it to pin nixpkgs, all the rest
# of our dependencies are *also* pinned - indirectly.
#
# sources.nix is managed using a tool called `niv`. as an example, to
# update to the most recent version of nixpkgs from the 21.11 maintenance
@ -10,93 +9,45 @@ let
#
# niv update nixpkgs-21.11
#
# or, to update the PyPI package database -- which is necessary to make any
# newly released packages visible -- you likewise run:
#
# niv update pypi-deps-db
#
# niv also supports chosing a specific revision, following a different
# branch, etc. find complete documentation for the tool at
# https://github.com/nmattia/niv
sources = import nix/sources.nix;
in
{
pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
# niv-managed sources data
, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
# for dependency resolution
, pythonVersion ? "python310" # a string choosing the python derivation from
# nixpkgs to target
, pythonVersion ? "python39" # a string choosing the python derivation from
# nixpkgs to target
, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
# the dependencies of which the resulting
# package will also depend on. Include all of the
# runtime extras by default because the incremental
# cost of including them is a lot smaller than the
# cost of re-building the whole thing to add them.
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
# the dependencies of which the resulting package
# will also depend on. Include all of the runtime
# extras by default because the incremental cost of
# including them is a lot smaller than the cost of
# re-building the whole thing to add them.
, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
# the tahoe-lafs package
inherit pkgs pypiData;
python = pythonVersion;
}
}:
# The project name, version, and most other metadata are automatically
# extracted from the source. Some requirements are not properly extracted
# and those cases are handled below. The version can only be extracted if
# `setup.py update_version` has been run (this is not at all ideal but it
# seems difficult to fix) - so for now just be sure to run that first.
mach-nix.buildPythonPackage rec {
# Define the location of the Tahoe-LAFS source to be packaged. Clean up all
# as many of the non-source files (eg the `.git` directory, `~` backup
# files, nix's own `result` symlink, etc) as possible to avoid needing to
# re-build when files that make no difference to the package have changed.
src = pkgs.lib.cleanSource ./.;
with (pkgs.${pythonVersion}.override {
packageOverrides = self: super: {
# Some dependencies aren't packaged in nixpkgs so supply our own packages.
pycddl = self.callPackage ./nix/pycddl.nix { };
txi2p = self.callPackage ./nix/txi2p.nix { };
};
}).pkgs;
callPackage ./nix/tahoe-lafs.nix {
# Select whichever package extras were requested.
inherit extras;
inherit extrasNames;
# Define some extra requirements that mach-nix does not automatically detect
# from inspection of the source. We typically don't need to put version
# constraints on any of these requirements. The pypi-deps-db we're
# operating with makes dependency resolution deterministic so as long as it
# works once it will always work. It could be that in the future we update
# pypi-deps-db and an incompatibility arises - in which case it would make
# sense to apply some version constraints here.
requirementsExtra = ''
# mach-nix does not yet support pyproject.toml which means it misses any
# build-time requirements of our dependencies which are declared in such a
# file. Tell it about them here.
setuptools_rust
# Define the location of the Tahoe-LAFS source to be packaged (the same
# directory as contains this file). Clean up as many of the non-source
# files (eg the `.git` directory, `~` backup files, nix's own `result`
# symlink, etc) as possible to avoid needing to re-build when files that
# make no difference to the package have changed.
tahoe-lafs-src = pkgs.lib.cleanSource ./.;
# mach-nix does not yet parse environment markers (e.g. "python > '3.0'")
# correctly. It misses all of our requirements which have an environment marker.
# Duplicate them here.
foolscap
eliot
pyrsistent
collections-extended
'';
# Specify where mach-nix should find packages for our Python dependencies.
# There are some reasonable defaults so we only need to specify certain
# packages where the default configuration runs into some issue.
providers = {
};
# Define certain overrides to the way Python dependencies are built.
_ = {
# Remove a click-default-group patch for a test suite problem which no
# longer applies because the project apparently no longer has a test suite
# in its source distribution.
click-default-group.patches = [];
};
passthru.meta.mach-nix = {
inherit providers _;
};
doCheck = false;
}

0
newsfragments/3987.minor Normal file
View File

51
nix/pycddl.nix Normal file
View File

@ -0,0 +1,51 @@
# package https://gitlab.com/tahoe-lafs/pycddl
#
# also in the process of being pushed upstream
# https://github.com/NixOS/nixpkgs/pull/221220
#
# we should switch to the upstream package when it is available from our
# minimum version of nixpkgs.
#
# if you need to update this package to a new pycddl release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. change the value given to `cargoDeps` for `hash` to lib.fakeHash`.
#
# 6. run `nix-build`
#
# 7. there will be an error about a hash mismatch. change the value given to
# `cargoDeps` for `hash` to the "actual" hash value report.
#
# 8. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
buildPythonPackage rec {
pname = "pycddl";
version = "0.4.0";
format = "pyproject";
src = fetchPypi {
inherit pname version;
sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
};
nativeBuildInputs = with rustPlatform; [
maturinBuildHook
cargoSetupHook
];
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
name = "${pname}-${version}";
hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs=";
};
}

50
nix/sources.json
View File

@ -1,16 +1,4 @@
{
"mach-nix": {
"branch": "switch-to-nix-pypi-fetcher-2",
"description": "Create highly reproducible python environments",
"homepage": "",
"owner": "PrivateStorageio",
"repo": "mach-nix",
"rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4",
"sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls",
"type": "tarball",
"url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"niv": {
"branch": "master",
"description": "Easy dependency management for Nix projects",
@ -23,40 +11,28 @@
"url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-21.05": {
"branch": "nixos-21.05",
"nixpkgs-22.11": {
"branch": "nixos-22.11",
"description": "Nix Packages collection",
"homepage": "",
"owner": "NixOS",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
"sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
"rev": "970402e6147c49603f4d06defe44d27fe51884ce",
"sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
"url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-21.11": {
"branch": "nixos-21.11",
"description": "Nix Packages collection",
"homepage": "",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "838eefb4f93f2306d4614aafb9b2375f315d917f",
"sha256": "1bm8cmh1wx4h8b4fhbs75hjci3gcrpi7k1m1pmiy3nc0gjim9vkg",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"pypi-deps-db": {
"nixpkgs-unstable": {
"branch": "master",
"description": "Probably the most complete python dependency database",
"description": "Nix Packages collection",
"homepage": "",
"owner": "DavHau",
"repo": "pypi-deps-db",
"rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6",
"sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "d0c9a536331227ab883b4f6964be638fa436d81f",
"sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1",
"type": "tarball",
"url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz",
"url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}

84
nix/tahoe-lafs.nix Normal file
View File

@ -0,0 +1,84 @@
{ lib
, pythonPackages
, buildPythonPackage
, tahoe-lafs-src
, extrasNames
# control how the test suite is run
, doCheck
}:
let
pname = "tahoe-lafs";
version = "1.18.0.post1";
pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras;
pythonExtraDependencies = with pythonPackages; {
tor = [ txtorcon ];
i2p = [ txi2p ];
};
pythonPackageDependencies = with pythonPackages; [
attrs
autobahn
cbor2
click
collections-extended
cryptography
distro
eliot
filelock
foolscap
future
klein
magic-wormhole
netifaces
psutil
pycddl
pyrsistent
pyutil
six
treq
twisted
# Get the dependencies for the Twisted extras we depend on, too.
twisted.passthru.optional-dependencies.tls
twisted.passthru.optional-dependencies.conch
werkzeug
zfec
zope_interface
] ++ pickExtraDependencies pythonExtraDependencies extrasNames;
pythonCheckDependencies = with pythonPackages; [
beautifulsoup4
fixtures
hypothesis
mock
paramiko
prometheus-client
pytest
pytest-timeout
pytest-twisted
tenacity
testtools
towncrier
];
in
buildPythonPackage {
inherit pname version;
src = tahoe-lafs-src;
propagatedBuildInputs = pythonPackageDependencies;
inherit doCheck;
checkInputs = pythonCheckDependencies;
checkPhase = ''
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
python -m twisted.trial -j $NIX_BUILD_CORES allmydata
'';
meta = with lib; {
homepage = "https://tahoe-lafs.org/";
description = "secure, decentralized, fault-tolerant file store";
# Also TGPPL
license = licenses.gpl2Plus;
};
}

4
nix/tests.nix Normal file
View File

@ -0,0 +1,4 @@
# Build the package with the test suite enabled.
args@{...}: (import ../. args).override {
doCheck = true;
}

39
nix/txi2p.nix Normal file
View File

@ -0,0 +1,39 @@
# package https://github.com/tahoe-lafs/txi2p
#
# if you need to update this package to a new txi2p release then
#
# 1. change value given to `buildPythonPackage` for `version` to match the new
# release
#
# 2. change the value given to `fetchPypi` for `sha256` to `lib.fakeHash`
#
# 3. run `nix-build`
#
# 4. there will be an error about a hash mismatch. change the value given to
# `fetchPypi` for `sha256` to the "actual" hash value report.
#
# 5. if there are new runtime dependencies then add them to the argument list
# at the top. if there are new test dependencies add them to the
# `checkInputs` list.
#
# 6. run `nix-build`. it should succeed. if it does not, seek assistance.
#
{ fetchPypi
, buildPythonPackage
, parsley
, twisted
, unittestCheckHook
}:
buildPythonPackage rec {
pname = "txi2p-tahoe";
version = "0.3.7";
src = fetchPypi {
inherit pname version;
hash = "sha256-+Vs9zaFS+ACI14JNxEme93lnWmncdZyFAmnTH0yhOiY=";
};
propagatedBuildInputs = [ twisted parsley ];
checkInputs = [ unittestCheckHook ];
pythonImportsCheck = [ "parsley" "ometa"];
}

2
setup.py
View File

@ -118,7 +118,7 @@ install_requires = [
"attrs >= 18.2.0",
# WebSocket library for twisted and asyncio
"autobahn < 22.4.1", # remove this when 22.4.3 is released
"autobahn",
# Support for Python 3 transition
"future >= 0.18.2",

88
tests.nix
View File

@ -1,88 +0,0 @@
let
sources = import nix/sources.nix;
in
# See default.nix for documentation about parameters.
{ pkgsVersion ? "nixpkgs-21.11"
, pkgs ? import sources.${pkgsVersion} { }
, pypiData ? sources.pypi-deps-db
, pythonVersion ? "python39"
, mach-nix ? import sources.mach-nix {
inherit pkgs pypiData;
python = pythonVersion;
}
}@args:
let
# We would like to know the test requirements but mach-nix does not directly
# expose this information to us. However, it is perfectly capable of
# determining it if we ask right... This is probably not meant to be a
# public mach-nix API but we pinned mach-nix so we can deal with mach-nix
# upgrade breakage in our own time.
mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" {
inherit pkgs;
lib = pkgs.lib;
};
tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test;
# Get the Tahoe-LAFS package itself. This does not include test
# requirements and we don't ask for test requirements so that we can just
# re-use the normal package if it is already built.
tahoe-lafs = import ./. args;
# If we want to get tahoe-lafs into a Python environment with a bunch of
# *other* Python modules and let them interact in the usual way then we have
# to ask mach-nix for tahoe-lafs and those other Python modules in the same
# way - i.e., using `requirements`. The other tempting mechanism,
# `packagesExtra`, inserts an extra layer of Python environment and prevents
# normal interaction between Python modules (as well as usually producing
# file collisions in the packages that are both runtime and test
# dependencies). To get the tahoe-lafs we just built into the environment,
# put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs
# from nixpkgs.
overridesPre = [(self: super: { inherit tahoe-lafs; })];
providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; };
# Make the Python environment in which we can run the tests.
python-env = mach-nix.mkPython {
# Get the packaging fixes we already know we need from putting together
# the runtime package.
inherit (tahoe-lafs.meta.mach-nix) _;
# Share the runtime package's provider configuration - combined with our
# own that causes the right tahoe-lafs to be picked up.
inherit providers overridesPre;
requirements = ''
# Here we pull in the Tahoe-LAFS package itself.
tahoe-lafs
# Unfortunately mach-nix misses all of the Python dependencies of the
# tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a
# bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will
# fix this soon.
#
# https://github.com/DavHau/mach-nix/issues/123
# https://github.com/DavHau/mach-nix/pull/386
${tahoe-lafs.requirements}
# And then all of the test-only dependencies.
${builtins.concatStringsSep "\n" tests_require}
# txi2p-tahoe is another dependency with an environment marker that
# mach-nix doesn't automatically pick up.
txi2p-tahoe
'';
};
in
# Make a derivation that runs the unit test suite.
pkgs.runCommand "tahoe-lafs-tests" { } ''
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
# It's not cool to put the whole _trial_temp into $out because it has weird
# files in it we don't want in the store. Plus, even all of the less weird
# files are mostly just trash that's not meaningful if the test suite passes
# (which is the only way we get $out anyway).
#
# The build log itself is typically available from `nix-store --read-log` so
# we don't need to record that either.
echo "passed" >$out
''