mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-01-18 02:40:07 +00:00
Replace the mach-nix-based package with a nixpkgs-based package
The built-in nixpkgs `buildPythonPackage` doesn't do metadata discovery so we have to duplicate a lot of the package metadata. However, mach-nix is unmaintained and incompatible with newer versions of nixpkgs.
This commit is contained in:
Jean-Paul Calderone
parent
2bb96d8452
commit
568e1b5317
@ -70,14 +70,18 @@ workflows:
|
||||
- "oraclelinux-8":
|
||||
{}
|
||||
|
||||
- "nixos":
|
||||
name: "NixOS 21.05"
|
||||
nixpkgs: "21.05"
|
||||
|
||||
- "nixos":
|
||||
name: "NixOS 21.11"
|
||||
nixpkgs: "21.11"
|
||||
|
||||
- "nixos":
|
||||
name: "NixOS 22.11"
|
||||
nixpkgs: "22.11"
|
||||
|
||||
- "nixos":
|
||||
name: "NixOS unstable"
|
||||
nixpkgs: "unstable"
|
||||
|
||||
# Eventually, test against PyPy 3.8
|
||||
#- "pypy27-buster":
|
||||
# {}
|
||||
|
||||
73
default.nix
73
default.nix
@ -21,16 +21,13 @@ let
|
||||
sources = import nix/sources.nix;
|
||||
in
|
||||
{
|
||||
pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
|
||||
pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the
|
||||
# niv-managed sources data
|
||||
|
||||
, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
|
||||
|
||||
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
|
||||
# for dependency resolution
|
||||
|
||||
, pythonVersion ? "python39" # a string choosing the python derivation from
|
||||
# nixpkgs to target
|
||||
, pythonVersion ? "python310" # a string choosing the python derivation from
|
||||
# nixpkgs to target
|
||||
|
||||
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
|
||||
# the dependencies of which the resulting package
|
||||
@ -39,64 +36,18 @@ in
|
||||
# including them is a lot smaller than the cost of
|
||||
# re-building the whole thing to add them.
|
||||
|
||||
, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
|
||||
# the tahoe-lafs package
|
||||
inherit pkgs pypiData;
|
||||
python = pythonVersion;
|
||||
}
|
||||
}:
|
||||
# The project name, version, and most other metadata are automatically
|
||||
# extracted from the source. Some requirements are not properly extracted
|
||||
# and those cases are handled below. The version can only be extracted if
|
||||
# `setup.py update_version` has been run (this is not at all ideal but it
|
||||
# seems difficult to fix) - so for now just be sure to run that first.
|
||||
mach-nix.buildPythonPackage rec {
|
||||
# Define the location of the Tahoe-LAFS source to be packaged. Clean up all
|
||||
# as many of the non-source files (eg the `.git` directory, `~` backup
|
||||
# files, nix's own `result` symlink, etc) as possible to avoid needing to
|
||||
# re-build when files that make no difference to the package have changed.
|
||||
src = pkgs.lib.cleanSource ./.;
|
||||
|
||||
with pkgs.${pythonVersion}.pkgs;
|
||||
callPackage ./nix/tahoe-lafs.nix {
|
||||
# Select whichever package extras were requested.
|
||||
inherit extras;
|
||||
|
||||
# Define some extra requirements that mach-nix does not automatically detect
|
||||
# from inspection of the source. We typically don't need to put version
|
||||
# constraints on any of these requirements. The pypi-deps-db we're
|
||||
# operating with makes dependency resolution deterministic so as long as it
|
||||
# works once it will always work. It could be that in the future we update
|
||||
# pypi-deps-db and an incompatibility arises - in which case it would make
|
||||
# sense to apply some version constraints here.
|
||||
requirementsExtra = ''
|
||||
# mach-nix does not yet support pyproject.toml which means it misses any
|
||||
# build-time requirements of our dependencies which are declared in such a
|
||||
# file. Tell it about them here.
|
||||
setuptools_rust
|
||||
# Define the location of the Tahoe-LAFS source to be packaged. Clean up as
|
||||
# many of the non-source files (eg the `.git` directory, `~` backup files,
|
||||
# nix's own `result` symlink, etc) as possible to avoid needing to re-build
|
||||
# when files that make no difference to the package have changed.
|
||||
tahoe-lafs-src = pkgs.lib.cleanSource ./.;
|
||||
|
||||
# mach-nix does not yet parse environment markers (e.g. "python > '3.0'")
|
||||
# correctly. It misses all of our requirements which have an environment marker.
|
||||
# Duplicate them here.
|
||||
foolscap
|
||||
eliot
|
||||
pyrsistent
|
||||
collections-extended
|
||||
'';
|
||||
|
||||
# Specify where mach-nix should find packages for our Python dependencies.
|
||||
# There are some reasonable defaults so we only need to specify certain
|
||||
# packages where the default configuration runs into some issue.
|
||||
providers = {
|
||||
};
|
||||
|
||||
# Define certain overrides to the way Python dependencies are built.
|
||||
_ = {
|
||||
# Remove a click-default-group patch for a test suite problem which no
|
||||
# longer applies because the project apparently no longer has a test suite
|
||||
# in its source distribution.
|
||||
click-default-group.patches = [];
|
||||
};
|
||||
|
||||
passthru.meta.mach-nix = {
|
||||
inherit providers _;
|
||||
};
|
||||
# pycddl isn't packaged in nixpkgs so supply our own package of it.
|
||||
pycddl = callPackage ./nix/pycddl.nix { };
|
||||
}
|
||||
|
||||
22
nix/pycddl.nix
Normal file
22
nix/pycddl.nix
Normal file
@ -0,0 +1,22 @@
|
||||
{ lib, fetchPypi, buildPythonPackage, rustPlatform }:
|
||||
buildPythonPackage rec {
|
||||
pname = "pycddl";
|
||||
version = "0.4.0";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "sha256-w0CGbPeiXyS74HqZXyiXhvaAMUaIj5onwjl9gWKAjqY=";
|
||||
};
|
||||
|
||||
nativeBuildInputs = with rustPlatform; [
|
||||
maturinBuildHook
|
||||
cargoSetupHook
|
||||
];
|
||||
|
||||
cargoDeps = rustPlatform.fetchCargoTarball {
|
||||
inherit src;
|
||||
name = "${pname}-${version}";
|
||||
hash = "sha256-g96eeaqN9taPED4u+UKUcoitf5aTGFrW2/TOHoHEVHs=";
|
||||
};
|
||||
}
|
||||
@ -1,16 +1,4 @@
|
||||
{
|
||||
"mach-nix": {
|
||||
"branch": "switch-to-nix-pypi-fetcher-2",
|
||||
"description": "Create highly reproducible python environments",
|
||||
"homepage": "",
|
||||
"owner": "PrivateStorageio",
|
||||
"repo": "mach-nix",
|
||||
"rev": "f6d1a1841d8778c199326f95d0703c16bee2f8c4",
|
||||
"sha256": "0krc4yhnpbzc4yhja9frnmym2vqm5zyacjnqb3fq9z9gav8vs9ls",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/PrivateStorageio/mach-nix/archive/f6d1a1841d8778c199326f95d0703c16bee2f8c4.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"niv": {
|
||||
"branch": "master",
|
||||
"description": "Easy dependency management for Nix projects",
|
||||
@ -23,18 +11,6 @@
|
||||
"url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs-21.05": {
|
||||
"branch": "nixos-21.05",
|
||||
"description": "Nix Packages collection",
|
||||
"homepage": "",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
|
||||
"sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs-21.11": {
|
||||
"branch": "nixos-21.11",
|
||||
"description": "Nix Packages collection",
|
||||
@ -47,16 +23,28 @@
|
||||
"url": "https://github.com/NixOS/nixpkgs/archive/838eefb4f93f2306d4614aafb9b2375f315d917f.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"pypi-deps-db": {
|
||||
"branch": "master",
|
||||
"description": "Probably the most complete python dependency database",
|
||||
"nixpkgs-22.11": {
|
||||
"branch": "nixos-22.11",
|
||||
"description": "Nix Packages collection",
|
||||
"homepage": "",
|
||||
"owner": "DavHau",
|
||||
"repo": "pypi-deps-db",
|
||||
"rev": "5440c9c76f6431f300fb6a1ecae762a5444de5f6",
|
||||
"sha256": "08r3iiaxzw9v2gq15y1m9bwajshyyz9280g6aia7mkgnjs9hnd1n",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "970402e6147c49603f4d06defe44d27fe51884ce",
|
||||
"sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/DavHau/pypi-deps-db/archive/5440c9c76f6431f300fb6a1ecae762a5444de5f6.tar.gz",
|
||||
"url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs-unstable": {
|
||||
"branch": "master",
|
||||
"description": "Nix Packages collection",
|
||||
"homepage": "",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "d0c9a536331227ab883b4f6964be638fa436d81f",
|
||||
"sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
}
|
||||
}
|
||||
|
||||
112
nix/tahoe-lafs.nix
Normal file
112
nix/tahoe-lafs.nix
Normal file
@ -0,0 +1,112 @@
|
||||
{ buildPythonPackage
|
||||
, tahoe-lafs-src
|
||||
, extras
|
||||
|
||||
# always dependencies
|
||||
, attrs
|
||||
, autobahn
|
||||
, cbor2
|
||||
, click
|
||||
, collections-extended
|
||||
, cryptography
|
||||
, distro
|
||||
, eliot
|
||||
, filelock
|
||||
, foolscap
|
||||
, future
|
||||
, klein
|
||||
, magic-wormhole
|
||||
, netifaces
|
||||
, psutil
|
||||
, pycddl
|
||||
, pyrsistent
|
||||
, pyutil
|
||||
, six
|
||||
, treq
|
||||
, twisted
|
||||
, werkzeug
|
||||
, zfec
|
||||
, zope_interface
|
||||
|
||||
# tor extra dependencies
|
||||
, txtorcon
|
||||
|
||||
# i2p extra dependencies
|
||||
, txi2p-tahoe
|
||||
|
||||
# test dependencies
|
||||
, beautifulsoup4
|
||||
, fixtures
|
||||
, hypothesis
|
||||
, mock
|
||||
, paramiko
|
||||
, prometheus-client
|
||||
, pytest
|
||||
, pytest-timeout
|
||||
, pytest-twisted
|
||||
, tenacity
|
||||
, testtools
|
||||
, towncrier
|
||||
}:
|
||||
let
|
||||
pname = "tahoe-lafs";
|
||||
version = "1.18.0.post1";
|
||||
|
||||
pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras;
|
||||
|
||||
pythonExtraDependencies = {
|
||||
tor = [ txtorcon ];
|
||||
i2p = [ txi2p-tahoe ];
|
||||
};
|
||||
|
||||
pythonPackageDependencies = [
|
||||
attrs
|
||||
autobahn
|
||||
cbor2
|
||||
click
|
||||
collections-extended
|
||||
cryptography
|
||||
distro
|
||||
eliot
|
||||
filelock
|
||||
foolscap
|
||||
future
|
||||
klein
|
||||
magic-wormhole
|
||||
netifaces
|
||||
psutil
|
||||
pycddl
|
||||
pyrsistent
|
||||
pyutil
|
||||
six
|
||||
treq
|
||||
twisted
|
||||
(twisted.passthru.optional-dependencies.tls)
|
||||
(twisted.passthru.optional-dependencies.conch)
|
||||
werkzeug
|
||||
zfec
|
||||
zope_interface
|
||||
] ++ pickExtraDependencies pythonExtraDependencies extras;
|
||||
|
||||
pythonCheckDependencies = [
|
||||
beautifulsoup4
|
||||
fixtures
|
||||
hypothesis
|
||||
mock
|
||||
paramiko
|
||||
prometheus-client
|
||||
pytest
|
||||
pytest-timeout
|
||||
pytest-twisted
|
||||
tenacity
|
||||
testtools
|
||||
towncrier
|
||||
];
|
||||
in
|
||||
buildPythonPackage {
|
||||
inherit pname version;
|
||||
src = tahoe-lafs-src;
|
||||
buildInputs = pythonPackageDependencies;
|
||||
checkInputs = pythonCheckDependencies;
|
||||
checkPhase = "TAHOE_LAFS_HYPOTHESIS_PROFILE=ci python -m twisted.trial -j $NIX_BUILD_CORES allmydata";
|
||||
}
|
||||
88
tests.nix
88
tests.nix
@ -1,88 +0,0 @@
|
||||
let
|
||||
sources = import nix/sources.nix;
|
||||
in
|
||||
# See default.nix for documentation about parameters.
|
||||
{ pkgsVersion ? "nixpkgs-21.11"
|
||||
, pkgs ? import sources.${pkgsVersion} { }
|
||||
, pypiData ? sources.pypi-deps-db
|
||||
, pythonVersion ? "python39"
|
||||
, mach-nix ? import sources.mach-nix {
|
||||
inherit pkgs pypiData;
|
||||
python = pythonVersion;
|
||||
}
|
||||
}@args:
|
||||
let
|
||||
# We would like to know the test requirements but mach-nix does not directly
|
||||
# expose this information to us. However, it is perfectly capable of
|
||||
# determining it if we ask right... This is probably not meant to be a
|
||||
# public mach-nix API but we pinned mach-nix so we can deal with mach-nix
|
||||
# upgrade breakage in our own time.
|
||||
mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" {
|
||||
inherit pkgs;
|
||||
lib = pkgs.lib;
|
||||
};
|
||||
tests_require = (mach-lib.extract "python39" ./. "extras_require" ).extras_require.test;
|
||||
|
||||
# Get the Tahoe-LAFS package itself. This does not include test
|
||||
# requirements and we don't ask for test requirements so that we can just
|
||||
# re-use the normal package if it is already built.
|
||||
tahoe-lafs = import ./. args;
|
||||
|
||||
# If we want to get tahoe-lafs into a Python environment with a bunch of
|
||||
# *other* Python modules and let them interact in the usual way then we have
|
||||
# to ask mach-nix for tahoe-lafs and those other Python modules in the same
|
||||
# way - i.e., using `requirements`. The other tempting mechanism,
|
||||
# `packagesExtra`, inserts an extra layer of Python environment and prevents
|
||||
# normal interaction between Python modules (as well as usually producing
|
||||
# file collisions in the packages that are both runtime and test
|
||||
# dependencies). To get the tahoe-lafs we just built into the environment,
|
||||
# put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs
|
||||
# from nixpkgs.
|
||||
overridesPre = [(self: super: { inherit tahoe-lafs; })];
|
||||
providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; };
|
||||
|
||||
# Make the Python environment in which we can run the tests.
|
||||
python-env = mach-nix.mkPython {
|
||||
# Get the packaging fixes we already know we need from putting together
|
||||
# the runtime package.
|
||||
inherit (tahoe-lafs.meta.mach-nix) _;
|
||||
# Share the runtime package's provider configuration - combined with our
|
||||
# own that causes the right tahoe-lafs to be picked up.
|
||||
inherit providers overridesPre;
|
||||
requirements = ''
|
||||
# Here we pull in the Tahoe-LAFS package itself.
|
||||
tahoe-lafs
|
||||
|
||||
# Unfortunately mach-nix misses all of the Python dependencies of the
|
||||
# tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a
|
||||
# bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will
|
||||
# fix this soon.
|
||||
#
|
||||
# https://github.com/DavHau/mach-nix/issues/123
|
||||
# https://github.com/DavHau/mach-nix/pull/386
|
||||
${tahoe-lafs.requirements}
|
||||
|
||||
# And then all of the test-only dependencies.
|
||||
${builtins.concatStringsSep "\n" tests_require}
|
||||
|
||||
# txi2p-tahoe is another dependency with an environment marker that
|
||||
# mach-nix doesn't automatically pick up.
|
||||
txi2p-tahoe
|
||||
'';
|
||||
};
|
||||
in
|
||||
# Make a derivation that runs the unit test suite.
|
||||
pkgs.runCommand "tahoe-lafs-tests" { } ''
|
||||
export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci
|
||||
${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
|
||||
|
||||
# It's not cool to put the whole _trial_temp into $out because it has weird
|
||||
# files in it we don't want in the store. Plus, even all of the less weird
|
||||
# files are mostly just trash that's not meaningful if the test suite passes
|
||||
# (which is the only way we get $out anyway).
|
||||
#
|
||||
# The build log itself is typically available from `nix-store --read-log` so
|
||||
# we don't need to record that either.
|
||||
echo "passed" >$out
|
||||
|
||||
''
|
||||
Loading…
Reference in New Issue
Block a user