mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-01-22 04:18:23 +00:00
Merge pull request #1175 from tahoe-lafs/3867.mach-nix
Switch Nix packaging to use mach-nix Fixes: ticket:3867
This commit is contained in:
Jean-Paul Calderone
GitHub
commit
210a0e773b
@ -39,11 +39,13 @@ workflows:
|
||||
- "centos-8":
|
||||
{}
|
||||
|
||||
- "nixos-19-09":
|
||||
{}
|
||||
- "nixos":
|
||||
name: "NixOS 21.05"
|
||||
nixpkgs: "21.05"
|
||||
|
||||
- "nixos-21-05":
|
||||
{}
|
||||
- "nixos":
|
||||
name: "NixOS 21.11"
|
||||
nixpkgs: "21.11"
|
||||
|
||||
# Test against PyPy 2.7
|
||||
- "pypy27-buster":
|
||||
@ -441,20 +443,58 @@ jobs:
|
||||
image: "tahoelafsci/fedora:29-py"
|
||||
user: "nobody"
|
||||
|
||||
nixos-19-09: &NIXOS
|
||||
nixos:
|
||||
parameters:
|
||||
nixpkgs:
|
||||
description: >-
|
||||
Reference the name of a niv-managed nixpkgs source (see `niv show`
|
||||
and nix/sources.json)
|
||||
type: "string"
|
||||
|
||||
docker:
|
||||
# Run in a highly Nix-capable environment.
|
||||
- <<: *DOCKERHUB_AUTH
|
||||
image: "nixorg/nix:circleci"
|
||||
image: "nixos/nix:2.3.16"
|
||||
|
||||
environment:
|
||||
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz"
|
||||
SOURCE: "nix/"
|
||||
# CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
|
||||
# allows us to push to CACHIX_NAME. We only need this set for
|
||||
# `cachix use` in this step.
|
||||
CACHIX_NAME: "tahoe-lafs-opensource"
|
||||
|
||||
steps:
|
||||
- "checkout"
|
||||
- "run":
|
||||
name: "Build and Test"
|
||||
# The nixos/nix image does not include ssh. Install it so the
|
||||
# `checkout` step will succeed. We also want cachix for
|
||||
# Nix-friendly caching.
|
||||
name: "Install Basic Dependencies"
|
||||
command: |
|
||||
nix-env \
|
||||
--file https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz \
|
||||
--install \
|
||||
-A openssh cachix bash
|
||||
|
||||
- "checkout"
|
||||
|
||||
- run:
|
||||
name: "Cachix setup"
|
||||
# Record the store paths that exist before we did much. There's no
|
||||
# reason to cache these, they're either in the image or have to be
|
||||
# retrieved before we can use cachix to restore from cache.
|
||||
command: |
|
||||
cachix use "${CACHIX_NAME}"
|
||||
nix path-info --all > /tmp/store-path-pre-build
|
||||
|
||||
- "run":
|
||||
# The Nix package doesn't know how to do this part, unfortunately.
|
||||
name: "Generate version"
|
||||
command: |
|
||||
nix-shell \
|
||||
-p 'python3.withPackages (ps: [ ps.setuptools ])' \
|
||||
--run 'python setup.py update_version'
|
||||
|
||||
- "run":
|
||||
name: "Build"
|
||||
command: |
|
||||
# CircleCI build environment looks like it has a zillion and a
|
||||
# half cores. Don't let Nix autodetect this high core count
|
||||
@ -466,17 +506,50 @@ jobs:
|
||||
# build a couple simple little dependencies that don't take
|
||||
# advantage of multiple cores and we get a little speedup by doing
|
||||
# them in parallel.
|
||||
nix-build --cores 3 --max-jobs 2 "$SOURCE"
|
||||
nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>"
|
||||
|
||||
nixos-21-05:
|
||||
<<: *NIXOS
|
||||
- "run":
|
||||
name: "Test"
|
||||
command: |
|
||||
# Let it go somewhat wild for the test suite itself
|
||||
nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" tests.nix
|
||||
|
||||
environment:
|
||||
# Note this doesn't look more similar to the 19.09 NIX_PATH URL because
|
||||
# there was some internal shuffling by the NixOS project about how they
|
||||
# publish stable revisions.
|
||||
NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz"
|
||||
SOURCE: "nix/py3.nix"
|
||||
- run:
|
||||
# Send any new store objects to cachix.
|
||||
name: "Push to Cachix"
|
||||
when: "always"
|
||||
command: |
|
||||
# Cribbed from
|
||||
# https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/
|
||||
if [ -n "$CIRCLE_PR_NUMBER" ]; then
|
||||
# I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the
|
||||
# number of the PR being built". Sorry, dear reader, you have
|
||||
# guessed poorly. It is also conditionally set based on whether
|
||||
# this is a PR from a fork or not.
|
||||
#
|
||||
# https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables
|
||||
echo "Skipping Cachix push for forked PR."
|
||||
else
|
||||
# If this *isn't* a build from a fork then we have the Cachix
|
||||
# write key in our environment and we can push any new objects
|
||||
# to Cachix.
|
||||
#
|
||||
# To decide what to push, we inspect the list of store objects
|
||||
# that existed before and after we did most of our work. Any
|
||||
# that are new after the work is probably a useful thing to have
|
||||
# around so push it to the cache. We exclude all derivation
|
||||
# objects (.drv files) because they're cheap to reconstruct and
|
||||
# by the time you know their cache key you've already done all
|
||||
# the work anyway.
|
||||
#
|
||||
# This shell expression for finding the objects and pushing them
|
||||
# was from the Cachix docs:
|
||||
#
|
||||
# https://docs.cachix.org/continuous-integration-setup/circleci.html
|
||||
#
|
||||
# but they seem to have removed it now.
|
||||
bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME"
|
||||
fi
|
||||
|
||||
typechecks:
|
||||
docker:
|
||||
|
||||
115
default.nix
Normal file
115
default.nix
Normal file
@ -0,0 +1,115 @@
|
||||
let
|
||||
# sources.nix contains information about which versions of some of our
|
||||
# dependencies we should use. since we use it to pin nixpkgs and the PyPI
|
||||
# package database, roughly all the rest of our dependencies are *also*
|
||||
# pinned - indirectly.
|
||||
#
|
||||
# sources.nix is managed using a tool called `niv`. as an example, to
|
||||
# update to the most recent version of nixpkgs from the 21.11 maintenance
|
||||
# release, in the top-level tahoe-lafs checkout directory you run:
|
||||
#
|
||||
# niv update nixpkgs-21.11
|
||||
#
|
||||
# or, to update the PyPI package database -- which is necessary to make any
|
||||
# newly released packages visible -- you likewise run:
|
||||
#
|
||||
# niv update pypi-deps-db
|
||||
#
|
||||
# niv also supports chosing a specific revision, following a different
|
||||
# branch, etc. find complete documentation for the tool at
|
||||
# https://github.com/nmattia/niv
|
||||
sources = import nix/sources.nix;
|
||||
in
|
||||
{
|
||||
pkgsVersion ? "nixpkgs-21.11" # a string which chooses a nixpkgs from the
|
||||
# niv-managed sources data
|
||||
|
||||
, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself
|
||||
|
||||
, pypiData ? sources.pypi-deps-db # the pypi package database snapshot to use
|
||||
# for dependency resolution
|
||||
|
||||
, pythonVersion ? "python37" # a string choosing the python derivation from
|
||||
# nixpkgs to target
|
||||
|
||||
, extras ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras,
|
||||
# the dependencies of which the resulting package
|
||||
# will also depend on. Include all of the runtime
|
||||
# extras by default because the incremental cost of
|
||||
# including them is a lot smaller than the cost of
|
||||
# re-building the whole thing to add them.
|
||||
|
||||
, mach-nix ? import sources.mach-nix { # the mach-nix package to use to build
|
||||
# the tahoe-lafs package
|
||||
inherit pkgs pypiData;
|
||||
python = pythonVersion;
|
||||
}
|
||||
}:
|
||||
# The project name, version, and most other metadata are automatically
|
||||
# extracted from the source. Some requirements are not properly extracted
|
||||
# and those cases are handled below. The version can only be extracted if
|
||||
# `setup.py update_version` has been run (this is not at all ideal but it
|
||||
# seems difficult to fix) - so for now just be sure to run that first.
|
||||
mach-nix.buildPythonPackage rec {
|
||||
# Define the location of the Tahoe-LAFS source to be packaged. Clean up all
|
||||
# as many of the non-source files (eg the `.git` directory, `~` backup
|
||||
# files, nix's own `result` symlink, etc) as possible to avoid needing to
|
||||
# re-build when files that make no difference to the package have changed.
|
||||
src = pkgs.lib.cleanSource ./.;
|
||||
|
||||
# Select whichever package extras were requested.
|
||||
inherit extras;
|
||||
|
||||
# Define some extra requirements that mach-nix does not automatically detect
|
||||
# from inspection of the source. We typically don't need to put version
|
||||
# constraints on any of these requirements. The pypi-deps-db we're
|
||||
# operating with makes dependency resolution deterministic so as long as it
|
||||
# works once it will always work. It could be that in the future we update
|
||||
# pypi-deps-db and an incompatibility arises - in which case it would make
|
||||
# sense to apply some version constraints here.
|
||||
requirementsExtra = ''
|
||||
# mach-nix does not yet support pyproject.toml which means it misses any
|
||||
# build-time requirements of our dependencies which are declared in such a
|
||||
# file. Tell it about them here.
|
||||
setuptools_rust
|
||||
|
||||
# mach-nix does not yet parse environment markers correctly. It misses
|
||||
# all of our requirements which have an environment marker. Duplicate them
|
||||
# here.
|
||||
foolscap
|
||||
eliot
|
||||
pyrsistent
|
||||
'';
|
||||
|
||||
# Specify where mach-nix should find packages for our Python dependencies.
|
||||
# There are some reasonable defaults so we only need to specify certain
|
||||
# packages where the default configuration runs into some issue.
|
||||
providers = {
|
||||
# Through zfec 1.5.5 the wheel has an incorrect runtime dependency
|
||||
# declared on argparse, not available for recent versions of Python 3.
|
||||
# Force mach-nix to use the sdist instead. This allows us to apply a
|
||||
# patch that removes the offending declaration.
|
||||
zfec = "sdist";
|
||||
};
|
||||
|
||||
# Define certain overrides to the way Python dependencies are built.
|
||||
_ = {
|
||||
# Apply the argparse declaration fix to zfec sdist.
|
||||
zfec.patches = with pkgs; [
|
||||
(fetchpatch {
|
||||
name = "fix-argparse.patch";
|
||||
url = "https://github.com/tahoe-lafs/zfec/commit/c3e736a72cccf44b8e1fb7d6c276400204c6bc1e.patch";
|
||||
sha256 = "1md9i2fx1ya7mgcj9j01z58hs3q9pj4ch5is5b5kq4v86cf6x33x";
|
||||
})
|
||||
];
|
||||
|
||||
# Remove a click-default-group patch for a test suite problem which no
|
||||
# longer applies because the project apparently no longer has a test suite
|
||||
# in its source distribution.
|
||||
click-default-group.patches = [];
|
||||
};
|
||||
|
||||
passthru.meta.mach-nix = {
|
||||
inherit providers _;
|
||||
};
|
||||
}
|
||||
0
newsfragments/3867.minor
Normal file
0
newsfragments/3867.minor
Normal file
@ -1,34 +0,0 @@
|
||||
{ lib, buildPythonPackage, fetchPypi, isPy3k,
|
||||
six, txaio, twisted, zope_interface, cffi, futures,
|
||||
mock, pytest, cryptography, pynacl
|
||||
}:
|
||||
buildPythonPackage rec {
|
||||
pname = "autobahn";
|
||||
version = "19.8.1";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "294e7381dd54e73834354832604ae85567caf391c39363fed0ea2bfa86aa4304";
|
||||
};
|
||||
|
||||
propagatedBuildInputs = [ six txaio twisted zope_interface cffi cryptography pynacl ] ++
|
||||
(lib.optionals (!isPy3k) [ futures ]);
|
||||
|
||||
checkInputs = [ mock pytest ];
|
||||
checkPhase = ''
|
||||
runHook preCheck
|
||||
USE_TWISTED=true py.test $out
|
||||
runHook postCheck
|
||||
'';
|
||||
|
||||
# Tests do no seem to be compatible yet with pytest 5.1
|
||||
# https://github.com/crossbario/autobahn-python/issues/1235
|
||||
doCheck = false;
|
||||
|
||||
meta = with lib; {
|
||||
description = "WebSocket and WAMP in Python for Twisted and asyncio.";
|
||||
homepage = "https://crossbar.io/autobahn";
|
||||
license = licenses.mit;
|
||||
maintainers = with maintainers; [ nand0p ];
|
||||
};
|
||||
}
|
||||
@ -1,20 +0,0 @@
|
||||
{ lib, buildPythonPackage, fetchPypi, setuptools_scm }:
|
||||
buildPythonPackage rec {
|
||||
pname = "cbor2";
|
||||
version = "5.2.0";
|
||||
|
||||
src = fetchPypi {
|
||||
sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3";
|
||||
inherit pname version;
|
||||
};
|
||||
|
||||
doCheck = false;
|
||||
|
||||
propagatedBuildInputs = [ setuptools_scm ];
|
||||
|
||||
meta = with lib; {
|
||||
homepage = https://github.com/agronholm/cbor2;
|
||||
description = "CBOR encoder/decoder";
|
||||
license = licenses.mit;
|
||||
};
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
{ lib, buildPythonPackage, fetchPypi }:
|
||||
buildPythonPackage rec {
|
||||
pname = "collections-extended";
|
||||
version = "1.0.3";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw";
|
||||
};
|
||||
|
||||
# Tests aren't in tarball, for 1.0.3 at least.
|
||||
doCheck = false;
|
||||
|
||||
meta = with lib; {
|
||||
homepage = https://github.com/mlenzen/collections-extended;
|
||||
description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict";
|
||||
license = licenses.asl20;
|
||||
};
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
# This is the main entrypoint for the Tahoe-LAFS derivation.
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
# Add our Python packages to nixpkgs to simplify the expression for the
|
||||
# Tahoe-LAFS derivation.
|
||||
let pkgs' = pkgs.extend (import ./overlays.nix);
|
||||
# Evaluate the expression for our Tahoe-LAFS derivation.
|
||||
in pkgs'.python2.pkgs.callPackage ./tahoe-lafs.nix { }
|
||||
@ -1,31 +0,0 @@
|
||||
{ lib, buildPythonPackage, fetchPypi, zope_interface, pyrsistent, boltons
|
||||
, hypothesis, testtools, pytest }:
|
||||
buildPythonPackage rec {
|
||||
pname = "eliot";
|
||||
version = "1.7.0";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "0ylyycf717s5qsrx8b9n6m38vyj2k8328lfhn8y6r31824991wv8";
|
||||
};
|
||||
|
||||
postPatch = ''
|
||||
substituteInPlace setup.py \
|
||||
--replace "boltons >= 19.0.1" boltons
|
||||
'';
|
||||
|
||||
# A seemingly random subset of the test suite fails intermittently. After
|
||||
# Tahoe-LAFS is ported to Python 3 we can update to a newer Eliot and, if
|
||||
# the test suite continues to fail, maybe it will be more likely that we can
|
||||
# have upstream fix it for us.
|
||||
doCheck = false;
|
||||
|
||||
checkInputs = [ testtools pytest hypothesis ];
|
||||
propagatedBuildInputs = [ zope_interface pyrsistent boltons ];
|
||||
|
||||
meta = with lib; {
|
||||
homepage = https://github.com/itamarst/eliot/;
|
||||
description = "Logging library that tells you why it happened";
|
||||
license = licenses.asl20;
|
||||
};
|
||||
}
|
||||
@ -1,35 +0,0 @@
|
||||
{ lib
|
||||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "future";
|
||||
version = "0.18.2";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "sha256:0zakvfj87gy6mn1nba06sdha63rn4njm7bhh0wzyrxhcny8avgmi";
|
||||
};
|
||||
|
||||
doCheck = false;
|
||||
|
||||
meta = {
|
||||
description = "Clean single-source support for Python 3 and 2";
|
||||
longDescription = ''
|
||||
python-future is the missing compatibility layer between Python 2 and
|
||||
Python 3. It allows you to use a single, clean Python 3.x-compatible
|
||||
codebase to support both Python 2 and Python 3 with minimal overhead.
|
||||
|
||||
It provides future and past packages with backports and forward ports
|
||||
of features from Python 3 and 2. It also comes with futurize and
|
||||
pasteurize, customized 2to3-based scripts that helps you to convert
|
||||
either Py2 or Py3 code easily to support both Python 2 and 3 in a
|
||||
single clean Py3-style codebase, module by module.
|
||||
'';
|
||||
homepage = https://python-future.org;
|
||||
downloadPage = https://github.com/PythonCharmers/python-future/releases;
|
||||
license = with lib.licenses; [ mit ];
|
||||
maintainers = with lib.maintainers; [ prikhi ];
|
||||
};
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
self: super: {
|
||||
python27 = super.python27.override {
|
||||
packageOverrides = python-self: python-super: {
|
||||
# eliot is not part of nixpkgs at all at this time.
|
||||
eliot = python-self.pythonPackages.callPackage ./eliot.nix { };
|
||||
|
||||
# NixOS autobahn package has trollius as a dependency, although
|
||||
# it is optional. Trollius is unmaintained and fails on CI.
|
||||
autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { };
|
||||
|
||||
# Porting to Python 3 is greatly aided by the future package. A
|
||||
# slightly newer version than appears in nixos 19.09 is helpful.
|
||||
future = python-super.pythonPackages.callPackage ./future.nix { };
|
||||
|
||||
# Need version of pyutil that supports Python 3. The version in 19.09
|
||||
# is too old.
|
||||
pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { };
|
||||
|
||||
# Need a newer version of Twisted, too.
|
||||
twisted = python-super.pythonPackages.callPackage ./twisted.nix { };
|
||||
|
||||
# collections-extended is not part of nixpkgs at this time.
|
||||
collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { };
|
||||
|
||||
# cbor2 is not part of nixpkgs at this time.
|
||||
cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { };
|
||||
};
|
||||
};
|
||||
|
||||
python39 = super.python39.override {
|
||||
packageOverrides = python-self: python-super: {
|
||||
# collections-extended is not part of nixpkgs at this time.
|
||||
collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { };
|
||||
};
|
||||
};
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
# This is the main entrypoint for the Tahoe-LAFS derivation.
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
# Add our Python packages to nixpkgs to simplify the expression for the
|
||||
# Tahoe-LAFS derivation.
|
||||
let pkgs' = pkgs.extend (import ./overlays.nix);
|
||||
# Evaluate the expression for our Tahoe-LAFS derivation.
|
||||
in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { }
|
||||
@ -1,48 +0,0 @@
|
||||
{ stdenv
|
||||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
, setuptoolsDarcs
|
||||
, setuptoolsTrial
|
||||
, simplejson
|
||||
, twisted
|
||||
, isPyPy
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "pyutil";
|
||||
version = "3.3.0";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "8c4d4bf668c559186389bb9bce99e4b1b871c09ba252a756ccaacd2b8f401848";
|
||||
};
|
||||
|
||||
buildInputs = [ setuptoolsDarcs setuptoolsTrial ] ++ (if doCheck then [ simplejson ] else []);
|
||||
propagatedBuildInputs = [ twisted ];
|
||||
|
||||
# Tests fail because they try to write new code into the twisted
|
||||
# package, apparently some kind of plugin.
|
||||
doCheck = false;
|
||||
|
||||
prePatch = stdenv.lib.optionalString isPyPy ''
|
||||
grep -rl 'utf-8-with-signature-unix' ./ | xargs sed -i -e "s|utf-8-with-signature-unix|utf-8|g"
|
||||
'';
|
||||
|
||||
meta = with stdenv.lib; {
|
||||
description = "Pyutil, a collection of mature utilities for Python programmers";
|
||||
|
||||
longDescription = ''
|
||||
These are a few data structures, classes and functions which
|
||||
we've needed over many years of Python programming and which
|
||||
seem to be of general use to other Python programmers. Many of
|
||||
the modules that have existed in pyutil over the years have
|
||||
subsequently been obsoleted by new features added to the
|
||||
Python language or its standard library, thus showing that
|
||||
we're not alone in wanting tools like these.
|
||||
'';
|
||||
|
||||
homepage = "http://allmydata.org/trac/pyutil";
|
||||
license = licenses.gpl2Plus;
|
||||
};
|
||||
|
||||
}
|
||||
62
nix/sources.json
Normal file
62
nix/sources.json
Normal file
@ -0,0 +1,62 @@
|
||||
{
|
||||
"mach-nix": {
|
||||
"branch": "master",
|
||||
"description": "Create highly reproducible python environments",
|
||||
"homepage": "",
|
||||
"owner": "davhau",
|
||||
"repo": "mach-nix",
|
||||
"rev": "bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b",
|
||||
"sha256": "12b3jc0g0ak6s93g3ifvdpwxbyqx276k1kl66bpwz8a67qjbcbwf",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/davhau/mach-nix/archive/bdc97ba6b2ecd045a467b008cff4ae337b6a7a6b.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"niv": {
|
||||
"branch": "master",
|
||||
"description": "Easy dependency management for Nix projects",
|
||||
"homepage": "https://github.com/nmattia/niv",
|
||||
"owner": "nmattia",
|
||||
"repo": "niv",
|
||||
"rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c",
|
||||
"sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs-21.05": {
|
||||
"branch": "nixos-21.05",
|
||||
"description": "Nix Packages collection",
|
||||
"homepage": "",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00",
|
||||
"sha256": "1mr2qgv5r2nmf6s3gqpcjj76zpsca6r61grzmqngwm0xlh958smx",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/NixOS/nixpkgs/archive/0fd9ee1aa36ce865ad273f4f07fdc093adeb5c00.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"nixpkgs-21.11": {
|
||||
"branch": "nixos-21.11",
|
||||
"description": "Nix Packages collection",
|
||||
"homepage": "",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca",
|
||||
"sha256": "1yl5gj0mzczhl1j8sl8iqpwa1jzsgr12fdszw9rq13cdig2a2r5f",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/nixos/nixpkgs/archive/6c4b9f1a2fd761e2d384ef86cff0d208ca27fdca.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
},
|
||||
"pypi-deps-db": {
|
||||
"branch": "master",
|
||||
"description": "Probably the most complete python dependency database",
|
||||
"homepage": "",
|
||||
"owner": "DavHau",
|
||||
"repo": "pypi-deps-db",
|
||||
"rev": "0f6de8bf1f186c275af862ec9667abb95aae8542",
|
||||
"sha256": "1ygw9pywyl4p25hx761d1sbwl3qjhm630fa36gdf6b649im4mx8y",
|
||||
"type": "tarball",
|
||||
"url": "https://github.com/DavHau/pypi-deps-db/archive/0f6de8bf1f186c275af862ec9667abb95aae8542.tar.gz",
|
||||
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
|
||||
}
|
||||
}
|
||||
174
nix/sources.nix
Normal file
174
nix/sources.nix
Normal file
@ -0,0 +1,174 @@
|
||||
# This file has been generated by Niv.
|
||||
|
||||
let
|
||||
|
||||
#
|
||||
# The fetchers. fetch_<type> fetches specs of type <type>.
|
||||
#
|
||||
|
||||
fetch_file = pkgs: name: spec:
|
||||
let
|
||||
name' = sanitizeName name + "-src";
|
||||
in
|
||||
if spec.builtin or true then
|
||||
builtins_fetchurl { inherit (spec) url sha256; name = name'; }
|
||||
else
|
||||
pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
|
||||
|
||||
fetch_tarball = pkgs: name: spec:
|
||||
let
|
||||
name' = sanitizeName name + "-src";
|
||||
in
|
||||
if spec.builtin or true then
|
||||
builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
|
||||
else
|
||||
pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
|
||||
|
||||
fetch_git = name: spec:
|
||||
let
|
||||
ref =
|
||||
if spec ? ref then spec.ref else
|
||||
if spec ? branch then "refs/heads/${spec.branch}" else
|
||||
if spec ? tag then "refs/tags/${spec.tag}" else
|
||||
abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
|
||||
in
|
||||
builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
|
||||
|
||||
fetch_local = spec: spec.path;
|
||||
|
||||
fetch_builtin-tarball = name: throw
|
||||
''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
|
||||
$ niv modify ${name} -a type=tarball -a builtin=true'';
|
||||
|
||||
fetch_builtin-url = name: throw
|
||||
''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
|
||||
$ niv modify ${name} -a type=file -a builtin=true'';
|
||||
|
||||
#
|
||||
# Various helpers
|
||||
#
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
|
||||
sanitizeName = name:
|
||||
(
|
||||
concatMapStrings (s: if builtins.isList s then "-" else s)
|
||||
(
|
||||
builtins.split "[^[:alnum:]+._?=-]+"
|
||||
((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
|
||||
)
|
||||
);
|
||||
|
||||
# The set of packages used when specs are fetched using non-builtins.
|
||||
mkPkgs = sources: system:
|
||||
let
|
||||
sourcesNixpkgs =
|
||||
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
|
||||
hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
|
||||
hasThisAsNixpkgsPath = <nixpkgs> == ./.;
|
||||
in
|
||||
if builtins.hasAttr "nixpkgs" sources
|
||||
then sourcesNixpkgs
|
||||
else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
|
||||
import <nixpkgs> {}
|
||||
else
|
||||
abort
|
||||
''
|
||||
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
|
||||
add a package called "nixpkgs" to your sources.json.
|
||||
'';
|
||||
|
||||
# The actual fetching function.
|
||||
fetch = pkgs: name: spec:
|
||||
|
||||
if ! builtins.hasAttr "type" spec then
|
||||
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
|
||||
else if spec.type == "file" then fetch_file pkgs name spec
|
||||
else if spec.type == "tarball" then fetch_tarball pkgs name spec
|
||||
else if spec.type == "git" then fetch_git name spec
|
||||
else if spec.type == "local" then fetch_local spec
|
||||
else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
|
||||
else if spec.type == "builtin-url" then fetch_builtin-url name
|
||||
else
|
||||
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
|
||||
|
||||
# If the environment variable NIV_OVERRIDE_${name} is set, then use
|
||||
# the path directly as opposed to the fetched source.
|
||||
replace = name: drv:
|
||||
let
|
||||
saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
|
||||
ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
|
||||
in
|
||||
if ersatz == "" then drv else
|
||||
# this turns the string into an actual Nix path (for both absolute and
|
||||
# relative paths)
|
||||
if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
|
||||
|
||||
# Ports of functions for older nix versions
|
||||
|
||||
# a Nix version of mapAttrs if the built-in doesn't exist
|
||||
mapAttrs = builtins.mapAttrs or (
|
||||
f: set: with builtins;
|
||||
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
|
||||
);
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
|
||||
range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
|
||||
stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
|
||||
stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
|
||||
concatMapStrings = f: list: concatStrings (map f list);
|
||||
concatStrings = builtins.concatStringsSep "";
|
||||
|
||||
# https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
|
||||
optionalAttrs = cond: as: if cond then as else {};
|
||||
|
||||
# fetchTarball version that is compatible between all the versions of Nix
|
||||
builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
|
||||
let
|
||||
inherit (builtins) lessThan nixVersion fetchTarball;
|
||||
in
|
||||
if lessThan nixVersion "1.12" then
|
||||
fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
|
||||
else
|
||||
fetchTarball attrs;
|
||||
|
||||
# fetchurl version that is compatible between all the versions of Nix
|
||||
builtins_fetchurl = { url, name ? null, sha256 }@attrs:
|
||||
let
|
||||
inherit (builtins) lessThan nixVersion fetchurl;
|
||||
in
|
||||
if lessThan nixVersion "1.12" then
|
||||
fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
|
||||
else
|
||||
fetchurl attrs;
|
||||
|
||||
# Create the final "sources" from the config
|
||||
mkSources = config:
|
||||
mapAttrs (
|
||||
name: spec:
|
||||
if builtins.hasAttr "outPath" spec
|
||||
then abort
|
||||
"The values in sources.json should not have an 'outPath' attribute"
|
||||
else
|
||||
spec // { outPath = replace name (fetch config.pkgs name spec); }
|
||||
) config.sources;
|
||||
|
||||
# The "config" used by the fetchers
|
||||
mkConfig =
|
||||
{ sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
|
||||
, sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
|
||||
, system ? builtins.currentSystem
|
||||
, pkgs ? mkPkgs sources system
|
||||
}: rec {
|
||||
# The sources, i.e. the attribute set of spec name to spec
|
||||
inherit sources;
|
||||
|
||||
# The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
|
||||
inherit pkgs;
|
||||
};
|
||||
|
||||
in
|
||||
mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }
|
||||
@ -1,126 +0,0 @@
|
||||
{ fetchFromGitHub, lib
|
||||
, git, python
|
||||
, twisted, foolscap, zfec
|
||||
, setuptools, setuptoolsTrial, pyasn1, zope_interface
|
||||
, service-identity, pyyaml, magic-wormhole, treq, appdirs
|
||||
, beautifulsoup4, eliot, autobahn, cryptography, netifaces
|
||||
, html5lib, pyutil, distro, configparser, klein, cbor2
|
||||
}:
|
||||
python.pkgs.buildPythonPackage rec {
|
||||
# Most of the time this is not exactly the release version (eg 1.17.1).
|
||||
# Give it a `post` component to make it look newer than the release version
|
||||
# and we'll bump this up at the time of each release.
|
||||
#
|
||||
# It's difficult to read the version from Git the way the Python code does
|
||||
# for two reasons. First, doing so involves populating the Nix expression
|
||||
# with values from the source. Nix calls this "import from derivation" or
|
||||
# "IFD" (<https://nixos.wiki/wiki/Import_From_Derivation>). This is
|
||||
# discouraged in most cases - including this one, I think. Second, the
|
||||
# Python code reads the contents of `.git` to determine its version. `.git`
|
||||
# is not a reproducable artifact (in the sense of "reproducable builds") so
|
||||
# it is excluded from the source tree by default. When it is included, the
|
||||
# package tends to be frequently spuriously rebuilt.
|
||||
version = "1.17.1.post1";
|
||||
name = "tahoe-lafs-${version}";
|
||||
src = lib.cleanSourceWith {
|
||||
src = ../.;
|
||||
filter = name: type:
|
||||
let
|
||||
basename = baseNameOf name;
|
||||
|
||||
split = lib.splitString ".";
|
||||
join = builtins.concatStringsSep ".";
|
||||
ext = join (builtins.tail (split basename));
|
||||
|
||||
# Build up a bunch of knowledge about what kind of file this is.
|
||||
isTox = type == "directory" && basename == ".tox";
|
||||
isTrialTemp = type == "directory" && basename == "_trial_temp";
|
||||
isVersion = basename == "_version.py";
|
||||
isBytecode = ext == "pyc" || ext == "pyo";
|
||||
isBackup = lib.hasSuffix "~" basename;
|
||||
isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename;
|
||||
isSymlink = type == "symlink";
|
||||
isGit = type == "directory" && basename == ".git";
|
||||
in
|
||||
# Exclude all these things
|
||||
! (isTox
|
||||
|| isTrialTemp
|
||||
|| isVersion
|
||||
|| isBytecode
|
||||
|| isBackup
|
||||
|| isTemporary
|
||||
|| isSymlink
|
||||
|| isGit
|
||||
);
|
||||
};
|
||||
|
||||
postPatch = ''
|
||||
# Chroots don't have /etc/hosts and /etc/resolv.conf, so work around
|
||||
# that.
|
||||
for i in $(find src/allmydata/test -type f)
|
||||
do
|
||||
sed -i "$i" -e"s/localhost/127.0.0.1/g"
|
||||
done
|
||||
|
||||
# Some tests are flaky or fail to skip when dependencies are missing.
|
||||
# This list is over-zealous because it's more work to disable individual
|
||||
# tests with in a module.
|
||||
|
||||
# Many of these tests don't properly skip when i2p or tor dependencies are
|
||||
# not supplied (and we are not supplying them).
|
||||
rm src/allmydata/test/test_i2p_provider.py
|
||||
rm src/allmydata/test/test_connections.py
|
||||
rm src/allmydata/test/cli/test_create.py
|
||||
|
||||
# Generate _version.py ourselves since we can't rely on the Python code
|
||||
# extracting the information from the .git directory we excluded.
|
||||
cat > src/allmydata/_version.py <<EOF
|
||||
|
||||
# This _version.py is generated from metadata by nix/tahoe-lafs.nix.
|
||||
|
||||
__pkgname__ = "tahoe-lafs"
|
||||
real_version = "${version}"
|
||||
full_version = "${version}"
|
||||
branch = "master"
|
||||
verstr = "${version}"
|
||||
__version__ = verstr
|
||||
EOF
|
||||
'';
|
||||
|
||||
|
||||
nativeBuildInputs = [
|
||||
git
|
||||
];
|
||||
|
||||
propagatedBuildInputs = with python.pkgs; [
|
||||
twisted foolscap zfec appdirs
|
||||
setuptoolsTrial pyasn1 zope_interface
|
||||
service-identity pyyaml magic-wormhole
|
||||
eliot autobahn cryptography netifaces setuptools
|
||||
future pyutil distro configparser collections-extended
|
||||
klein cbor2 treq
|
||||
];
|
||||
|
||||
checkInputs = with python.pkgs; [
|
||||
hypothesis
|
||||
testtools
|
||||
fixtures
|
||||
beautifulsoup4
|
||||
html5lib
|
||||
tenacity
|
||||
prometheus_client
|
||||
];
|
||||
|
||||
checkPhase = ''
|
||||
if ! $out/bin/tahoe --version | grep --fixed-strings "${version}"; then
|
||||
echo "Package version:"
|
||||
$out/bin/tahoe --version
|
||||
echo "Did not contain expected:"
|
||||
echo "${version}"
|
||||
exit 1
|
||||
else
|
||||
echo "Version string contained expected value \"${version}.\""
|
||||
fi
|
||||
${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
|
||||
'';
|
||||
}
|
||||
@ -1,63 +0,0 @@
|
||||
{ stdenv
|
||||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
, python
|
||||
, zope_interface
|
||||
, incremental
|
||||
, automat
|
||||
, constantly
|
||||
, hyperlink
|
||||
, pyhamcrest
|
||||
, attrs
|
||||
, pyopenssl
|
||||
, service-identity
|
||||
, setuptools
|
||||
, idna
|
||||
, bcrypt
|
||||
}:
|
||||
buildPythonPackage rec {
|
||||
pname = "Twisted";
|
||||
version = "19.10.0";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
extension = "tar.bz2";
|
||||
sha256 = "7394ba7f272ae722a74f3d969dcf599bc4ef093bc392038748a490f1724a515d";
|
||||
};
|
||||
|
||||
propagatedBuildInputs = [ zope_interface incremental automat constantly hyperlink pyhamcrest attrs setuptools bcrypt ];
|
||||
|
||||
passthru.extras.tls = [ pyopenssl service-identity idna ];
|
||||
|
||||
# Patch t.p._inotify to point to libc. Without this,
|
||||
# twisted.python.runtime.platform.supportsINotify() == False
|
||||
patchPhase = stdenv.lib.optionalString stdenv.isLinux ''
|
||||
substituteInPlace src/twisted/python/_inotify.py --replace \
|
||||
"ctypes.util.find_library('c')" "'${stdenv.glibc.out}/lib/libc.so.6'"
|
||||
'';
|
||||
|
||||
# Generate Twisted's plug-in cache. Twisted users must do it as well. See
|
||||
# http://twistedmatrix.com/documents/current/core/howto/plugin.html#auto3
|
||||
# and http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=477103 for
|
||||
# details.
|
||||
postFixup = ''
|
||||
$out/bin/twistd --help > /dev/null
|
||||
'';
|
||||
|
||||
checkPhase = ''
|
||||
${python.interpreter} -m unittest discover -s twisted/test
|
||||
'';
|
||||
# Tests require network
|
||||
doCheck = false;
|
||||
|
||||
meta = with stdenv.lib; {
|
||||
homepage = https://twistedmatrix.com/;
|
||||
description = "Twisted, an event-driven networking engine written in Python";
|
||||
longDescription = ''
|
||||
Twisted is an event-driven networking engine written in Python
|
||||
and licensed under the MIT license.
|
||||
'';
|
||||
license = licenses.mit;
|
||||
maintainers = [ ];
|
||||
};
|
||||
}
|
||||
87
tests.nix
Normal file
87
tests.nix
Normal file
@ -0,0 +1,87 @@
|
||||
let
|
||||
sources = import nix/sources.nix;
|
||||
in
|
||||
# See default.nix for documentation about parameters.
|
||||
{ pkgsVersion ? "nixpkgs-21.11"
|
||||
, pkgs ? import sources.${pkgsVersion} { }
|
||||
, pypiData ? sources.pypi-deps-db
|
||||
, pythonVersion ? "python37"
|
||||
, mach-nix ? import sources.mach-nix {
|
||||
inherit pkgs pypiData;
|
||||
python = pythonVersion;
|
||||
}
|
||||
}@args:
|
||||
let
|
||||
# We would like to know the test requirements but mach-nix does not directly
|
||||
# expose this information to us. However, it is perfectly capable of
|
||||
# determining it if we ask right... This is probably not meant to be a
|
||||
# public mach-nix API but we pinned mach-nix so we can deal with mach-nix
|
||||
# upgrade breakage in our own time.
|
||||
mach-lib = import "${sources.mach-nix}/mach_nix/nix/lib.nix" {
|
||||
inherit pkgs;
|
||||
lib = pkgs.lib;
|
||||
};
|
||||
tests_require = (mach-lib.extract "python37" ./. "extras_require" ).extras_require.test;
|
||||
|
||||
# Get the Tahoe-LAFS package itself. This does not include test
|
||||
# requirements and we don't ask for test requirements so that we can just
|
||||
# re-use the normal package if it is already built.
|
||||
tahoe-lafs = import ./. args;
|
||||
|
||||
# If we want to get tahoe-lafs into a Python environment with a bunch of
|
||||
# *other* Python modules and let them interact in the usual way then we have
|
||||
# to ask mach-nix for tahoe-lafs and those other Python modules in the same
|
||||
# way - i.e., using `requirements`. The other tempting mechanism,
|
||||
# `packagesExtra`, inserts an extra layer of Python environment and prevents
|
||||
# normal interaction between Python modules (as well as usually producing
|
||||
# file collisions in the packages that are both runtime and test
|
||||
# dependencies). To get the tahoe-lafs we just built into the environment,
|
||||
# put it into nixpkgs using an overlay and tell mach-nix to get tahoe-lafs
|
||||
# from nixpkgs.
|
||||
overridesPre = [(self: super: { inherit tahoe-lafs; })];
|
||||
providers = tahoe-lafs.meta.mach-nix.providers // { tahoe-lafs = "nixpkgs"; };
|
||||
|
||||
# Make the Python environment in which we can run the tests.
|
||||
python-env = mach-nix.mkPython {
|
||||
# Get the packaging fixes we already know we need from putting together
|
||||
# the runtime package.
|
||||
inherit (tahoe-lafs.meta.mach-nix) _;
|
||||
# Share the runtime package's provider configuration - combined with our
|
||||
# own that causes the right tahoe-lafs to be picked up.
|
||||
inherit providers overridesPre;
|
||||
requirements = ''
|
||||
# Here we pull in the Tahoe-LAFS package itself.
|
||||
tahoe-lafs
|
||||
|
||||
# Unfortunately mach-nix misses all of the Python dependencies of the
|
||||
# tahoe-lafs satisfied from nixpkgs. Drag them in here. This gives a
|
||||
# bit of a pyrrhic flavor to the whole endeavor but maybe mach-nix will
|
||||
# fix this soon.
|
||||
#
|
||||
# https://github.com/DavHau/mach-nix/issues/123
|
||||
# https://github.com/DavHau/mach-nix/pull/386
|
||||
${tahoe-lafs.requirements}
|
||||
|
||||
# And then all of the test-only dependencies.
|
||||
${builtins.concatStringsSep "\n" tests_require}
|
||||
|
||||
# txi2p-tahoe is another dependency with an environment marker that
|
||||
# mach-nix doesn't automatically pick up.
|
||||
txi2p-tahoe
|
||||
'';
|
||||
};
|
||||
in
|
||||
# Make a derivation that runs the unit test suite.
|
||||
pkgs.runCommand "tahoe-lafs-tests" { } ''
|
||||
${python-env}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
|
||||
|
||||
# It's not cool to put the whole _trial_temp into $out because it has weird
|
||||
# files in it we don't want in the store. Plus, even all of the less weird
|
||||
# files are mostly just trash that's not meaningful if the test suite passes
|
||||
# (which is the only way we get $out anyway).
|
||||
#
|
||||
# The build log itself is typically available from `nix-store --read-log` so
|
||||
# we don't need to record that either.
|
||||
echo "passed" >$out
|
||||
|
||||
''
|
||||
Loading…
Reference in New Issue
Block a user