From 92474375350022fc12521b060160beeba777e3db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:43:23 -0400 Subject: [PATCH 01/54] Add a flake with some packages and apps and an overlay --- default.nix | 29 +------ flake.lock | 80 +++++++++++++++++++ flake.nix | 162 +++++++++++++++++++++++++++++++++++++++ nix/overlay.nix | 10 +++ nix/python-overrides.nix | 42 +++++++--- nix/tahoe-lafs.nix | 74 ++++++++---------- nix/tests.nix | 4 - 7 files changed, 323 insertions(+), 78 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/overlay.nix delete mode 100644 nix/tests.nix diff --git a/default.nix b/default.nix index d616f63b8..196db4030 100644 --- a/default.nix +++ b/default.nix @@ -18,32 +18,11 @@ in pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the # niv-managed sources data -, pkgs ? import sources.${pkgsVersion} { } # nixpkgs itself +, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself + overlays = [ (import ./nix/overlay.nix) ]; +} , pythonVersion ? "python310" # a string choosing the python derivation from # nixpkgs to target - -, extrasNames ? [ "tor" "i2p" ] # a list of strings identifying tahoe-lafs extras, - # the dependencies of which the resulting - # package will also depend on. Include all of the - # runtime extras by default because the incremental - # cost of including them is a lot smaller than the - # cost of re-building the whole thing to add them. - }: -with (pkgs.${pythonVersion}.override { - packageOverrides = import ./nix/python-overrides.nix; -}).pkgs; -callPackage ./nix/tahoe-lafs.nix { - # Select whichever package extras were requested. - inherit extrasNames; - - # Define the location of the Tahoe-LAFS source to be packaged (the same - # directory as contains this file). Clean up as many of the non-source - # files (eg the `.git` directory, `~` backup files, nix's own `result` - # symlink, etc) as possible to avoid needing to re-build when files that - # make no difference to the package have changed. - tahoe-lafs-src = pkgs.lib.cleanSource ./.; - - doCheck = false; -} +pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ]) diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..0fd3c90c9 --- /dev/null +++ b/flake.lock @@ -0,0 +1,80 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1687709756, + "narHash": "sha256-Y5wKlQSkgEK2weWdOu4J3riRd+kV/VCgHsqLNTTWQ/0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "dbabf0ca0c0c4bce6ea5eaf65af5cb694d2082c7", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs-22_11": { + "locked": { + "lastModified": 1688392541, + "narHash": "sha256-lHrKvEkCPTUO+7tPfjIcb7Trk6k31rz18vkyqmkeJfY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ea4c80b39be4c09702b0cb3b42eab59e2ba4f24b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-22.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1688486599, + "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": [ + "nixpkgs-22_11" + ], + "nixpkgs-22_11": "nixpkgs-22_11", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..89a4a6f14 --- /dev/null +++ b/flake.nix @@ -0,0 +1,162 @@ +{ + description = "Tahoe-LAFS, free and open decentralized data store"; + + inputs = { + # Two alternate nixpkgs pins. Ideally these could be selected easily from + # the command line but there seems to be no syntax/support for that. + # However, these at least cause certain revisions to be pinned in our lock + # file where you *can* dig them out - and the CI configuration does. + "nixpkgs-22_11" = { + url = github:NixOS/nixpkgs?ref=nixos-22.11; + }; + "nixpkgs-unstable" = { + url = github:NixOS/nixpkgs; + }; + + # Point the default nixpkgs at one of those. This avoids having getting a + # _third_ package set involved and gives a way to provide what should be a + # working experience by default (that is, if nixpkgs doesn't get + # overridden). + nixpkgs.follows = "nixpkgs-22_11"; + + # Also get flake-utils for simplified multi-system definitions. + flake-utils = { + url = github:numtide/flake-utils; + }; + }; + + outputs = { self, nixpkgs, flake-utils, ... }: + { + # Expose an overlay which adds our version of Tahoe-LAFS to the Python + # package sets we specify, as well as all of the correct versions of its + # dependencies. + # + # We will also use this to define some other outputs since it gives us + # the most succinct way to get a working Tahoe-LAFS package. + overlays.default = import ./nix/overlay.nix; + + } // (flake-utils.lib.eachDefaultSystem (system: let + + # First get the package set for this system architecture. + pkgs = import nixpkgs { + inherit system; + # And include our Tahoe-LAFS package in that package set. + overlays = [ self.overlays.default ]; + }; + + # Find out what Python versions we're working with. + pythonVersions = builtins.attrNames ( + pkgs.lib.attrsets.filterAttrs + # Match attribute names that look like a Python derivation - CPython + # or PyPy. We take care to avoid things like "python-foo" and + # "python3Full-unittest" though. We only want things like "pypy38" + # or "python311". + (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name) + pkgs + ); + + # An element of pythonVersions which we'll use for the default package. + defaultPyVersion = "python310"; + + # Retrieve the actual Python package for each configured version. We + # already applied our overlay to pkgs so our packages will already be + # available. + pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions; + + # string -> string + # + # Construct the Tahoe-LAFS package name for the given Python runtime. + packageName = pyVersion: "${pyVersion}-tahoe-lafs"; + + # string -> string + # + # Construct the unit test application name for the given Python runtime. + unitTestName = pyVersion: "${pyVersion}-unittest"; + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeRuntimeEnv = pyVersion: { + ${packageName pyVersion} = makeRuntimeEnv' pyVersion; + }; + + makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and + # all of its dependencies. + makeTestEnv = pyVersion: { + ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor ++ + tahoe-lafs.passthru.extras.unittest + )).overrideAttrs (old: { + name = packageName pyVersion; + }); + }; + in { + # Define the flake's package outputs. We'll define one version of the + # package for each version of Python we could find. We'll also point + # the flake's "default" package at one of these somewhat arbitrarily. + # The package consists of a Python environment with Tahoe-LAFS available + # to it. + packages = with pkgs.lib; + foldr mergeAttrs {} ([ + { default = self.packages.${system}.${packageName defaultPyVersion}; } + ] ++ (builtins.map makeRuntimeEnv pythonVersions)); + + # Define the flake's app outputs. We'll define a version of an app for + # running the test suite for each version of Python we could find. + # We'll also define a version of an app for running the "tahoe" + # command-line entrypoint for each version of Python we could find. + apps = + let + # We avoid writeShellApplication here because it has ghc as a + # dependency but ghc has Python as a dependency and our Python + # package override triggers a rebuild of ghc which takes a looong + # time. + writeScript = name: text: + let script = pkgs.writeShellScript name text; + in "${script}"; + + # A helper function to define the runtime entrypoint for a certain + # Python runtime. + makeTahoeApp = pyVersion: { + "tahoe-${pyVersion}" = { + type = "app"; + program = + writeScript "tahoe" + '' + ${makeRuntimeEnv' pyVersion}/bin/tahoe "$@" + ''; + }; + }; + + # A helper function to define the unit test entrypoint for a certain + # Python runtime. + makeUnitTestsApp = pyVersion: { + "${unitTestName pyVersion}" = { + type = "app"; + program = + writeScript "unit-tests" + '' + export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ''; + }; + }; + in + with pkgs.lib; + foldr mergeAttrs + { default = self.apps.${system}."tahoe-python3"; } + ( + (builtins.map makeUnitTestsApp pythonVersions) ++ + (builtins.map makeTahoeApp pythonVersions) + ); + })); +} diff --git a/nix/overlay.nix b/nix/overlay.nix new file mode 100644 index 000000000..41f0e3086 --- /dev/null +++ b/nix/overlay.nix @@ -0,0 +1,10 @@ +# This overlay adds Tahoe-LAFS and all of its properly-configured Python +# package dependencies to a Python package set. Downstream consumers can +# apply it to their own nixpkgs derivation to produce a Tahoe-LAFS package. +final: prev: { + # Add our overrides such that they will be applied to any Python derivation + # in nixpkgs. + pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [ + (import ./python-overrides.nix) + ]; +} diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 0ed415691..a333808be 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -9,14 +9,39 @@ let # Disable a Python package's test suite. dontCheck = drv: drv.overrideAttrs (old: { doInstallCheck = false; }); + # string -> any -> derivation -> derivation + # + # If the overrideable function for the given derivation accepts an argument + # with the given name, override it with the given value. + # + # Since we try to work with multiple versions of nixpkgs, sometimes we need + # to override a parameter that exists in one version but not others. This + # makes it a bit easier to do so. + overrideIfPresent = name: value: drv: + if (drv.override.__functionArgs ? ${name}) + then drv.override { "${name}" = value; } + else drv; + # Disable building a Python package's documentation. - dontBuildDocs = alsoDisable: drv: (drv.override ({ - sphinxHook = null; - } // alsoDisable)).overrideAttrs ({ outputs, ... }: { + dontBuildDocs = drv: ( + overrideIfPresent "sphinxHook" null ( + overrideIfPresent "sphinx-rtd-theme" null + drv + ) + ).overrideAttrs ({ outputs, ... }: { outputs = builtins.filter (x: "doc" != x) outputs; }); in { + tahoe-lafs = self.callPackage ./tahoe-lafs.nix { + # Define the location of the Tahoe-LAFS source to be packaged (the same + # directory as contains this file). Clean up as many of the non-source + # files (eg the `.git` directory, `~` backup files, nix's own `result` + # symlink, etc) as possible to avoid needing to re-build when files that + # make no difference to the package have changed. + tahoe-lafs-src = self.lib.cleanSource ../.; + }; + # Some dependencies aren't packaged in nixpkgs so supply our own packages. pycddl = self.callPackage ./pycddl.nix { }; txi2p = self.callPackage ./txi2p.nix { }; @@ -33,11 +58,10 @@ in { # Update the version of pyopenssl. pyopenssl = self.callPackage ./pyopenssl.nix { pyopenssl = - # Building the docs requires sphinx which brings in a dependency on babel, - # the test suite of which fails. - onPyPy (dontBuildDocs { sphinx-rtd-theme = null; }) - # Avoid infinite recursion. - super.pyopenssl; + # Building the docs requires sphinx which brings in a dependency on + # babel, the test suite of which fails. Avoid infinite recursion here + # by taking pyopenssl from the `super` package set. + onPyPy dontBuildDocs super.pyopenssl; }; # collections-extended is currently broken for Python 3.11 in nixpkgs but @@ -74,7 +98,7 @@ in { six = onPyPy dontCheck super.six; # Likewise for beautifulsoup4. - beautifulsoup4 = onPyPy (dontBuildDocs {}) super.beautifulsoup4; + beautifulsoup4 = onPyPy dontBuildDocs super.beautifulsoup4; # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index bf3ea83d3..f7037e1ae 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -1,24 +1,16 @@ +let + pname = "tahoe-lafs"; + version = "1.18.0.post1"; +in { lib , pythonPackages , buildPythonPackage , tahoe-lafs-src -, extrasNames - -# control how the test suite is run -, doCheck }: -let - pname = "tahoe-lafs"; - version = "1.18.0.post1"; - - pickExtraDependencies = deps: extras: builtins.foldl' (accum: extra: accum ++ deps.${extra}) [] extras; - - pythonExtraDependencies = with pythonPackages; { - tor = [ txtorcon ]; - i2p = [ txi2p ]; - }; - - pythonPackageDependencies = with pythonPackages; [ +buildPythonPackage rec { + inherit pname version; + src = tahoe-lafs-src; + propagatedBuildInputs = with pythonPackages; [ attrs autobahn cbor2 @@ -41,35 +33,37 @@ let six treq twisted - # Get the dependencies for the Twisted extras we depend on, too. - twisted.passthru.optional-dependencies.tls - twisted.passthru.optional-dependencies.conch werkzeug zfec zope_interface - ] ++ pickExtraDependencies pythonExtraDependencies extrasNames; + ] ++ + # Get the dependencies for the Twisted extras we depend on, too. + twisted.passthru.optional-dependencies.tls ++ + twisted.passthru.optional-dependencies.conch; - unitTestDependencies = with pythonPackages; [ - beautifulsoup4 - fixtures - hypothesis - mock - prometheus-client - testtools - ]; + # The test suite lives elsewhere. + doCheck = false; -in -buildPythonPackage { - inherit pname version; - src = tahoe-lafs-src; - propagatedBuildInputs = pythonPackageDependencies; - - inherit doCheck; - checkInputs = unitTestDependencies; - checkPhase = '' - export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - python -m twisted.trial -j $NIX_BUILD_CORES allmydata - ''; + passthru = { + extras = with pythonPackages; { + tor = [ txtorcon ]; + i2p = [ txi2p ]; + unittest = [ + beautifulsoup4 + fixtures + hypothesis + mock + prometheus-client + testtools + ]; + integrationtest = [ + pytest + pytest-twisted + paramiko + pytest-timeout + ]; + }; + }; meta = with lib; { homepage = "https://tahoe-lafs.org/"; diff --git a/nix/tests.nix b/nix/tests.nix deleted file mode 100644 index 42ca9f882..000000000 --- a/nix/tests.nix +++ /dev/null @@ -1,4 +0,0 @@ -# Build the package with the test suite enabled. -args@{...}: (import ../. args).override { - doCheck = true; -} From 452ba6af32de98046bcb964a692b2fd59e9a363f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:48:45 -0400 Subject: [PATCH 02/54] try to get ci to use the flake --- .circleci/config.yml | 47 ++++++++++++++++++++++---------------------- .circleci/lib.sh | 29 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 54b2706cd..59864675b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -403,14 +403,17 @@ jobs: - "run": name: "Unit Test" command: | - # The dependencies are all built so we can allow more - # parallelism here. source .circleci/lib.sh - cache_if_able nix-build \ - --cores 8 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - nix/tests.nix + + # Translate the nixpkgs selection into a flake reference we + # can use to override the default nixpkgs input. + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + + cache_if_able nix run \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-unittest -- \ + --jobs $UNITTEST_CORES \ + allmydata typechecks: docker: @@ -588,29 +591,25 @@ commands: - "run": name: "Build Dependencies" command: | - # CircleCI build environment looks like it has a zillion and a - # half cores. Don't let Nix autodetect this high core count - # because it blows up memory usage and fails the test run. Pick a - # number of cores that suits the build environment we're paying - # for (the free one!). source .circleci/lib.sh - # nix-shell will build all of the dependencies of the target but + + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + + # `nix develop` will build all of the dependencies of the target but # not the target itself. - cache_if_able nix-shell \ - --run "" \ - --cores 3 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - ./default.nix + cache_if_able nix develop \ + --command "true" \ + --cores $DEPENDENCY_CORES \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-tahoe-lafs - "run": name: "Build Package" command: | source .circleci/lib.sh - cache_if_able nix-build \ - --cores 4 \ - --argstr pkgsVersion "nixpkgs-<>" \ - --argstr pythonVersion "<>" \ - ./default.nix + NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + cache_if_able nix build \ + --override-input nixpkgs "$NIXPKGS" \ + .#<>-tahoe-lafs - steps: "<>" diff --git a/.circleci/lib.sh b/.circleci/lib.sh index c692b5f88..a53c33dce 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -1,3 +1,13 @@ +# CircleCI build environment looks like it has a zillion and a half cores. +# Don't let Nix autodetect this high core count because it blows up memory +# usage and fails the test run. Pick a number of cores that suits the build +# environment we're paying for (the free one!). +DEPENDENCY_CORES=3 + +# Once dependencies are built, we can allow some more concurrency for our own +# test suite. +UNITTEST_CORES=8 + # Run a command, enabling cache writes to cachix if possible. The command is # accepted as a variable number of positional arguments (like argv). function cache_if_able() { @@ -117,3 +127,22 @@ function describe_build() { echo "Cache not writeable." fi } + +# Inspect the flake input metadata for an input of a given name and return the +# revision at which that input is pinned. If the input does not exist then +# return garbage (probably "null"). +read_input_revision() { + input_name=$1 + shift + + nix flake metadata --json | jp --unquoted 'locks.nodes."'"$input_name"'".locked.rev' +} + +# Return a flake reference that refers to a certain revision of nixpkgs. The +# certain revision is the revision to which the specified input is pinned. +nixpkgs_flake_reference() { + input_name=$1 + shift + + echo "github:NixOS/nixpkgs?rev=$(read_input_revision $input_name)" +} From 2ddb0970a36415cb4b48ba01dfb74bff9b559d71 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 12:50:48 -0400 Subject: [PATCH 03/54] access to "experimental" nix commands --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 59864675b..03af0f8a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -545,6 +545,9 @@ executors: # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push # to. CACHIX_NAME: "tahoe-lafs-opensource" + # Let us use features marked "experimental". For example, most/all of + # the `nix ` forms. + NIX_CONFIG: "experimental-features = nix-command flakes" commands: nix-build: From 95ee3994a944ae959ba8c89bf82a3a3b160a4ccb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:08:00 -0400 Subject: [PATCH 04/54] get a jmespath interpreter --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 03af0f8a1..f86bdfeee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -575,7 +575,7 @@ commands: nix-env \ --file $NIXPKGS \ --install \ - -A cachix bash + -A cachix bash jp # Activate it for "binary substitution". This sets up # configuration tht lets Nix download something from the cache # instead of building it locally, if possible. From 3d62b9d0501586f3b4d53ce69552cc97ee74b744 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:12:28 -0400 Subject: [PATCH 05/54] drop the separate build-dependencies step Since we don't run the unit tests as part of the package build now we don't need it --- .circleci/config.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f86bdfeee..4426b0827 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -591,21 +591,6 @@ commands: -p 'python3.withPackages (ps: [ ps.setuptools ])' \ --run 'python setup.py update_version' - - "run": - name: "Build Dependencies" - command: | - source .circleci/lib.sh - - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) - - # `nix develop` will build all of the dependencies of the target but - # not the target itself. - cache_if_able nix develop \ - --command "true" \ - --cores $DEPENDENCY_CORES \ - --override-input nixpkgs "$NIXPKGS" \ - .#<>-tahoe-lafs - - "run": name: "Build Package" command: | From 84eb1a3e3242fa401dd280c8b368ad31ed2e2f10 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:13:16 -0400 Subject: [PATCH 06/54] still limit cores during build though --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4426b0827..e503679ed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,6 +597,7 @@ commands: source .circleci/lib.sh NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ + --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 4646c5ea38626896cb1cf1a984000145f30e0691 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:22:37 -0400 Subject: [PATCH 07/54] identify the nixpkgs input correctly --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e503679ed..f3965d242 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "22.11" + nixpkgs: "22_11" matrix: parameters: pythonVersion: From f5dd14e0867d3dddbc521c03831853231645addb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:24:33 -0400 Subject: [PATCH 08/54] use a constant nixpkgs for build environment setup --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3965d242..f2d1e7a04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -571,7 +571,7 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-23.05.tar.gz" nix-env \ --file $NIXPKGS \ --install \ From 35254adebf9723d1936296137baf53266a45acc4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:28:24 -0400 Subject: [PATCH 09/54] try to make noise to avoid ci timeouts --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2d1e7a04..1c3c52d17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,6 +597,7 @@ commands: source .circleci/lib.sh NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ + --verbose \ --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 4b932b86991b2cdefdb956f9ad782318bb067314 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:43:18 -0400 Subject: [PATCH 10/54] turn off a lot of autobahn dependencies we don't need --- nix/python-overrides.nix | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index a333808be..c6e25fd80 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -103,7 +103,22 @@ in { # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* # disable just some of it but this is easier. - autobahn = onPyPy dontCheck super.autobahn; + autobahn = onPyPy dontCheck (super.autobahn.override { + base58 = null; + click = null; + ecdsa = null; + eth-abi = null; + jinja2 = null; + hkdf = null; + mnemonic = null; + py-ecc = null; + py-eth-sig-utils = null; + py-multihash = null; + rlp = null; + spake2 = null; + yapf = null; + eth-account = null; + }); # and python-dotenv tests pulls in a lot of dependencies, including jedi, # which does not work on PyPy. From d455e3c88fd32218a48e7ee13fc0c669adbd2ca7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:45:01 -0400 Subject: [PATCH 11/54] get more output from the build step --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1c3c52d17..01cd999ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -598,6 +598,7 @@ commands: NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) cache_if_able nix build \ --verbose \ + --print-build-logs \ --cores "$DEPENDENCY_CORES" \ --override-input nixpkgs "$NIXPKGS" \ .#<>-tahoe-lafs From 8a53175655faf7f6756fe958a9e23f8b16daa455 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 13:50:33 -0400 Subject: [PATCH 12/54] compatibility with newer nixpkgs that includes the fix --- nix/python-overrides.nix | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index c6e25fd80..9cdf63306 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -149,11 +149,10 @@ in { # This also drops a bunch of unnecessary build-time dependencies, some of # which are broken on PyPy. Fixed in nixpkgs in # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. - zfec = (super.zfec.override { - setuptoolsTrial = null; - }).overrideAttrs (old: { - checkPhase = "trial zfec"; - }); + zfec = (overrideIfPresent "setuptoolsTrial" null super.zfec).overrideAttrs ( + old: { + checkPhase = "trial zfec"; + }); # collections-extended is packaged with poetry-core. poetry-core test suite # uses virtualenv and virtualenv test suite fails on PyPy. From bf1db852197d22d307777a0005104d987ca92a48 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:20:50 -0400 Subject: [PATCH 13/54] slightly regularize nixpkgs handling --- .circleci/config.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01cd999ca..21d6a6114 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,7 +89,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "22_11" + nixpkgs: "nixpkgs-22_11" matrix: parameters: pythonVersion: @@ -99,7 +99,7 @@ workflows: - "nixos": name: "<>" - nixpkgs: "unstable" + nixpkgs: "nixpkgs-unstable" matrix: parameters: pythonVersion: @@ -385,8 +385,8 @@ jobs: parameters: nixpkgs: description: >- - Reference the name of a niv-managed nixpkgs source (see `niv show` - and nix/sources.json) + Reference the name of a flake-managed nixpkgs input (see `nix flake + metadata` and flake.nix) type: "string" pythonVersion: description: >- @@ -407,7 +407,7 @@ jobs: # Translate the nixpkgs selection into a flake reference we # can use to override the default nixpkgs input. - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + NIXPKGS=$(nixpkgs_flake_reference <>) cache_if_able nix run \ --override-input nixpkgs "$NIXPKGS" \ @@ -595,7 +595,7 @@ commands: name: "Build Package" command: | source .circleci/lib.sh - NIXPKGS=$(nixpkgs_flake_reference nixpkgs-<>) + NIXPKGS=$(nixpkgs_flake_reference <>) cache_if_able nix build \ --verbose \ --print-build-logs \ From 3871a92890f68bf4e4fbc0a208b143e8093347d9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:36:27 -0400 Subject: [PATCH 14/54] fix another outdated comment --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21d6a6114..d0942d504 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -554,8 +554,8 @@ commands: parameters: nixpkgs: description: >- - Reference the name of a niv-managed nixpkgs source (see `niv show` - and nix/sources.json) + Reference the name of a flake-managed nixpkgs input (see `nix flake + metadata` and flake.nix) type: "string" pythonVersion: description: >- From 9788e4c12f8f10f5ce7376e5db47b1901f073e70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:36:46 -0400 Subject: [PATCH 15/54] fix the test app definition --- flake.nix | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 89a4a6f14..e8c501bd1 100644 --- a/flake.nix +++ b/flake.nix @@ -90,15 +90,17 @@ # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeTestEnv = pyVersion: { - ${packageName pyVersion} = (pkgs.${pyVersion}.withPackages (ps: with ps; - [ tahoe-lafs ] ++ - tahoe-lafs.passthru.extras.i2p ++ - tahoe-lafs.passthru.extras.tor ++ - tahoe-lafs.passthru.extras.unittest - )).overrideAttrs (old: { - name = packageName pyVersion; - }); + ${packageName pyVersion} = makeTestEnv' pyVersion; }; + + makeTestEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + [ tahoe-lafs ] ++ + tahoe-lafs.passthru.extras.i2p ++ + tahoe-lafs.passthru.extras.tor ++ + tahoe-lafs.passthru.extras.unittest + )).overrideAttrs (old: { + name = packageName pyVersion; + }); in { # Define the flake's package outputs. We'll define one version of the # package for each version of Python we could find. We'll also point @@ -146,7 +148,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv' pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From 72539ddfc7adf22bb3c5df256718a1a07277e3fa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 4 Jul 2023 14:39:26 -0400 Subject: [PATCH 16/54] refactor the env builders a bit --- flake.nix | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index e8c501bd1..8a29786c9 100644 --- a/flake.nix +++ b/flake.nix @@ -73,12 +73,14 @@ # Construct the unit test application name for the given Python runtime. unitTestName = pyVersion: "${pyVersion}-unittest"; + # (string -> a) -> (string -> b) -> string -> attrset a b + # + # Make a singleton attribute set from the result of two functions. + singletonOf = f: g: x: { ${f x} = g x; }; + # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. - makeRuntimeEnv = pyVersion: { - ${packageName pyVersion} = makeRuntimeEnv' pyVersion; - }; - + makeRuntimeEnv = singletonOf packageName makeRuntimeEnv'; makeRuntimeEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ @@ -89,11 +91,7 @@ # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. - makeTestEnv = pyVersion: { - ${packageName pyVersion} = makeTestEnv' pyVersion; - }; - - makeTestEnv' = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; + makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor ++ @@ -148,7 +146,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv' pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From 94e608f1363763e03df720e39828c9d281f05981 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 10:02:52 -0400 Subject: [PATCH 17/54] more python package tweaks also point nixpkgs-unstable at HEAD of a PR with a cryptography upgrade I tried just overriding the upgrade into place but it results in infinite recursion, I suppose because cryptography is a dependency of some of the build tools and needs extra handling that I don't feel like figuring out for this short-term hack. someday the upgrade will land in nixpkgs master and we can switch back. --- flake.lock | 26 ++++++++++-- flake.nix | 11 +++++- nix/python-overrides.nix | 85 ++++++++++++---------------------------- nix/service-identity.nix | 61 ++++++++++++++++++++++++++++ nix/tahoe-lafs.nix | 8 +++- nix/twisted.patch | 12 ++++++ 6 files changed, 135 insertions(+), 68 deletions(-) create mode 100644 nix/service-identity.nix create mode 100644 nix/twisted.patch diff --git a/flake.lock b/flake.lock index 0fd3c90c9..165ff7230 100644 --- a/flake.lock +++ b/flake.lock @@ -34,17 +34,34 @@ "type": "github" } }, - "nixpkgs-unstable": { + "nixpkgs-23_05": { "locked": { - "lastModified": 1688486599, - "narHash": "sha256-K8v2wCfHjA0LS6QeCZ/x+OU2hhINZG4qAAO6zvvqYhE=", + "lastModified": 1688722168, + "narHash": "sha256-UDqeQd2neUXICpHAZSS965kGCJsfHkrOFS/vl80I7d8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "712caf8eb1c2ea0944d2f34f96570bca7193c1c8", + "rev": "28d812a63a0f0d6c1170aac16f5219e506c44b79", "type": "github" }, "original": { "owner": "NixOS", + "ref": "release-23.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1687349636, + "narHash": "sha256-wpWWNoKJ6z8Nt9egpeiKzsCgkkDO2SO4g6ab9SxgvpM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "49b7c90c06e557e7473ef467f40d98e7c368d29f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "pull/238965/head", "repo": "nixpkgs", "type": "github" } @@ -56,6 +73,7 @@ "nixpkgs-22_11" ], "nixpkgs-22_11": "nixpkgs-22_11", + "nixpkgs-23_05": "nixpkgs-23_05", "nixpkgs-unstable": "nixpkgs-unstable" } }, diff --git a/flake.nix b/flake.nix index 8a29786c9..83ee24bec 100644 --- a/flake.nix +++ b/flake.nix @@ -9,8 +9,11 @@ "nixpkgs-22_11" = { url = github:NixOS/nixpkgs?ref=nixos-22.11; }; + "nixpkgs-23_05" = { + url = github:NixOS/nixpkgs?ref=release-23.05; + }; "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs; + url = github:NixOS/nixpkgs?ref=pull/238965/head; }; # Point the default nixpkgs at one of those. This avoids having getting a @@ -100,6 +103,8 @@ name = packageName pyVersion; }); in { + legacyPackages = pkgs; + # Define the flake's package outputs. We'll define one version of the # package for each version of Python we could find. We'll also point # the flake's "default" package at one of these somewhat arbitrarily. @@ -108,7 +113,9 @@ packages = with pkgs.lib; foldr mergeAttrs {} ([ { default = self.packages.${system}.${packageName defaultPyVersion}; } - ] ++ (builtins.map makeRuntimeEnv pythonVersions)); + ] ++ (builtins.map makeRuntimeEnv pythonVersions) + ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) + ); # Define the flake's app outputs. We'll define a version of an app for # running the test suite for each version of Python we could find. diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 9cdf63306..de0ae028c 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -55,14 +55,22 @@ in { inherit (super) txtorcon; }; - # Update the version of pyopenssl. - pyopenssl = self.callPackage ./pyopenssl.nix { - pyopenssl = - # Building the docs requires sphinx which brings in a dependency on - # babel, the test suite of which fails. Avoid infinite recursion here - # by taking pyopenssl from the `super` package set. - onPyPy dontBuildDocs super.pyopenssl; - }; + # With our customized package set a Twisted unit test fails. Patch the + # Twisted test suite to skip that test. + twisted = super.twisted.overrideAttrs (old: { + patches = (old.patches or []) ++ [ ./twisted.patch ]; + }); + + # Update the version of pyopenssl - and since we're doing that anyway, we + # don't need the docs. Unfortunately this triggers a lot of rebuilding of + # dependent packages. + pyopenssl = dontBuildDocs (self.callPackage ./pyopenssl.nix { + inherit (super) pyopenssl; + }); + + # The cryptography that we get from nixpkgs to satisfy the pyopenssl upgrade + # that we did breaks service-identity ... so get a newer version that works. + service-identity = self.callPackage ./service-identity.nix { }; # collections-extended is currently broken for Python 3.11 in nixpkgs but # we know where a working version lives. @@ -76,16 +84,19 @@ in { # tornado and tk pull in a huge dependency trees for functionality we don't # care about, also tkinter doesn't work on PyPy. - matplotlib = super.matplotlib.override { tornado = null; enableTk = false; }; + matplotlib = onPyPy (matplotlib: matplotlib.override { + tornado = null; + enableTk = false; + }) super.matplotlib; - tqdm = super.tqdm.override { + tqdm = onPyPy (tqdm: tqdm.override { # ibid. tkinter = null; # pandas is only required by the part of the test suite covering # integration with pandas that we don't care about. pandas is a huge # dependency. pandas = null; - }; + }) super.tqdm; # The treq test suite depends on httpbin. httpbin pulls in babel (flask -> # jinja2 -> babel) and arrow (brotlipy -> construct -> arrow). babel fails @@ -103,57 +114,20 @@ in { # The autobahn test suite pulls in a vast number of dependencies for # functionality we don't care about. It might be nice to *selectively* # disable just some of it but this is easier. - autobahn = onPyPy dontCheck (super.autobahn.override { - base58 = null; - click = null; - ecdsa = null; - eth-abi = null; - jinja2 = null; - hkdf = null; - mnemonic = null; - py-ecc = null; - py-eth-sig-utils = null; - py-multihash = null; - rlp = null; - spake2 = null; - yapf = null; - eth-account = null; - }); + autobahn = dontCheck super.autobahn; # and python-dotenv tests pulls in a lot of dependencies, including jedi, # which does not work on PyPy. python-dotenv = onPyPy dontCheck super.python-dotenv; - # Upstream package unaccountably includes a sqlalchemy dependency ... but - # the project has no such dependency. Fixed in nixpkgs in - # da10e809fff70fbe1d86303b133b779f09f56503. - aiocontextvars = super.aiocontextvars.override { sqlalchemy = null; }; - # By default, the sphinx docs are built, which pulls in a lot of # dependencies - including jedi, which does not work on PyPy. - hypothesis = - (let h = super.hypothesis; - in - if (h.override.__functionArgs.enableDocumentation or false) - then h.override { enableDocumentation = false; } - else h).overrideAttrs ({ nativeBuildInputs, ... }: { - # The nixpkgs expression is missing the tzdata check input. - nativeBuildInputs = nativeBuildInputs ++ [ super.tzdata ]; - }); + hypothesis = onPyPy dontBuildDocs super.hypothesis; # flaky's test suite depends on nose and nose appears to have Python 3 # incompatibilities (it includes `print` statements, for example). flaky = onPyPy dontCheck super.flaky; - # Replace the deprecated way of running the test suite with the modern way. - # This also drops a bunch of unnecessary build-time dependencies, some of - # which are broken on PyPy. Fixed in nixpkgs in - # 5feb5054bb08ba779bd2560a44cf7d18ddf37fea. - zfec = (overrideIfPresent "setuptoolsTrial" null super.zfec).overrideAttrs ( - old: { - checkPhase = "trial zfec"; - }); - # collections-extended is packaged with poetry-core. poetry-core test suite # uses virtualenv and virtualenv test suite fails on PyPy. poetry-core = onPyPy dontCheck super.poetry-core; @@ -172,15 +146,6 @@ in { # since we actually depend directly and significantly on Foolscap. foolscap = onPyPy dontCheck super.foolscap; - # Fixed by nixpkgs PR https://github.com/NixOS/nixpkgs/pull/222246 - psutil = super.psutil.overrideAttrs ({ pytestFlagsArray, disabledTests, ...}: { - # Upstream already disables some tests but there are even more that have - # build impurities that come from build system hardware configuration. - # Skip them too. - pytestFlagsArray = [ "-v" ] ++ pytestFlagsArray; - disabledTests = disabledTests ++ [ "sensors_temperatures" ]; - }); - # CircleCI build systems don't have enough memory to run this test suite. - lz4 = dontCheck super.lz4; + lz4 = onPyPy dontCheck super.lz4; } diff --git a/nix/service-identity.nix b/nix/service-identity.nix new file mode 100644 index 000000000..fef68b16e --- /dev/null +++ b/nix/service-identity.nix @@ -0,0 +1,61 @@ +{ lib +, attrs +, buildPythonPackage +, cryptography +, fetchFromGitHub +, hatch-fancy-pypi-readme +, hatch-vcs +, hatchling +, idna +, pyasn1 +, pyasn1-modules +, pytestCheckHook +, pythonOlder +, setuptools +}: + +buildPythonPackage rec { + pname = "service-identity"; + version = "23.1.0"; + format = "pyproject"; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "pyca"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-PGDtsDgRwh7GuuM4OuExiy8L4i3Foo+OD0wMrndPkvo="; + }; + + nativeBuildInputs = [ + hatch-fancy-pypi-readme + hatch-vcs + hatchling + setuptools + ]; + + propagatedBuildInputs = [ + attrs + cryptography + idna + pyasn1 + pyasn1-modules + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ + "service_identity" + ]; + + meta = with lib; { + description = "Service identity verification for pyOpenSSL"; + homepage = "https://service-identity.readthedocs.io"; + changelog = "https://github.com/pyca/service-identity/releases/tag/${version}"; + license = licenses.mit; + maintainers = with maintainers; [ fab ]; + }; +} diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index f7037e1ae..5d86de4b2 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -46,8 +46,12 @@ buildPythonPackage rec { passthru = { extras = with pythonPackages; { - tor = [ txtorcon ]; - i2p = [ txi2p ]; + tor = [ + txtorcon + ]; + i2p = [ + txi2p + ]; unittest = [ beautifulsoup4 fixtures diff --git a/nix/twisted.patch b/nix/twisted.patch new file mode 100644 index 000000000..1b6846c8e --- /dev/null +++ b/nix/twisted.patch @@ -0,0 +1,12 @@ +diff --git a/src/twisted/internet/test/test_endpoints.py b/src/twisted/internet/test/test_endpoints.py +index c650fd8aa6..a1754fd533 100644 +--- a/src/twisted/internet/test/test_endpoints.py ++++ b/src/twisted/internet/test/test_endpoints.py +@@ -4214,6 +4214,7 @@ class WrapClientTLSParserTests(unittest.TestCase): + connectionCreator = connectionCreatorFromEndpoint(reactor, endpoint) + self.assertEqual(connectionCreator._hostname, "\xe9xample.example.com") + ++ @skipIf(True, "self.assertFalse(plainClient.transport.disconnecting) fails") + def test_tls(self): + """ + When passed a string endpoint description beginning with C{tls:}, From ddf4777153262a0fe4ccc27b5e992c744572379a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 10:37:05 -0400 Subject: [PATCH 18/54] link to the ticket --- nix/python-overrides.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index de0ae028c..74a74b278 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -57,6 +57,7 @@ in { # With our customized package set a Twisted unit test fails. Patch the # Twisted test suite to skip that test. + # Filed upstream at https://github.com/twisted/twisted/issues/11892 twisted = super.twisted.overrideAttrs (old: { patches = (old.patches or []) ++ [ ./twisted.patch ]; }); From 2271ca4698a297c9b2edf4a3f3b1e54b8e3feb0f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 11:47:04 -0400 Subject: [PATCH 19/54] add missing test dependency --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 5d86de4b2..dbc4f37f9 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -54,6 +54,7 @@ buildPythonPackage rec { ]; unittest = [ beautifulsoup4 + html5lib fixtures hypothesis mock From f7dd63407f3c58e64ea47b394414d224fb22d65d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 12:17:39 -0400 Subject: [PATCH 20/54] move all of the Python jobs to the nixpkgs unstable branch This is the only place we can get a compatible pyOpenSSL/cryptography at the moment --- .circleci/config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0942d504..e588cfa88 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -89,20 +89,13 @@ workflows: - "nixos": name: "<>" - nixpkgs: "nixpkgs-22_11" + nixpkgs: "nixpkgs-unstable" matrix: parameters: pythonVersion: - "python38" - "python39" - "python310" - - - "nixos": - name: "<>" - nixpkgs: "nixpkgs-unstable" - matrix: - parameters: - pythonVersion: - "python311" # Eventually, test against PyPy 3.8 From 0eb160f42cb7b433f3a2ea17006d6f93414249a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 14:49:36 -0400 Subject: [PATCH 21/54] switch to the working version of nixpkgs by default --- flake.lock | 2 +- flake.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 165ff7230..17270a7d0 100644 --- a/flake.lock +++ b/flake.lock @@ -70,7 +70,7 @@ "inputs": { "flake-utils": "flake-utils", "nixpkgs": [ - "nixpkgs-22_11" + "nixpkgs-unstable" ], "nixpkgs-22_11": "nixpkgs-22_11", "nixpkgs-23_05": "nixpkgs-23_05", diff --git a/flake.nix b/flake.nix index 83ee24bec..48dbfe7eb 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ # _third_ package set involved and gives a way to provide what should be a # working experience by default (that is, if nixpkgs doesn't get # overridden). - nixpkgs.follows = "nixpkgs-22_11"; + nixpkgs.follows = "nixpkgs-unstable"; # Also get flake-utils for simplified multi-system definitions. flake-utils = { From fd056026086f265236d3397c7fa9d4c27144e796 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Jul 2023 16:00:00 -0400 Subject: [PATCH 22/54] Reduce the amount of test suite gymnastics with new WebishServer API Instead of forcing the test suite to try to discover the location of an unnamed temporary file, let it just assert that the file is created in the directory specified in the temporary file factory. --- src/allmydata/client.py | 7 ++- src/allmydata/test/web/test_web.py | 2 +- src/allmydata/test/web/test_webish.py | 89 +++++++++------------------ src/allmydata/webish.py | 40 ++++++++---- 4 files changed, 61 insertions(+), 77 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e85ed4fe2..fefd29657 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1,5 +1,5 @@ """ -Ported to Python 3. +Functionality related to operating a Tahoe-LAFS node (client _or_ server). """ from __future__ import annotations @@ -7,6 +7,7 @@ import os import stat import time import weakref +import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial @@ -1029,14 +1030,14 @@ class _Client(node.Node, pollmixin.PollMixin): def init_web(self, webport): self.log("init_web(webport=%s)", args=(webport,)) - from allmydata.webish import WebishServer + from allmydata.webish import WebishServer, anonymous_tempfile nodeurl_path = self.config.get_config_path("node.url") staticdir_config = self.config.get_config("node", "web.static", "public_html") staticdir = self.config.get_config_path(staticdir_config) ws = WebishServer( self, webport, - self._get_tempdir(), + anonymous_tempfile(self._get_tempdir()), nodeurl_path, staticdir, ) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 08dce0ac0..bd7ca3f7f 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -341,7 +341,7 @@ class WebMixin(TimezoneMixin): self.ws = webish.WebishServer( self.s, "0", - tempdir=tempdir.path, + webish.anonymous_tempfile(tempdir.path), staticdir=self.staticdir, clock=self.clock, now_fn=lambda:self.fakeTime, diff --git a/src/allmydata/test/web/test_webish.py b/src/allmydata/test/web/test_webish.py index 050f77d1c..d444df436 100644 --- a/src/allmydata/test/web/test_webish.py +++ b/src/allmydata/test/web/test_webish.py @@ -1,17 +1,8 @@ """ Tests for ``allmydata.webish``. - -Ported to Python 3. """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import tempfile from uuid import ( uuid4, ) @@ -59,6 +50,7 @@ from ..common import ( from ...webish import ( TahoeLAFSRequest, TahoeLAFSSite, + anonymous_tempfile, ) @@ -183,8 +175,14 @@ class TahoeLAFSSiteTests(SyncTestCase): :return: ``None`` if the logging looks good. """ logPath = self.mktemp() + tempdir = self.mktemp() + FilePath(tempdir).makedirs() - site = TahoeLAFSSite(self.mktemp(), Resource(), logPath=logPath) + site = TahoeLAFSSite( + anonymous_tempfile(tempdir), + Resource(), + logPath=logPath, + ) site.startFactory() channel = DummyChannel() @@ -257,11 +255,17 @@ class TahoeLAFSSiteTests(SyncTestCase): Create and return a new ``TahoeLAFSRequest`` hooked up to a ``TahoeLAFSSite``. - :param bytes tempdir: The temporary directory to give to the site. + :param FilePath tempdir: The temporary directory to configure the site + to write large temporary request bodies to. The temporary files + will be named for ease of testing. :return TahoeLAFSRequest: The new request instance. """ - site = TahoeLAFSSite(tempdir.path, Resource(), logPath=self.mktemp()) + site = TahoeLAFSSite( + lambda: tempfile.NamedTemporaryFile(dir=tempdir.path), + Resource(), + logPath=self.mktemp(), + ) site.startFactory() channel = DummyChannel() @@ -275,6 +279,7 @@ class TahoeLAFSSiteTests(SyncTestCase): A request body smaller than 1 MiB is kept in memory. """ tempdir = FilePath(self.mktemp()) + tempdir.makedirs() request = self._create_request(tempdir) request.gotLength(request_body_size) self.assertThat( @@ -284,57 +289,21 @@ class TahoeLAFSSiteTests(SyncTestCase): def _large_request_test(self, request_body_size): """ - Assert that when a request with a body of of the given size is received - its content is written to the directory the ``TahoeLAFSSite`` is - configured with. + Assert that when a request with a body of the given size is + received its content is written a temporary file created by the given + tempfile factory. """ tempdir = FilePath(self.mktemp()) tempdir.makedirs() request = self._create_request(tempdir) - - # So. Bad news. The temporary file for the uploaded content is - # unnamed (and this isn't even necessarily a bad thing since it is how - # you get automatic on-process-exit cleanup behavior on POSIX). It's - # not visible by inspecting the filesystem. It has no name we can - # discover. Then how do we verify it is written to the right place? - # The question itself is meaningless if we try to be too precise. It - # *has* no filesystem location. However, it is still stored *on* some - # filesystem. We still want to make sure it is on the filesystem we - # specified because otherwise it might be on a filesystem that's too - # small or undesirable in some other way. - # - # I don't know of any way to ask a file descriptor which filesystem - # it's on, either, though. It might be the case that the [f]statvfs() - # result could be compared somehow to infer the filesystem but - # ... it's not clear what the failure modes might be there, across - # different filesystems and runtime environments. - # - # Another approach is to make the temp directory unwriteable and - # observe the failure when an attempt is made to create a file there. - # This is hardly a lovely solution but at least it's kind of simple. - # - # It would be nice if it worked consistently cross-platform but on - # Windows os.chmod is more or less broken. - if platform.isWindows(): - request.gotLength(request_body_size) - self.assertThat( - tempdir.children(), - HasLength(1), - ) - else: - tempdir.chmod(0o550) - with self.assertRaises(OSError) as ctx: - request.gotLength(request_body_size) - raise Exception( - "OSError not raised, instead tempdir.children() = {}".format( - tempdir.children(), - ), - ) - - self.assertThat( - ctx.exception.errno, - Equals(EACCES), - ) + request.gotLength(request_body_size) + # We can see the temporary file in the temporary directory we + # specified because _create_request makes a request that uses named + # temporary files instead of the usual anonymous temporary files. + self.assertThat( + tempdir.children(), + HasLength(1), + ) def test_unknown_request_size(self): """ diff --git a/src/allmydata/webish.py b/src/allmydata/webish.py index ec2582f80..2bae7f8a5 100644 --- a/src/allmydata/webish.py +++ b/src/allmydata/webish.py @@ -4,7 +4,7 @@ General web server-related utilities. from __future__ import annotations from six import ensure_str - +from typing import BinaryIO, Callable, Optional import re, time, tempfile from urllib.parse import parse_qsl, urlencode @@ -217,36 +217,50 @@ def censor(queryargs: bytes) -> bytes: return urlencode(result, safe="[]").encode("ascii") +def anonymous_tempfile(tempdir: bytes) -> BinaryIO: + """ + Create a no-argument callable for creating a new temporary file in the + given directory. + + :param tempdir: The directory in which temporary files with be created. + + :return: The callable. + """ + return lambda: tempfile.TemporaryFile(dir=tempdir) + + class TahoeLAFSSite(Site, object): """ The HTTP protocol factory used by Tahoe-LAFS. Among the behaviors provided: - * A configurable temporary directory where large request bodies can be - written so they don't stay in memory. + * A configurable temporary file factory for large request bodies to avoid + keeping them in memory. * A log formatter that writes some access logs but omits capability strings to help keep them secret. """ requestFactory = TahoeLAFSRequest - def __init__(self, tempdir, *args, **kwargs): + def __init__(self, make_tempfile: Callable[[], BinaryIO], *args, **kwargs): Site.__init__(self, *args, logFormatter=_logFormatter, **kwargs) - self._tempdir = tempdir + assert callable(make_tempfile) + with make_tempfile(): + pass + self._make_tempfile = make_tempfile - def getContentFile(self, length): + def getContentFile(self, length: Optional[int]) -> BinaryIO: if length is None or length >= 1024 * 1024: - return tempfile.TemporaryFile(dir=self._tempdir) + return self._make_tempfile() return BytesIO() - class WebishServer(service.MultiService): # The type in Twisted for services is wrong in 22.10... # https://github.com/twisted/twisted/issues/10135 name = "webish" # type: ignore[assignment] - def __init__(self, client, webport, tempdir, nodeurl_path=None, staticdir=None, + def __init__(self, client, webport, make_tempfile, nodeurl_path=None, staticdir=None, clock=None, now_fn=time.time): service.MultiService.__init__(self) # the 'data' argument to all render() methods default to the Client @@ -256,7 +270,7 @@ class WebishServer(service.MultiService): # time in a deterministic manner. self.root = root.Root(client, clock, now_fn) - self.buildServer(webport, tempdir, nodeurl_path, staticdir) + self.buildServer(webport, make_tempfile, nodeurl_path, staticdir) # If set, clock is a twisted.internet.task.Clock that the tests # use to test ophandle expiration. @@ -266,9 +280,9 @@ class WebishServer(service.MultiService): self.root.putChild(b"storage-plugins", StoragePlugins(client)) - def buildServer(self, webport, tempdir, nodeurl_path, staticdir): + def buildServer(self, webport, make_tempfile, nodeurl_path, staticdir): self.webport = webport - self.site = TahoeLAFSSite(tempdir, self.root) + self.site = TahoeLAFSSite(make_tempfile, self.root) self.staticdir = staticdir # so tests can check if staticdir: self.root.putChild(b"static", static.File(staticdir)) @@ -346,4 +360,4 @@ class IntroducerWebishServer(WebishServer): def __init__(self, introducer, webport, nodeurl_path=None, staticdir=None): service.MultiService.__init__(self) self.root = introweb.IntroducerRoot(introducer) - self.buildServer(webport, tempfile.tempdir, nodeurl_path, staticdir) + self.buildServer(webport, tempfile.TemporaryFile, nodeurl_path, staticdir) From e3e223c8d6863ff4ab8c2cdfb1243a823b5d23d9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:08:47 -0400 Subject: [PATCH 23/54] bump to a newer nixos image --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e588cfa88..6b19bdfd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -532,7 +532,7 @@ executors: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.10.3" + image: "nixos/nix:2.16.1" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 6710d625a2ea41506b8325dd55faa2368f182a44 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:08:54 -0400 Subject: [PATCH 24/54] can we just run as an unprivileged user? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b19bdfd5..bbb10d478 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,6 +533,7 @@ executors: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.16.1" + user: "nobody" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 454ab223d1407f4076c4c0ae8e7509fb3e0ebb42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:10:52 -0400 Subject: [PATCH 25/54] nope, we can't just do that we lose permission to install stuff --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index bbb10d478..6b19bdfd5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,7 +533,6 @@ executors: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH image: "nixos/nix:2.16.1" - user: "nobody" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us # to push to CACHIX_NAME. CACHIX_NAME tells cachix which cache to push From 5553019c4ec6c99eeec7c100b37f1da689fd7ea0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Jul 2023 08:14:27 -0400 Subject: [PATCH 26/54] switch to new `nix profile`-based installation --- .circleci/config.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b19bdfd5..d8a70b2c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,15 +564,12 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-23.05.tar.gz" - nix-env \ - --file $NIXPKGS \ - --install \ - -A cachix bash jp - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" + NIXPKGS="nixpkgs/nixos-23.05" + nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" - "checkout" From 93f2a7a717f7025ad2f8895221b49a3837745732 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 13:51:47 -0400 Subject: [PATCH 27/54] refer to non-duplicate ticket --- nix/python-overrides.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/python-overrides.nix b/nix/python-overrides.nix index 74a74b278..006c2682d 100644 --- a/nix/python-overrides.nix +++ b/nix/python-overrides.nix @@ -57,7 +57,7 @@ in { # With our customized package set a Twisted unit test fails. Patch the # Twisted test suite to skip that test. - # Filed upstream at https://github.com/twisted/twisted/issues/11892 + # Filed upstream at https://github.com/twisted/twisted/issues/11877 twisted = super.twisted.overrideAttrs (old: { patches = (old.patches or []) ++ [ ./twisted.patch ]; }); From b0397d3d089bf815fb2f5345fe77ff6feb16f8f5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 13:57:55 -0400 Subject: [PATCH 28/54] Replace default.nix with a compatibility shim This also means we drop our niv dependency --- default.nix | 40 ++++------- flake.lock | 17 +++++ flake.nix | 6 ++ nix/sources.json | 38 ----------- nix/sources.nix | 174 ----------------------------------------------- 5 files changed, 35 insertions(+), 240 deletions(-) delete mode 100644 nix/sources.json delete mode 100644 nix/sources.nix diff --git a/default.nix b/default.nix index 196db4030..1b9368b5f 100644 --- a/default.nix +++ b/default.nix @@ -1,28 +1,12 @@ -let - # sources.nix contains information about which versions of some of our - # dependencies we should use. since we use it to pin nixpkgs, all the rest - # of our dependencies are *also* pinned - indirectly. - # - # sources.nix is managed using a tool called `niv`. as an example, to - # update to the most recent version of nixpkgs from the 21.11 maintenance - # release, in the top-level tahoe-lafs checkout directory you run: - # - # niv update nixpkgs-21.11 - # - # niv also supports chosing a specific revision, following a different - # branch, etc. find complete documentation for the tool at - # https://github.com/nmattia/niv - sources = import nix/sources.nix; -in -{ - pkgsVersion ? "nixpkgs-22.11" # a string which chooses a nixpkgs from the - # niv-managed sources data - -, pkgs ? import sources.${pkgsVersion} { # nixpkgs itself - overlays = [ (import ./nix/overlay.nix) ]; -} - -, pythonVersion ? "python310" # a string choosing the python derivation from - # nixpkgs to target -}: -pkgs.${pythonVersion}.withPackages (ps: [ ps.tahoe-lafs ]) +# This is the flake-compat glue code. It loads the flake and gives us its +# outputs. +(import + ( + let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in + fetchTarball { + url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz"; + sha256 = lock.nodes.flake-compat.locked.narHash; + } + ) + { src = ./.; } +).defaultNix.default diff --git a/flake.lock b/flake.lock index 17270a7d0..4d3764e4d 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,21 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -68,6 +84,7 @@ }, "root": { "inputs": { + "flake-compat": "flake-compat", "flake-utils": "flake-utils", "nixpkgs": [ "nixpkgs-unstable" diff --git a/flake.nix b/flake.nix index 48dbfe7eb..84a31da9c 100644 --- a/flake.nix +++ b/flake.nix @@ -26,6 +26,12 @@ flake-utils = { url = github:numtide/flake-utils; }; + + # And get a helper that lets us easily continue to provide a default.nix. + flake-compat = { + url = "github:edolstra/flake-compat"; + flake = false; + }; }; outputs = { self, nixpkgs, flake-utils, ... }: diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index ddf05d39d..000000000 --- a/nix/sources.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "niv": { - "branch": "master", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "nmattia", - "repo": "niv", - "rev": "5830a4dd348d77e39a0f3c4c762ff2663b602d4c", - "sha256": "1d3lsrqvci4qz2hwjrcnd8h5vfkg8aypq3sjd4g3izbc8frwz5sm", - "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/5830a4dd348d77e39a0f3c4c762ff2663b602d4c.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs-22.11": { - "branch": "nixos-22.11", - "description": "Nix Packages collection", - "homepage": "", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "970402e6147c49603f4d06defe44d27fe51884ce", - "sha256": "1v0ljy7wqq14ad3gd1871fgvd4psr7dy14q724k0wwgxk7inbbwh", - "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/970402e6147c49603f4d06defe44d27fe51884ce.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs-unstable": { - "branch": "master", - "description": "Nix Packages collection", - "homepage": "", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "d0c9a536331227ab883b4f6964be638fa436d81f", - "sha256": "1gg6v5rk1p26ciygdg262zc5vqws753rvgcma5rim2s6gyfrjaq1", - "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/d0c9a536331227ab883b4f6964be638fa436d81f.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - } -} diff --git a/nix/sources.nix b/nix/sources.nix deleted file mode 100644 index 1938409dd..000000000 --- a/nix/sources.nix +++ /dev/null @@ -1,174 +0,0 @@ -# This file has been generated by Niv. - -let - - # - # The fetchers. fetch_ fetches specs of type . - # - - fetch_file = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; name = name'; } - else - pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; - - fetch_tarball = pkgs: name: spec: - let - name' = sanitizeName name + "-src"; - in - if spec.builtin or true then - builtins_fetchTarball { name = name'; inherit (spec) url sha256; } - else - pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; - - fetch_git = name: spec: - let - ref = - if spec ? ref then spec.ref else - if spec ? branch then "refs/heads/${spec.branch}" else - if spec ? tag then "refs/tags/${spec.tag}" else - abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; - in - builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; }; - - fetch_local = spec: spec.path; - - fetch_builtin-tarball = name: throw - ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=tarball -a builtin=true''; - - fetch_builtin-url = name: throw - ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. - $ niv modify ${name} -a type=file -a builtin=true''; - - # - # Various helpers - # - - # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 - sanitizeName = name: - ( - concatMapStrings (s: if builtins.isList s then "-" else s) - ( - builtins.split "[^[:alnum:]+._?=-]+" - ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) - ) - ); - - # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: system: - let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; - hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; - hasThisAsNixpkgsPath = == ./.; - in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then - import {} - else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - # The actual fetching function. - fetch = pkgs: name: spec: - - if ! builtins.hasAttr "type" spec then - abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs name spec - else if spec.type == "tarball" then fetch_tarball pkgs name spec - else if spec.type == "git" then fetch_git name spec - else if spec.type == "local" then fetch_local spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball name - else if spec.type == "builtin-url" then fetch_builtin-url name - else - abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; - - # If the environment variable NIV_OVERRIDE_${name} is set, then use - # the path directly as opposed to the fetched source. - replace = name: drv: - let - saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; - ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; - in - if ersatz == "" then drv else - # this turns the string into an actual Nix path (for both absolute and - # relative paths) - if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; - - # Ports of functions for older nix versions - - # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 - range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 - stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); - - # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 - stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); - concatMapStrings = f: list: concatStrings (map f list); - concatStrings = builtins.concatStringsSep ""; - - # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 - optionalAttrs = cond: as: if cond then as else {}; - - # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if lessThan nixVersion "1.12" then - fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchTarball attrs; - - # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, name ? null, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if lessThan nixVersion "1.12" then - fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) - else - fetchurl attrs; - - # Create the final "sources" from the config - mkSources = config: - mapAttrs ( - name: spec: - if builtins.hasAttr "outPath" spec - then abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = replace name (fetch config.pkgs name spec); } - ) config.sources; - - # The "config" used by the fetchers - mkConfig = - { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null - , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) - , system ? builtins.currentSystem - , pkgs ? mkPkgs sources system - }: rec { - # The sources, i.e. the attribute set of spec name to spec - inherit sources; - - # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers - inherit pkgs; - }; - -in -mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } From d85f8d7caff9f942fc71a463621ed534bd94f040 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 14:27:28 -0400 Subject: [PATCH 29/54] some more comments on the flake parts --- flake.nix | 79 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 24 deletions(-) diff --git a/flake.nix b/flake.nix index 84a31da9c..a68600c72 100644 --- a/flake.nix +++ b/flake.nix @@ -46,33 +46,50 @@ } // (flake-utils.lib.eachDefaultSystem (system: let - # First get the package set for this system architecture. + # The package set for this system architecture. pkgs = import nixpkgs { inherit system; # And include our Tahoe-LAFS package in that package set. overlays = [ self.overlays.default ]; }; - # Find out what Python versions we're working with. - pythonVersions = builtins.attrNames ( - pkgs.lib.attrsets.filterAttrs + # pythonVersions :: [string] + # + # The version strings for the Python runtimes we'll work with. + pythonVersions = + let # Match attribute names that look like a Python derivation - CPython # or PyPy. We take care to avoid things like "python-foo" and # "python3Full-unittest" though. We only want things like "pypy38" # or "python311". - (name: _: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name) - pkgs - ); + nameMatches = name: null != builtins.match "(python|pypy)3[[:digit:]]{0,2}" name; + # Sometimes an old version is left in the package set as an error + # saying something like "we remove this". Make sure we whatever we + # found by name evaluates without error, too. + notError = drv: (builtins.tryEval drv).success; + in + # Discover all of the Python runtime derivations by inspecting names + # and filtering out derivations with errors. + builtins.attrNames ( + pkgs.lib.attrsets.filterAttrs + (name: drv: nameMatches name && notError drv) + pkgs + ); + + # defaultPyVersion :: string + # # An element of pythonVersions which we'll use for the default package. - defaultPyVersion = "python310"; + defaultPyVersion = "python3"; + # pythons :: [derivation] + # # Retrieve the actual Python package for each configured version. We # already applied our overlay to pkgs so our packages will already be # available. pythons = builtins.map (pyVer: pkgs.${pyVer}) pythonVersions; - # string -> string + # packageName :: string -> string # # Construct the Tahoe-LAFS package name for the given Python runtime. packageName = pyVersion: "${pyVersion}-tahoe-lafs"; @@ -87,6 +104,8 @@ # Make a singleton attribute set from the result of two functions. singletonOf = f: g: x: { ${f x} = g x; }; + # makeRuntimeEnv :: string -> derivation + # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeRuntimeEnv = singletonOf packageName makeRuntimeEnv'; @@ -98,6 +117,8 @@ name = packageName pyVersion; }); + # makeTestEnv :: string -> derivation + # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and # all of its dependencies. makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; @@ -109,13 +130,14 @@ name = packageName pyVersion; }); in { + # A package set with out overlay on it. legacyPackages = pkgs; - # Define the flake's package outputs. We'll define one version of the - # package for each version of Python we could find. We'll also point - # the flake's "default" package at one of these somewhat arbitrarily. - # The package consists of a Python environment with Tahoe-LAFS available - # to it. + # The flake's package outputs. We'll define one version of the package + # for each version of Python we could find. We'll also point the + # flake's "default" package at the derivation corresponding to the + # default Python version we defined above. The package consists of a + # Python environment with Tahoe-LAFS available to it. packages = with pkgs.lib; foldr mergeAttrs {} ([ { default = self.packages.${system}.${packageName defaultPyVersion}; } @@ -123,22 +145,28 @@ ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) ); - # Define the flake's app outputs. We'll define a version of an app for - # running the test suite for each version of Python we could find. - # We'll also define a version of an app for running the "tahoe" - # command-line entrypoint for each version of Python we could find. + # The flake's app outputs. We'll define a version of an app for running + # the test suite for each version of Python we could find. We'll also + # define a version of an app for running the "tahoe" command-line + # entrypoint for each version of Python we could find. apps = let + # writeScript :: string -> string -> path + # + # Write a shell program to a file so it can be run later. + # # We avoid writeShellApplication here because it has ghc as a # dependency but ghc has Python as a dependency and our Python - # package override triggers a rebuild of ghc which takes a looong - # time. + # package override triggers a rebuild of ghc and many Haskell + # packages which takes a looong time. writeScript = name: text: let script = pkgs.writeShellScript name text; in "${script}"; - # A helper function to define the runtime entrypoint for a certain - # Python runtime. + # makeTahoeApp :: string -> attrset + # + # A helper function to define the Tahoe-LAFS runtime entrypoint for + # a certain Python runtime. makeTahoeApp = pyVersion: { "tahoe-${pyVersion}" = { type = "app"; @@ -150,8 +178,10 @@ }; }; - # A helper function to define the unit test entrypoint for a certain - # Python runtime. + # makeUnitTestsApp :: string -> attrset + # + # A helper function to define the Tahoe-LAFS unit test entrypoint + # for a certain Python runtime. makeUnitTestsApp = pyVersion: { "${unitTestName pyVersion}" = { type = "app"; @@ -165,6 +195,7 @@ }; in with pkgs.lib; + # Merge a default app definition with the rest of the apps. foldr mergeAttrs { default = self.apps.${system}."tahoe-python3"; } ( From ddfd95faff51cd121a3688e6d4e98e0ff559976d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 14:33:49 -0400 Subject: [PATCH 30/54] comment tweak --- .circleci/config.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8a70b2c4..ae7288509 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -564,10 +564,15 @@ commands: # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | + # Get some build environment dependencies and let them float on a + # certain release branch. These aren't involved in the actual + # package build (only in CI environment setup) so the fact that + # they float shouldn't hurt reproducibility. NIXPKGS="nixpkgs/nixos-23.05" nix profile install $NIXPKGS#cachix $NIXPKGS#bash $NIXPKGS#jp - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache + + # Activate our cachix cache for "binary substitution". This sets + # up configuration tht lets Nix download something from the cache # instead of building it locally, if possible. cachix use "${CACHIX_NAME}" From a3f50aa481b4c2187b0327e1998572d593800b7b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 17:29:50 -0400 Subject: [PATCH 31/54] bump to the newer nixpkgs branch --- flake.lock | 8 ++++---- flake.nix | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 4d3764e4d..2b955d4d6 100644 --- a/flake.lock +++ b/flake.lock @@ -68,16 +68,16 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1687349636, - "narHash": "sha256-wpWWNoKJ6z8Nt9egpeiKzsCgkkDO2SO4g6ab9SxgvpM=", + "lastModified": 1689791806, + "narHash": "sha256-QpXjfiyBFwa7MV/J6nM5FoBreks9O7j9cAZxV22MR8A=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "49b7c90c06e557e7473ef467f40d98e7c368d29f", + "rev": "439ba0789ff84dddea64eb2d47a4a0d4887dbb1f", "type": "github" }, "original": { "owner": "NixOS", - "ref": "pull/238965/head", + "ref": "pull/244135/head", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a68600c72..0dfa3d03a 100644 --- a/flake.nix +++ b/flake.nix @@ -12,8 +12,12 @@ "nixpkgs-23_05" = { url = github:NixOS/nixpkgs?ref=release-23.05; }; + + # We depend on a very new python-cryptography which is not yet available + # from any release branch of nixpkgs. However, it is contained in a PR + # currently up for review. Point our nixpkgs at that for now. "nixpkgs-unstable" = { - url = github:NixOS/nixpkgs?ref=pull/238965/head; + url = github:NixOS/nixpkgs?ref=pull/244135/head; }; # Point the default nixpkgs at one of those. This avoids having getting a From 44502b8620e8cf4d04bf6851a844cac6119cac13 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 19 Jul 2023 18:09:10 -0400 Subject: [PATCH 32/54] remove unused import --- src/allmydata/client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 13909d5ca..a1501f1ef 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -7,7 +7,6 @@ import os import stat import time import weakref -import tempfile from typing import Optional, Iterable from base64 import urlsafe_b64encode from functools import partial From 08e364bbab2dad609d14e5a4c83e9d24d30baf32 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 09:06:05 -0400 Subject: [PATCH 33/54] numpy is not supported on python38 anymore --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae7288509..0c831af04 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,6 @@ workflows: matrix: parameters: pythonVersion: - - "python38" - "python39" - "python310" - "python311" From 90e08314c21877989a3aba934f141a495dd94481 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 10:58:10 -0400 Subject: [PATCH 34/54] try to shed root privileges We have root on CircleCI in the docker container. We can't currently shed them before we get inside the flake app because we can't run `nix build` as non-root inside the nix container. :/ https://github.com/nix-community/docker-nixpkgs/issues/62 --- flake.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 0dfa3d03a..53870eac2 100644 --- a/flake.nix +++ b/flake.nix @@ -193,7 +193,14 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + if [ $(id -u) = "0" ]; then + # The test suite assumes non-root permissions. Get rid + # of the root permissions we seem to have. + SUDO="sudo -u nobody" + else + SUDO="" + fi + $SUDO ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From baadf1fad4fa5078845118890efa030aba813a06 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:12:10 -0400 Subject: [PATCH 35/54] try su to get rid of root sudo fails because it isn't setuid root... I don't know why su would be, but maybe it is. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 53870eac2..329363d6f 100644 --- a/flake.nix +++ b/flake.nix @@ -196,7 +196,7 @@ if [ $(id -u) = "0" ]; then # The test suite assumes non-root permissions. Get rid # of the root permissions we seem to have. - SUDO="sudo -u nobody" + SUDO="${pkgs.su}/bin/su --shell /bin/sh - nobody --" else SUDO="" fi From 9585925627bbd66cc788d7a19b0dd0db425cc39f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:21:08 -0400 Subject: [PATCH 36/54] abandon user-switching effort su fails with "su: pam_start: error 26" --- flake.nix | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 329363d6f..0dfa3d03a 100644 --- a/flake.nix +++ b/flake.nix @@ -193,14 +193,7 @@ writeScript "unit-tests" '' export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - if [ $(id -u) = "0" ]; then - # The test suite assumes non-root permissions. Get rid - # of the root permissions we seem to have. - SUDO="${pkgs.su}/bin/su --shell /bin/sh - nobody --" - else - SUDO="" - fi - $SUDO ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" ''; }; }; From 2091b7ee86fabc5600d2afeb10fca73f9685ed51 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:36:30 -0400 Subject: [PATCH 37/54] skip permission-related tests if the environment is not suitable posix superuser can do anything on the filesystem --- src/allmydata/test/cli/test_grid_manager.py | 3 ++- src/allmydata/test/test_client.py | 25 +++++++------------- src/allmydata/test/test_node.py | 26 ++++++--------------- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 2ed738544..970315af7 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -222,7 +222,8 @@ class GridManagerCommandLine(TestCase): result.output, ) - @skipIf(not platform.isLinux(), "I only know how permissions work on linux") + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 04d8e6160..848f98a13 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -1,17 +1,7 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import os -import sys +from unittest import skipIf from functools import ( partial, ) @@ -42,6 +32,7 @@ from twisted.internet import defer from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from testtools.matchers import ( Equals, AfterPreprocessing, @@ -156,12 +147,12 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): yield client.create_client(basedir) self.assertIn("[client]helper.furl", str(ctx.exception)) + # if somebody knows a clever way to do this (cause + # EnvironmentError when reading a file that really exists), on + # windows, please fix this + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_unreadable_config(self): - if sys.platform == "win32": - # if somebody knows a clever way to do this (cause - # EnvironmentError when reading a file that really exists), on - # windows, please fix this - raise unittest.SkipTest("can't make unreadable files on windows") basedir = "test_client.Basic.test_unreadable_config" os.mkdir(basedir) fn = os.path.join(basedir, "tahoe.cfg") diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 0b371569a..7d6b09d12 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -1,14 +1,4 @@ -""" -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from __future__ import annotations import base64 import os @@ -31,6 +21,7 @@ from unittest import skipIf from twisted.python.filepath import ( FilePath, ) +from twisted.python.runtime import platform from twisted.trial import unittest from twisted.internet import defer @@ -333,10 +324,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): default = [("hello", "world")] self.assertEqual(config.items("nosuch", default), default) - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_private_config_unreadable(self): """ Asking for inaccessible private config is an error @@ -351,10 +340,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): with self.assertRaises(Exception): config.get_or_create_private_config("foo") - @skipIf( - "win32" in sys.platform.lower() or "cygwin" in sys.platform.lower(), - "We don't know how to set permissions on Windows.", - ) + @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_private_config_unreadable_preexisting(self): """ error if reading private config data fails @@ -411,6 +398,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(len(counter), 1) # don't call unless necessary self.assertEqual(value, "newer") + @skipIf(os.getuid() == 0, "cannot test as superuser which all of the permissions") def test_write_config_unwritable_file(self): """ Existing behavior merely logs any errors upon writing From 3e18301f86b00fe07f56c025610c3b5fa11d600f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 11:54:19 -0400 Subject: [PATCH 38/54] Try to get the version right --- flake.nix | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0dfa3d03a..cc3eed5a1 100644 --- a/flake.nix +++ b/flake.nix @@ -190,10 +190,14 @@ "${unitTestName pyVersion}" = { type = "app"; program = - writeScript "unit-tests" - '' + let + py = makeTestEnv pyVersion; + in + writeScript "unit-tests" + '' + ${py} setup.py update_version export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${makeTestEnv pyVersion}/bin/python -m twisted.trial "$@" + ${py}/bin/python -m twisted.trial "$@" ''; }; }; From d82ade538ce4230f888ed2b0e28942a1a8ba11c7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 12:42:26 -0400 Subject: [PATCH 39/54] Use the working tree as the source of allmydata package --- flake.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index cc3eed5a1..17e142184 100644 --- a/flake.nix +++ b/flake.nix @@ -123,8 +123,9 @@ # makeTestEnv :: string -> derivation # - # Create a derivation that includes a Python runtime, Tahoe-LAFS, and - # all of its dependencies. + # Create a derivation that includes a Python runtime and all of the + # Tahoe-LAFS dependencies, but not Tahoe-LAFS itself, which we'll get + # from the working directory. makeTestEnv = pyVersion: (pkgs.${pyVersion}.withPackages (ps: with ps; [ tahoe-lafs ] ++ tahoe-lafs.passthru.extras.i2p ++ @@ -191,13 +192,14 @@ type = "app"; program = let - py = makeTestEnv pyVersion; + python = "${makeTestEnv pyVersion}/bin/python"; in writeScript "unit-tests" '' - ${py} setup.py update_version + ${python} setup.py update_version export TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - ${py}/bin/python -m twisted.trial "$@" + export PYTHONPATH=$PWD/src + ${python} -m twisted.trial "$@" ''; }; }; From 2c40185ef682400d107a721177877e18040e2318 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 20 Jul 2023 14:00:55 -0400 Subject: [PATCH 40/54] slight simplification --- flake.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 17e142184..791015988 100644 --- a/flake.nix +++ b/flake.nix @@ -164,9 +164,7 @@ # dependency but ghc has Python as a dependency and our Python # package override triggers a rebuild of ghc and many Haskell # packages which takes a looong time. - writeScript = name: text: - let script = pkgs.writeShellScript name text; - in "${script}"; + writeScript = name: text: "${pkgs.writeShellScript name text}"; # makeTahoeApp :: string -> attrset # From 10941a02f8a7dbf04681f1e6e2274d70280cb670 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 09:42:00 -0400 Subject: [PATCH 41/54] Go with the successfully-built release branch release-XX.YY is the source branch for NixOS Hydra (CI) runs nixos-XX.YY is updated after a successful release-XX.YY build --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 791015988..2a3290ca2 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ url = github:NixOS/nixpkgs?ref=nixos-22.11; }; "nixpkgs-23_05" = { - url = github:NixOS/nixpkgs?ref=release-23.05; + url = github:NixOS/nixpkgs?ref=nixos-23.05; }; # We depend on a very new python-cryptography which is not yet available From d61029c8bbb47d3e73739d19683e30eab60235fb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 09:54:59 -0400 Subject: [PATCH 42/54] a few more words about the nixpkgs inputs --- flake.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 2a3290ca2..5b3102c46 100644 --- a/flake.nix +++ b/flake.nix @@ -2,10 +2,16 @@ description = "Tahoe-LAFS, free and open decentralized data store"; inputs = { - # Two alternate nixpkgs pins. Ideally these could be selected easily from - # the command line but there seems to be no syntax/support for that. + # A couple possible nixpkgs pins. Ideally these could be selected easily + # from the command line but there seems to be no syntax/support for that. # However, these at least cause certain revisions to be pinned in our lock # file where you *can* dig them out - and the CI configuration does. + # + # These are really just examples for the time being since neither of these + # releases contains a package set that is completely compatible with our + # requirements. We could decide in the future that supporting multiple + # releases of NixOS at a time is worthwhile and then pins like these will + # help us test each of those releases. "nixpkgs-22_11" = { url = github:NixOS/nixpkgs?ref=nixos-22.11; }; From 7e972e4a53182ac64f4f8b80852662344082ba32 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 10:24:09 -0400 Subject: [PATCH 43/54] further clarifying comments --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 1b9368b5f..62dfc8176 100644 --- a/default.nix +++ b/default.nix @@ -1,5 +1,6 @@ # This is the flake-compat glue code. It loads the flake and gives us its -# outputs. +# outputs. This gives us backwards compatibility with pre-flake consumers. +# All of the real action is in flake.nix. (import ( let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in From f4949c699aaca13b29b5a07c6a7f17bff4fdf4aa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:17:26 -0400 Subject: [PATCH 44/54] relock with release/nixos-23.05 change --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 2b955d4d6..b7b74a0e4 100644 --- a/flake.lock +++ b/flake.lock @@ -52,16 +52,16 @@ }, "nixpkgs-23_05": { "locked": { - "lastModified": 1688722168, - "narHash": "sha256-UDqeQd2neUXICpHAZSS965kGCJsfHkrOFS/vl80I7d8=", + "lastModified": 1689885880, + "narHash": "sha256-2ikAcvHKkKh8J/eUrwMA+wy1poscC+oL1RkN1V3RmT8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "28d812a63a0f0d6c1170aac16f5219e506c44b79", + "rev": "fa793b06f56896b7d1909e4b69977c7bf842b2f0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.05", + "ref": "nixos-23.05", "repo": "nixpkgs", "type": "github" } From c350d8b7362eb2f4c89b366e05a6aa08819c30ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:18:37 -0400 Subject: [PATCH 45/54] slightly reduce repetition by pulling out a mergeAttrs definition --- flake.nix | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 5b3102c46..2eb87334c 100644 --- a/flake.nix +++ b/flake.nix @@ -114,6 +114,12 @@ # Make a singleton attribute set from the result of two functions. singletonOf = f: g: x: { ${f x} = g x; }; + # [attrset] -> attrset + # + # Merge a list of attrset into a single attrset with overlap preferring + # rightmost values. + mergeAttrs = pkgs.lib.foldr pkgs.lib.mergeAttrs {}; + # makeRuntimeEnv :: string -> derivation # # Create a derivation that includes a Python runtime, Tahoe-LAFS, and @@ -149,11 +155,11 @@ # flake's "default" package at the derivation corresponding to the # default Python version we defined above. The package consists of a # Python environment with Tahoe-LAFS available to it. - packages = with pkgs.lib; - foldr mergeAttrs {} ([ - { default = self.packages.${system}.${packageName defaultPyVersion}; } - ] ++ (builtins.map makeRuntimeEnv pythonVersions) - ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) + packages = + mergeAttrs ( + [ { default = self.packages.${system}.${packageName defaultPyVersion}; } ] + ++ (builtins.map makeRuntimeEnv pythonVersions) + ++ (builtins.map (singletonOf unitTestName makeTestEnv) pythonVersions) ); # The flake's app outputs. We'll define a version of an app for running @@ -208,13 +214,11 @@ }; }; in - with pkgs.lib; # Merge a default app definition with the rest of the apps. - foldr mergeAttrs - { default = self.apps.${system}."tahoe-python3"; } - ( - (builtins.map makeUnitTestsApp pythonVersions) ++ - (builtins.map makeTahoeApp pythonVersions) - ); + mergeAttrs ( + [ { default = self.apps.${system}."tahoe-python3"; } ] + ++ (builtins.map makeUnitTestsApp pythonVersions) + ++ (builtins.map makeTahoeApp pythonVersions) + ); })); } From b4a6a90e9f56ff7187e9cf8a510797b1cdfd0926 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:19:33 -0400 Subject: [PATCH 46/54] more clarifying comments --- flake.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 2eb87334c..44000c9ab 100644 --- a/flake.nix +++ b/flake.nix @@ -130,6 +130,9 @@ tahoe-lafs.passthru.extras.i2p ++ tahoe-lafs.passthru.extras.tor )).overrideAttrs (old: { + # By default, withPackages gives us a derivation with a fairly generic + # name (like "python-env"). Put our name in there for legibility. + # See the similar override in makeTestEnv. name = packageName pyVersion; }); @@ -144,10 +147,15 @@ tahoe-lafs.passthru.extras.tor ++ tahoe-lafs.passthru.extras.unittest )).overrideAttrs (old: { + # See the similar override in makeRuntimeEnv'. name = packageName pyVersion; }); in { - # A package set with out overlay on it. + # Include a package set with out overlay on it in our own output. This + # is mainly a development/debugging convenience as it will expose all of + # our Python package overrides beneath it. The magic name + # "legacyPackages" is copied from nixpkgs and has special support in the + # nix command line tool. legacyPackages = pkgs; # The flake's package outputs. We'll define one version of the package From 91866154d3d5b120e9ac132f861074cb188960ee Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 21 Jul 2023 11:29:06 -0400 Subject: [PATCH 47/54] expose our cache to anyone who wants it --- flake.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/flake.nix b/flake.nix index 44000c9ab..bde792db3 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,14 @@ { description = "Tahoe-LAFS, free and open decentralized data store"; + nixConfig = { + # Supply configuration for the build cache updated by our CI system. This + # should allow most users to avoid having to build a large number of + # packages (otherwise necessary due to our Python package overrides). + substituters = ["https://tahoe-lafs-opensource.cachix.org"]; + trusted-public-keys = ["tahoe-lafs-opensource.cachix.org-1:eIKCHOPJYceJ2gb74l6e0mayuSdXqiavxYeAio0LFGo="]; + }; + inputs = { # A couple possible nixpkgs pins. Ideally these could be selected easily # from the command line but there seems to be no syntax/support for that. From bf2451bbcdbde50c72b90a20354ae4bc281b7b81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:31:18 -0400 Subject: [PATCH 48/54] Correct type. --- src/allmydata/test/test_storage_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 30f6a527d..52c47c6c3 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -172,7 +172,7 @@ class ExtractSecretsTests(SyncTestCase): ``ClientSecretsException``. """ with self.assertRaises(ClientSecretsException): - _extract_secrets(["FOO eA=="], {}) + _extract_secrets(["FOO eA=="], set()) def test_bad_secret_not_base64(self): """ From 46d10a6281d145f974bb5120120845e1eb6339e9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:31:30 -0400 Subject: [PATCH 49/54] Ensure and test (and necessary refactor) that lack of content-type is same as CBOR content-type, as per spec. --- src/allmydata/storage/http_server.py | 119 ++++++++++++------------ src/allmydata/test/test_storage_http.py | 46 +++++++++ 2 files changed, 108 insertions(+), 57 deletions(-) diff --git a/src/allmydata/storage/http_server.py b/src/allmydata/storage/http_server.py index c63a4ca08..3ff98e933 100644 --- a/src/allmydata/storage/http_server.py +++ b/src/allmydata/storage/http_server.py @@ -530,6 +530,60 @@ def _add_error_handling(app: Klein): return str(failure.value).encode("utf-8") +async def read_encoded( + reactor, request, schema: Schema, max_size: int = 1024 * 1024 +) -> Any: + """ + Read encoded request body data, decoding it with CBOR by default. + + Somewhat arbitrarily, limit body size to 1MiB by default. + """ + content_type = get_content_type(request.requestHeaders) + if content_type is None: + content_type = CBOR_MIME_TYPE + if content_type != CBOR_MIME_TYPE: + raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) + + # Make sure it's not too large: + request.content.seek(0, SEEK_END) + size = request.content.tell() + if size > max_size: + raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) + request.content.seek(0, SEEK_SET) + + # We don't want to load the whole message into memory, cause it might + # be quite large. The CDDL validator takes a read-only bytes-like + # thing. Luckily, for large request bodies twisted.web will buffer the + # data in a file, so we can use mmap() to get a memory view. The CDDL + # validator will not make a copy, so it won't increase memory usage + # beyond that. + try: + fd = request.content.fileno() + except (ValueError, OSError): + fd = -1 + if fd >= 0: + # It's a file, so we can use mmap() to save memory. + message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) + else: + message = request.content.read() + + # Pycddl will release the GIL when validating larger documents, so + # let's take advantage of multiple CPUs: + if size > 10_000: + await defer_to_thread(reactor, schema.validate_cbor, message) + else: + schema.validate_cbor(message) + + # The CBOR parser will allocate more memory, but at least we can feed + # it the file-like object, so that if it's large it won't be make two + # copies. + request.content.seek(SEEK_SET, 0) + # Typically deserialization to Python will not release the GIL, and + # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL + # in the decode path. As such, running it in a different thread has no benefit. + return cbor2.load(request.content) + + class HTTPServer(object): """ A HTTP interface to the storage server. @@ -587,56 +641,6 @@ class HTTPServer(object): # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3861 raise _HTTPError(http.NOT_ACCEPTABLE) - async def _read_encoded( - self, request, schema: Schema, max_size: int = 1024 * 1024 - ) -> Any: - """ - Read encoded request body data, decoding it with CBOR by default. - - Somewhat arbitrarily, limit body size to 1MiB by default. - """ - content_type = get_content_type(request.requestHeaders) - if content_type != CBOR_MIME_TYPE: - raise _HTTPError(http.UNSUPPORTED_MEDIA_TYPE) - - # Make sure it's not too large: - request.content.seek(0, SEEK_END) - size = request.content.tell() - if size > max_size: - raise _HTTPError(http.REQUEST_ENTITY_TOO_LARGE) - request.content.seek(0, SEEK_SET) - - # We don't want to load the whole message into memory, cause it might - # be quite large. The CDDL validator takes a read-only bytes-like - # thing. Luckily, for large request bodies twisted.web will buffer the - # data in a file, so we can use mmap() to get a memory view. The CDDL - # validator will not make a copy, so it won't increase memory usage - # beyond that. - try: - fd = request.content.fileno() - except (ValueError, OSError): - fd = -1 - if fd >= 0: - # It's a file, so we can use mmap() to save memory. - message = mmap.mmap(fd, 0, access=mmap.ACCESS_READ) - else: - message = request.content.read() - - # Pycddl will release the GIL when validating larger documents, so - # let's take advantage of multiple CPUs: - if size > 10_000: - await defer_to_thread(self._reactor, schema.validate_cbor, message) - else: - schema.validate_cbor(message) - - # The CBOR parser will allocate more memory, but at least we can feed - # it the file-like object, so that if it's large it won't be make two - # copies. - request.content.seek(SEEK_SET, 0) - # Typically deserialization to Python will not release the GIL, and - # indeed as of Jan 2023 cbor2 didn't have any code to release the GIL - # in the decode path. As such, running it in a different thread has no benefit. - return cbor2.load(request.content) ##### Generic APIs ##### @@ -677,8 +681,8 @@ class HTTPServer(object): """Allocate buckets.""" upload_secret = authorization[Secrets.UPLOAD] # It's just a list of up to ~256 shares, shouldn't use many bytes. - info = await self._read_encoded( - request, _SCHEMAS["allocate_buckets"], max_size=8192 + info = await read_encoded( + self._reactor, request, _SCHEMAS["allocate_buckets"], max_size=8192 ) # We do NOT validate the upload secret for existing bucket uploads. @@ -849,7 +853,8 @@ class HTTPServer(object): # The reason can be a string with explanation, so in theory it could be # longish? - info = await self._read_encoded( + info = await read_encoded( + self._reactor, request, _SCHEMAS["advise_corrupt_share"], max_size=32768, @@ -868,8 +873,8 @@ class HTTPServer(object): @async_to_deferred async def mutable_read_test_write(self, request, authorization, storage_index): """Read/test/write combined operation for mutables.""" - rtw_request = await self._read_encoded( - request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 + rtw_request = await read_encoded( + self._reactor, request, _SCHEMAS["mutable_read_test_write"], max_size=2**48 ) secrets = ( authorization[Secrets.WRITE_ENABLER], @@ -955,8 +960,8 @@ class HTTPServer(object): # The reason can be a string with explanation, so in theory it could be # longish? - info = await self._read_encoded( - request, _SCHEMAS["advise_corrupt_share"], max_size=32768 + info = await read_encoded( + self._reactor, request, _SCHEMAS["advise_corrupt_share"], max_size=32768 ) self._storage_server.advise_corrupt_share( b"mutable", storage_index, share_number, info["reason"].encode("utf-8") diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 52c47c6c3..48ca072bc 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -42,6 +42,7 @@ from werkzeug.exceptions import NotFound as WNotFound from testtools.matchers import Equals from zope.interface import implementer +from ..util.deferredutil import async_to_deferred from .common import SyncTestCase from ..storage.http_common import ( get_content_type, @@ -59,6 +60,8 @@ from ..storage.http_server import ( _authorized_route, StorageIndexConverter, _add_error_handling, + read_encoded, + _SCHEMAS as SERVER_SCHEMAS, ) from ..storage.http_client import ( StorageClient, @@ -303,6 +306,14 @@ class TestApp(object): request.transport.loseConnection() return Deferred() + @_authorized_route(_app, set(), "/read_body", methods=["POST"]) + @async_to_deferred + async def read_body(self, request, authorization): + data = await read_encoded( + self.clock, request, SERVER_SCHEMAS["advise_corrupt_share"] + ) + return data["reason"] + def result_of(d): """ @@ -320,6 +331,7 @@ def result_of(d): + "This is probably a test design issue." ) + class CustomHTTPServerTests(SyncTestCase): """ Tests that use a custom HTTP server. @@ -504,6 +516,40 @@ class CustomHTTPServerTests(SyncTestCase): result_of(d) self.assertEqual(len(self._http_server.clock.getDelayedCalls()), 0) + def test_request_with_no_content_type_same_as_cbor(self): + """ + If no ``Content-Type`` header is set when sending a body, it is assumed + to be CBOR. + """ + response = result_of( + self.client.request( + "POST", + DecodedURL.from_text("http://127.0.0.1/read_body"), + data=dumps({"reason": "test"}), + ) + ) + self.assertEqual( + result_of(limited_content(response, self._http_server.clock, 100)).read(), + b"test", + ) + + def test_request_with_wrong_content(self): + """ + If a non-CBOR ``Content-Type`` header is set when sending a body, the + server complains appropriatly. + """ + headers = Headers() + headers.setRawHeaders("content-type", ["some/value"]) + response = result_of( + self.client.request( + "POST", + DecodedURL.from_text("http://127.0.0.1/read_body"), + data=dumps({"reason": "test"}), + headers=headers, + ) + ) + self.assertEqual(response.code, http.UNSUPPORTED_MEDIA_TYPE) + @implementer(IReactorFromThreads) class Reactor(Clock): From 7bac6996d18dc9d5f5b48686990a66a50a51021a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:32:14 -0400 Subject: [PATCH 50/54] Updates based on changing specs. --- docs/proposed/http-storage-node-protocol.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5009a992e..fed5bb5dc 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -279,7 +279,7 @@ Such an announcement will resemble this:: { "anonymous-storage-FURL": "pb://...", # The old key - "gbs-anonymous-storage-url": "pb://...#v=1" # The new key + "anonymous-storage-NURLs": ["pb://...#v=1"] # The new keys } The transition process will proceed in three stages: @@ -320,12 +320,7 @@ The follow sequence of events is likely: Ideally, the client would not rely on an update from the introducer to give it the GBS NURL for the updated storage server. -Therefore, -when an updated client connects to a storage server using Foolscap, -it should request the server's version information. -If this information indicates that GBS is supported then the client should cache this GBS information. -On subsequent connection attempts, -it should make use of this GBS information. +In practice, we have decided not to implement this functionality. Server Details -------------- From 792af1c9189e0d9c93d69eb450c680eb8eaa7de1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 25 Jul 2023 15:32:30 -0400 Subject: [PATCH 51/54] News fragment --- newsfragments/4042.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4042.minor diff --git a/newsfragments/4042.minor b/newsfragments/4042.minor new file mode 100644 index 000000000..e69de29bb From ffe2e9773916e32a8a64e9eb0d0e1ae0939c7215 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:54:46 -0400 Subject: [PATCH 52/54] Better phrasing --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index fed5bb5dc..db400fb2b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -278,8 +278,8 @@ This NURL will be announced alongside their existing Foolscap-based server's fUR Such an announcement will resemble this:: { - "anonymous-storage-FURL": "pb://...", # The old key - "anonymous-storage-NURLs": ["pb://...#v=1"] # The new keys + "anonymous-storage-FURL": "pb://...", # The old entry + "anonymous-storage-NURLs": ["pb://...#v=1"] # The new, additional entry } The transition process will proceed in three stages: From 341a32708b718caab6311ed2e517c548e493df35 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:55:45 -0400 Subject: [PATCH 53/54] Docstring. --- src/allmydata/test/test_storage_http.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_storage_http.py b/src/allmydata/test/test_storage_http.py index 48ca072bc..1f17621d7 100644 --- a/src/allmydata/test/test_storage_http.py +++ b/src/allmydata/test/test_storage_http.py @@ -309,6 +309,11 @@ class TestApp(object): @_authorized_route(_app, set(), "/read_body", methods=["POST"]) @async_to_deferred async def read_body(self, request, authorization): + """ + Accept an advise_corrupt_share message, return the reason. + + I.e. exercise codepaths used for reading CBOR from the body. + """ data = await read_encoded( self.clock, request, SERVER_SCHEMAS["advise_corrupt_share"] ) From 3b66afbdeac45ae3050f98b4103536e72a377d8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 1 Aug 2023 10:56:52 -0400 Subject: [PATCH 54/54] Be consistent with HTTPS --- docs/architecture.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.rst b/docs/architecture.rst index 64e81ea99..71ad67305 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -64,7 +64,7 @@ There are two supported protocols: By default HTTPS is disabled (this will change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). When HTTPS is enabled on -the server, the server transparently listens for both Foolscap and HTTP on the +the server, the server transparently listens for both Foolscap and HTTPS on the same port. Clients can use either; by default they will only use Foolscap, but when configured appropriately they will use HTTPS when possible (this will change in https://tahoe-lafs.org/trac/tahoe-lafs/ticket/4041). At this time the